diff options
Diffstat (limited to 'sys/ofed/drivers/net/mlx4/mlx4_en.h')
| -rw-r--r-- | sys/ofed/drivers/net/mlx4/mlx4_en.h | 716 |
1 files changed, 504 insertions, 212 deletions
diff --git a/sys/ofed/drivers/net/mlx4/mlx4_en.h b/sys/ofed/drivers/net/mlx4/mlx4_en.h index 0dc5fe6..9a87bf7 100644 --- a/sys/ofed/drivers/net/mlx4/mlx4_en.h +++ b/sys/ofed/drivers/net/mlx4/mlx4_en.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2007 Mellanox Technologies. All rights reserved. + * Copyright (c) 2007, 2014 Mellanox Technologies. All rights reserved. * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU @@ -34,13 +34,17 @@ #ifndef _MLX4_EN_H_ #define _MLX4_EN_H_ -#include <sys/cdefs.h> - -#include <linux/types.h> +#include <linux/bitops.h> #include <linux/compiler.h> #include <linux/list.h> #include <linux/mutex.h> +#include <linux/kobject.h> #include <linux/netdevice.h> +#include <linux/if_vlan.h> +#include <linux/if_ether.h> +#ifdef CONFIG_MLX4_EN_DCB +#include <linux/dcbnl.h> +#endif #include <linux/mlx4/device.h> #include <linux/mlx4/qp.h> @@ -49,56 +53,17 @@ #include <linux/mlx4/doorbell.h> #include <linux/mlx4/cmd.h> -#include <net/if_media.h> #include <netinet/tcp_lro.h> #include "en_port.h" +#include "mlx4_stats.h" #define DRV_NAME "mlx4_en" -#define DRV_VERSION "1.5.2" -#define DRV_RELDATE "July 2010" - -/* XXX */ -#define NETIF_MSG_LINK 0x1 -#define NETIF_MSG_IFDOWN 0x2 -#define NETIF_MSG_HW 0x4 -#define NETIF_MSG_DRV 0x8 -#define NETIF_MSG_INTR 0x10 -#define NETIF_MSG_RX_ERR 0x20 +#define DRV_VERSION "2.1" +#define DRV_RELDATE __DATE__ #define MLX4_EN_MSG_LEVEL (NETIF_MSG_LINK | NETIF_MSG_IFDOWN) -#define en_print(level, priv, format, arg...) \ - { \ - if ((priv)->registered) \ - printk(level "%s: %s: " format, DRV_NAME, \ - (priv->dev)->if_xname, ## arg); \ - else \ - printk(level "%s: %s: Port %d: " format, \ - DRV_NAME, dev_name(&priv->mdev->pdev->dev), \ - (priv)->port, ## arg); \ - } - -#define en_dbg(mlevel, priv, format, arg...) \ - if (NETIF_MSG_##mlevel & priv->msg_enable) \ - en_print(KERN_DEBUG, priv, format, ## arg) -#define en_warn(priv, format, arg...) \ - en_print(KERN_WARNING, priv, format, ## arg) -#define en_err(priv, format, arg...) \ - en_print(KERN_ERR, priv, format, ## arg) -#define en_info(priv, format, arg...) \ - en_print(KERN_INFO, priv, format, ## arg) - -#define mlx4_err(mdev, format, arg...) \ - printk(KERN_ERR "%s %s: " format , DRV_NAME ,\ - dev_name(&mdev->pdev->dev) , ## arg) -#define mlx4_info(mdev, format, arg...) \ - printk(KERN_INFO "%s %s: " format , DRV_NAME ,\ - dev_name(&mdev->pdev->dev) , ## arg) -#define mlx4_warn(mdev, format, arg...) \ - printk(KERN_WARNING "%s %s: " format , DRV_NAME ,\ - dev_name(&mdev->pdev->dev) , ## arg) - /* * Device constants */ @@ -106,8 +71,9 @@ #define MLX4_EN_PAGE_SHIFT 12 #define MLX4_EN_PAGE_SIZE (1 << MLX4_EN_PAGE_SHIFT) -#define MAX_TX_RINGS (MLX4_EN_NUM_HASH_RINGS + 1 + MLX4_EN_NUM_PPP_RINGS) -#define MAX_RX_RINGS 16 +#define DEF_RX_RINGS 16 +#define MAX_RX_RINGS 128 +#define MIN_RX_RINGS 4 #define TXBB_SIZE 64 #define HEADROOM (2048 / TXBB_SIZE + 1) #define STAMP_STRIDE 64 @@ -115,6 +81,19 @@ #define STAMP_SHIFT 31 #define STAMP_VAL 0x7fffffff #define STATS_DELAY (HZ / 4) +#define SERVICE_TASK_DELAY (HZ / 4) +#define MAX_NUM_OF_FS_RULES 256 + +#define MLX4_EN_FILTER_HASH_SHIFT 4 +#define MLX4_EN_FILTER_EXPIRY_QUOTA 60 + +#ifdef CONFIG_NET_RX_BUSY_POLL +#define LL_EXTENDED_STATS +#endif + +/* vlan valid range */ +#define VLAN_MIN_VALUE 1 +#define VLAN_MAX_VALUE 4094 /* Typical TSO descriptor with 16 gather entries is 352 bytes... */ #define MAX_DESC_SIZE 512 @@ -126,8 +105,13 @@ #define MLX4_EN_WATCHDOG_TIMEOUT (15 * HZ) -#define MLX4_EN_MAX_LRO_DESCRIPTORS 32 -#define MLX4_EN_NUM_IPFRAG_SESSIONS 16 +#define MLX4_EN_ALLOC_SIZE PAGE_ALIGN(PAGE_SIZE) +#define MLX4_EN_ALLOC_ORDER get_order(MLX4_EN_ALLOC_SIZE) + +enum mlx4_en_alloc_type { + MLX4_EN_ALLOC_NEW = 0, + MLX4_EN_ALLOC_REPLACEMENT = 1, +}; /* Receive fragment sizes; we use at most 3 fragments (for 9600 byte MTU * and 4K allocations) */ @@ -150,28 +134,37 @@ enum { #endif /* Maximum ring sizes */ +#define MLX4_EN_DEF_TX_QUEUE_SIZE 4096 + +/* Minimum packet number till arming the CQ */ +#define MLX4_EN_MIN_RX_ARM 2048 +#define MLX4_EN_MIN_TX_ARM 2048 + +/* Maximum ring sizes */ #define MLX4_EN_MAX_TX_SIZE 8192 #define MLX4_EN_MAX_RX_SIZE 8192 -#define MLX4_EN_MIN_RX_SIZE (128) +/* Minimum ring sizes */ +#define MLX4_EN_MIN_RX_SIZE (4096 / TXBB_SIZE) #define MLX4_EN_MIN_TX_SIZE (4096 / TXBB_SIZE) #define MLX4_EN_SMALL_PKT_SIZE 64 -#define MLX4_EN_TX_HASH_SIZE 256 -#define MLX4_EN_TX_HASH_MASK (MLX4_EN_TX_HASH_SIZE - 1) -#define MLX4_EN_NUM_HASH_RINGS 4 -#define MLX4_EN_NUM_PPP_RINGS 8 -#define MLX4_EN_DEF_TX_RING_SIZE 512 -#define MLX4_EN_DEF_TX_QUEUE_SIZE 4096 + +#define MLX4_EN_MAX_TX_RING_P_UP 32 +#define MLX4_EN_NUM_UP 1 + +#define MAX_TX_RINGS (MLX4_EN_MAX_TX_RING_P_UP * \ + (MLX4_EN_NUM_UP + 1)) + +#define MLX4_EN_DEF_TX_RING_SIZE 1024 #define MLX4_EN_DEF_RX_RING_SIZE 1024 -#define MLX4_EN_MAX_RX_POLL 1024 /* Target number of bytes to coalesce with interrupt moderation */ #define MLX4_EN_RX_COAL_TARGET 0x20000 #define MLX4_EN_RX_COAL_TIME 0x10 -#define MLX4_EN_TX_COAL_PKTS 5 -#define MLX4_EN_TX_COAL_TIME 0x80 +#define MLX4_EN_TX_COAL_PKTS 64 +#define MLX4_EN_TX_COAL_TIME 64 #define MLX4_EN_RX_RATE_LOW 400000 #define MLX4_EN_RX_COAL_TIME_LOW 0 @@ -187,14 +180,13 @@ enum { #define MLX4_EN_DEF_RX_PAUSE 1 #define MLX4_EN_DEF_TX_PAUSE 1 -/* Interval between sucessive polls in the Tx routine when polling is used +/* Interval between successive polls in the Tx routine when polling is used instead of interrupts (in per-core Tx rings) - should be power of 2 */ #define MLX4_EN_TX_POLL_MODER 16 #define MLX4_EN_TX_POLL_TIMEOUT (HZ / 4) -#define ETH_LLC_SNAP_SIZE 8 - -#define SMALL_PACKET_SIZE (MHLEN) +#define MLX4_EN_64_ALIGN (64 - NET_SKB_PAD) +#define SMALL_PACKET_SIZE (256 - NET_IP_ALIGN) #define HEADER_COPY_SIZE (128) #define MLX4_LOOPBACK_TEST_PAYLOAD (HEADER_COPY_SIZE - ETHER_HDR_LEN) @@ -208,7 +200,6 @@ enum { /* Number of samples to 'average' */ #define AVG_SIZE 128 #define AVG_FACTOR 1024 -#define NUM_PERF_STATS NUM_PERF_COUNTERS #define INC_PERF_COUNTER(cnt) (++(cnt)) #define ADD_PERF_COUNTER(cnt, add) ((cnt) += (add)) @@ -219,7 +210,6 @@ enum { #else -#define NUM_PERF_STATS 0 #define INC_PERF_COUNTER(cnt) do {} while (0) #define ADD_PERF_COUNTER(cnt, add) do {} while (0) #define AVG_PERF_COUNTER(cnt, sample) do {} while (0) @@ -244,13 +234,17 @@ enum cq_type { #define XNOR(x, y) (!(x) == !(y)) #define ILLEGAL_MAC(addr) (addr == 0xffffffffffffULL || addr == 0x0) - struct mlx4_en_tx_info { - struct mbuf *mb; - u32 nr_txbb; - u8 nr_segs; - u8 data_offset; - u8 inl; + struct mbuf *mb; + u32 nr_txbb; + u32 nr_bytes; + u8 linear; + u8 nr_segs; + u8 data_offset; + u8 inl; +#if 0 + u8 ts_requested; +#endif }; @@ -271,8 +265,14 @@ struct mlx4_en_tx_desc { #define MLX4_EN_USE_SRQ 0x01000000 +#define MLX4_EN_TX_BUDGET 64*4 //Compensate for no NAPI in freeBSD - might need some fine tunning in the future. +#define MLX4_EN_RX_BUDGET 64 + +#define MLX4_EN_CX3_LOW_ID 0x1000 +#define MLX4_EN_CX3_HIGH_ID 0x1005 + struct mlx4_en_tx_ring { - spinlock_t tx_lock; + spinlock_t tx_lock; struct mlx4_hwq_resources wqres; u32 size ; /* number of TXBBs */ u32 size_mask; @@ -285,9 +285,11 @@ struct mlx4_en_tx_ring { void *buf; u16 poll_cnt; int blocked; - struct buf_ring *br; struct mlx4_en_tx_info *tx_info; u8 *bounce_buf; + u8 queue_index; + cpuset_t affinity_mask; + struct buf_ring *br; u32 last_nr_txbb; struct mlx4_qp qp; struct mlx4_qp_context context; @@ -296,29 +298,30 @@ struct mlx4_en_tx_ring { struct mlx4_srq dummy; unsigned long bytes; unsigned long packets; - unsigned long errors; - spinlock_t comp_lock; + unsigned long tx_csum; + unsigned long queue_stopped; + unsigned long wake_queue; struct mlx4_bf bf; bool bf_enabled; + struct netdev_queue *tx_queue; + int hwtstamp_tx_type; + spinlock_t comp_lock; + int full_size; + int inline_thold; u64 watchdog_time; }; -struct mlx4_en_ipfrag { - struct mbuf *fragments; - struct mbuf *last; - __be32 saddr; - __be32 daddr; - __be16 id; - u8 protocol; - int total_len; - u16 offset; -}; - struct mlx4_en_rx_desc { /* actual number of entries depends on rx ring stride */ struct mlx4_wqe_data_seg data[0]; }; +struct mlx4_en_rx_buf { + dma_addr_t dma; + struct page *page; + unsigned int page_offset; +}; + struct mlx4_en_rx_ring { struct mlx4_hwq_resources wqres; u32 size ; /* number of Rx descs*/ @@ -330,38 +333,66 @@ struct mlx4_en_rx_ring { u32 prod; u32 cons; u32 buf_size; + u8 fcs_del; + u16 rx_alloc_order; + u32 rx_alloc_size; + u32 rx_buf_size; + u32 rx_mb_size; + int qpn; void *buf; void *rx_info; + unsigned long errors; unsigned long bytes; unsigned long packets; - unsigned long errors; +#ifdef LL_EXTENDED_STATS + unsigned long yields; + unsigned long misses; + unsigned long cleaned; +#endif + unsigned long csum_ok; + unsigned long csum_none; + int hwtstamp_rx_filter; + int numa_node; struct lro_ctrl lro; - struct mlx4_en_ipfrag ipfrag[MLX4_EN_NUM_IPFRAG_SESSIONS]; }; - static inline int mlx4_en_can_lro(__be16 status) { - return (status & cpu_to_be16(MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPV4F | - MLX4_CQE_STATUS_IPV6 | - MLX4_CQE_STATUS_IPV4OPT | - MLX4_CQE_STATUS_TCP | - MLX4_CQE_STATUS_UDP | - MLX4_CQE_STATUS_IPOK)) == - cpu_to_be16(MLX4_CQE_STATUS_IPV4 | - MLX4_CQE_STATUS_IPOK | - MLX4_CQE_STATUS_TCP); + static __be16 status_all; + static __be16 status_ipv4_ipok_tcp; + static __be16 status_ipv6_ipok_tcp; + + status_all = cpu_to_be16( + MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_IPV4F | + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_IPV4OPT | + MLX4_CQE_STATUS_TCP | + MLX4_CQE_STATUS_UDP | + MLX4_CQE_STATUS_IPOK); + status_ipv4_ipok_tcp = cpu_to_be16( + MLX4_CQE_STATUS_IPV4 | + MLX4_CQE_STATUS_IPOK | + MLX4_CQE_STATUS_TCP); + status_ipv6_ipok_tcp = cpu_to_be16( + MLX4_CQE_STATUS_IPV6 | + MLX4_CQE_STATUS_IPOK | + MLX4_CQE_STATUS_TCP); + + status &= status_all; + return (status == status_ipv4_ipok_tcp || + status == status_ipv6_ipok_tcp); } + struct mlx4_en_cq { struct mlx4_cq mcq; struct mlx4_hwq_resources wqres; int ring; spinlock_t lock; struct net_device *dev; - /* Per-core Tx cq processing support */ - struct timer_list timer; + /* Per-core Tx cq processing support */ + struct timer_list timer; int size; int buf_size; unsigned vector; @@ -373,6 +404,20 @@ struct mlx4_en_cq { struct taskqueue *tq; #define MLX4_EN_OPCODE_ERROR 0x1e u32 tot_rx; + u32 tot_tx; + +#ifdef CONFIG_NET_RX_BUSY_POLL + unsigned int state; +#define MLX4_EN_CQ_STATEIDLE 0 +#define MLX4_EN_CQ_STATENAPI 1 /* NAPI owns this CQ */ +#define MLX4_EN_CQ_STATEPOLL 2 /* poll owns this CQ */ +#define MLX4_CQ_LOCKED (MLX4_EN_CQ_STATENAPI | MLX4_EN_CQ_STATEPOLL) +#define MLX4_EN_CQ_STATENAPI_YIELD 4 /* NAPI yielded this CQ */ +#define MLX4_EN_CQ_STATEPOLL_YIELD 8 /* poll yielded this CQ */ +#define CQ_YIELD (MLX4_EN_CQ_STATENAPI_YIELD | MLX4_EN_CQ_STATEPOLL_YIELD) +#define CQ_USER_PEND (MLX4_EN_CQ_STATEPOLL | MLX4_EN_CQ_STATEPOLL_YIELD) + spinlock_t poll_lock; /* protects from LLS/napi conflicts */ +#endif /* CONFIG_NET_RX_BUSY_POLL */ }; struct mlx4_en_port_profile { @@ -382,41 +427,42 @@ struct mlx4_en_port_profile { u32 tx_ring_size; u32 rx_ring_size; u8 rx_pause; + u8 rx_ppp; u8 tx_pause; - u32 rx_ppp; - u32 tx_ppp; + u8 tx_ppp; + int rss_rings; }; struct mlx4_en_profile { int rss_xor; - int num_lro; - int ip_reasm; - int tcp_rss; int udp_rss; u8 rss_mask; u32 active_ports; u32 small_pkt_int; u8 no_reset; + u8 num_tx_rings_p_up; struct mlx4_en_port_profile prof[MLX4_MAX_PORTS + 1]; }; struct mlx4_en_dev { - struct mlx4_dev *dev; + struct mlx4_dev *dev; struct pci_dev *pdev; struct mutex state_lock; - struct net_device *pndev[MLX4_MAX_PORTS + 1]; - u32 port_cnt; + struct net_device *pndev[MLX4_MAX_PORTS + 1]; + u32 port_cnt; bool device_up; - struct mlx4_en_profile profile; + struct mlx4_en_profile profile; u32 LSO_support; struct workqueue_struct *workqueue; - struct device *dma_device; - void __iomem *uar_map; - struct mlx4_uar priv_uar; + struct device *dma_device; + void __iomem *uar_map; + struct mlx4_uar priv_uar; struct mlx4_mr mr; - u32 priv_pdn; - spinlock_t uar_lock; + u32 priv_pdn; + spinlock_t uar_lock; u8 mac_removed[MLX4_MAX_PORTS + 1]; + unsigned long last_overflow_check; + unsigned long overflow_period; }; @@ -428,71 +474,76 @@ struct mlx4_en_rss_map { enum mlx4_qp_state indir_state; }; -struct mlx4_en_rss_context { - __be32 base_qpn; - __be32 default_qpn; - u16 reserved; - u8 hash_fn; - u8 flags; - __be32 rss_key[10]; - __be32 base_qpn_udp; -}; - struct mlx4_en_port_state { int link_state; int link_speed; int transciver; + int autoneg; }; -struct mlx4_en_pkt_stats { - unsigned long broadcast; - unsigned long rx_prio[8]; - unsigned long tx_prio[8]; -#define NUM_PKT_STATS 17 +enum mlx4_en_mclist_act { + MCLIST_NONE, + MCLIST_REM, + MCLIST_ADD, }; -struct mlx4_en_port_stats { - unsigned long tso_packets; - unsigned long queue_stopped; - unsigned long wake_queue; - unsigned long tx_timeout; - unsigned long rx_alloc_failed; - unsigned long rx_chksum_good; - unsigned long rx_chksum_none; - unsigned long tx_chksum_offload; +struct mlx4_en_mc_list { + struct list_head list; + enum mlx4_en_mclist_act action; + u8 addr[ETH_ALEN]; + u64 reg_id; }; -struct mlx4_en_perf_stats { - u32 tx_poll; - u64 tx_pktsz_avg; - u32 inflight_avg; - u32 tx_coal_avg; - u32 rx_coal_avg; +#ifdef CONFIG_MLX4_EN_DCB +/* Minimal TC BW - setting to 0 will block traffic */ +#define MLX4_EN_BW_MIN 1 +#define MLX4_EN_BW_MAX 100 /* Utilize 100% of the line */ + +#define MLX4_EN_TC_ETS 7 + +#endif + + +enum { + MLX4_EN_FLAG_PROMISC = (1 << 0), + MLX4_EN_FLAG_MC_PROMISC = (1 << 1), + /* whether we need to enable hardware loopback by putting dmac + * in Tx WQE + */ + MLX4_EN_FLAG_ENABLE_HW_LOOPBACK = (1 << 2), + /* whether we need to drop packets that hardware loopback-ed */ + MLX4_EN_FLAG_RX_FILTER_NEEDED = (1 << 3), + MLX4_EN_FLAG_FORCE_PROMISC = (1 << 4), +#ifdef CONFIG_MLX4_EN_DCB + MLX4_EN_FLAG_DCB_ENABLED = (1 << 5) +#endif }; -struct mlx4_en_frag_info { - u16 frag_size; - u16 frag_prefix_size; +#define MLX4_EN_MAC_HASH_SIZE (1 << BITS_PER_BYTE) +#define MLX4_EN_MAC_HASH_IDX 5 + +struct en_port { + struct kobject kobj; + struct mlx4_dev *dev; + u8 port_num; + u8 vport_num; }; -struct mlx4_en_tx_hash_entry { - u8 cnt; - unsigned int small_pkts; - unsigned int big_pkts; - unsigned int ring; +struct mlx4_en_frag_info { + u16 frag_size; + u16 frag_prefix_size; }; + struct mlx4_en_priv { struct mlx4_en_dev *mdev; struct mlx4_en_port_profile *prof; struct net_device *dev; - bool vlgrp_modified; - u32 vlan_register[VLAN_FLTR_SIZE]; - u32 vlan_unregister[VLAN_FLTR_SIZE]; - u32 vlans[VLAN_FLTR_SIZE]; - spinlock_t vlan_lock; + unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; struct mlx4_en_port_state port_state; spinlock_t stats_lock; + /* To allow rules removal while port is going down */ + struct list_head ethtool_list; unsigned long last_moder_packets[MAX_RX_RINGS]; unsigned long last_moder_tx_packets; @@ -520,139 +571,380 @@ struct mlx4_en_priv { int port; int registered; int allocated; - int rx_csum; - u64 mac; + int stride; + unsigned char current_mac[ETH_ALEN + 2]; + u64 mac; int mac_index; unsigned max_mtu; int base_qpn; + int cqe_factor; struct mlx4_en_rss_map rss_map; - u16 tx_prio_map[8]; + __be32 ctrl_flags; u32 flags; -#define MLX4_EN_FLAG_PROMISC 0x1 + u8 num_tx_rings_p_up; u32 tx_ring_num; u32 rx_ring_num; u32 rx_mb_size; - struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS]; - u16 num_frags; + struct mlx4_en_frag_info frag_info[MLX4_EN_MAX_RX_FRAGS]; + u16 rx_alloc_order; + u32 rx_alloc_size; + u32 rx_buf_size; + u16 num_frags; u16 log_rx_info; - int ip_reasm; - bool wol; - struct mlx4_en_tx_ring tx_ring[MAX_TX_RINGS]; - struct mlx4_en_rx_ring rx_ring[MAX_RX_RINGS]; - struct mlx4_en_cq tx_cq[MAX_TX_RINGS]; - struct mlx4_en_cq rx_cq[MAX_RX_RINGS]; - struct mlx4_en_tx_hash_entry tx_hash[MLX4_EN_TX_HASH_SIZE]; - struct work_struct mcast_task; - struct work_struct start_port_task; - struct work_struct stop_port_task; + struct mlx4_en_tx_ring **tx_ring; + struct mlx4_en_rx_ring *rx_ring[MAX_RX_RINGS]; + struct mlx4_en_cq **tx_cq; + struct mlx4_en_cq *rx_cq[MAX_RX_RINGS]; + struct mlx4_qp drop_qp; + struct work_struct rx_mode_task; struct work_struct watchdog_task; struct work_struct linkstate_task; struct delayed_work stats_task; + struct delayed_work service_task; struct mlx4_en_perf_stats pstats; struct mlx4_en_pkt_stats pkstats; + struct mlx4_en_flow_stats flowstats[MLX4_NUM_PRIORITIES]; struct mlx4_en_port_stats port_stats; + struct mlx4_en_vport_stats vport_stats; + struct mlx4_en_vf_stats vf_stats; + DECLARE_BITMAP(stats_bitmap, NUM_ALL_STATS); + struct list_head mc_list; + struct list_head curr_list; + u64 broadcast_id; struct mlx4_en_stat_out_mbox hw_stats; - struct ifmedia media; + int vids[128]; + bool wol; + struct device *ddev; + struct dentry *dev_root; + u32 counter_index; eventhandler_tag vlan_attach; eventhandler_tag vlan_detach; struct callout watchdog_timer; + struct ifmedia media; volatile int blocked; struct sysctl_oid *sysctl; struct sysctl_ctx_list conf_ctx; struct sysctl_ctx_list stat_ctx; +#define MLX4_EN_MAC_HASH_IDX 5 + struct hlist_head mac_hash[MLX4_EN_MAC_HASH_SIZE]; + +#ifdef CONFIG_MLX4_EN_DCB + struct ieee_ets ets; + u16 maxrate[IEEE_8021QAZ_MAX_TCS]; + u8 dcbx_cap; +#endif +#ifdef CONFIG_RFS_ACCEL + spinlock_t filters_lock; + int last_filter_id; + struct list_head filters; + struct hlist_head filter_hash[1 << MLX4_EN_FILTER_HASH_SHIFT]; +#endif + struct en_port *vf_ports[MLX4_MAX_NUM_VF]; + unsigned long last_ifq_jiffies; + u64 if_counters_rx_errors; + u64 if_counters_rx_no_buffer; + }; enum mlx4_en_wol { MLX4_EN_WOL_MAGIC = (1ULL << 61), MLX4_EN_WOL_ENABLED = (1ULL << 62), - MLX4_EN_WOL_DO_MODIFY = (1ULL << 63), }; -int mlx4_en_transmit(struct net_device *dev, struct mbuf *mb); -void mlx4_en_qflush(struct net_device *dev); +struct mlx4_mac_entry { + struct hlist_node hlist; + unsigned char mac[ETH_ALEN + 2]; + u64 reg_id; +}; + +#ifdef CONFIG_NET_RX_BUSY_POLL +static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq) +{ + spin_lock_init(&cq->poll_lock); + cq->state = MLX4_EN_CQ_STATEIDLE; +} + +/* called from the device poll rutine to get ownership of a cq */ +static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq) +{ + int rc = true; + spin_lock(&cq->poll_lock); + if (cq->state & MLX4_CQ_LOCKED) { + WARN_ON(cq->state & MLX4_EN_CQ_STATENAPI); + cq->state |= MLX4_EN_CQ_STATENAPI_YIELD; + rc = false; + } else + /* we don't care if someone yielded */ + cq->state = MLX4_EN_CQ_STATENAPI; + spin_unlock(&cq->poll_lock); + return rc; +} + +/* returns true is someone tried to get the cq while napi had it */ +static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq) +{ + int rc = false; + spin_lock(&cq->poll_lock); + WARN_ON(cq->state & (MLX4_EN_CQ_STATEPOLL | + MLX4_EN_CQ_STATENAPI_YIELD)); + + if (cq->state & MLX4_EN_CQ_STATEPOLL_YIELD) + rc = true; + cq->state = MLX4_EN_CQ_STATEIDLE; + spin_unlock(&cq->poll_lock); + return rc; +} + +/* called from mlx4_en_low_latency_poll() */ +static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq) +{ + int rc = true; + spin_lock_bh(&cq->poll_lock); + if ((cq->state & MLX4_CQ_LOCKED)) { + struct net_device *dev = cq->dev; + struct mlx4_en_priv *priv = netdev_priv(dev); + struct mlx4_en_rx_ring *rx_ring = priv->rx_ring[cq->ring]; + + cq->state |= MLX4_EN_CQ_STATEPOLL_YIELD; + rc = false; +#ifdef LL_EXTENDED_STATS + rx_ring->yields++; +#endif + } else + /* preserve yield marks */ + cq->state |= MLX4_EN_CQ_STATEPOLL; + spin_unlock_bh(&cq->poll_lock); + return rc; +} + +/* returns true if someone tried to get the cq while it was locked */ +static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq) +{ + int rc = false; + spin_lock_bh(&cq->poll_lock); + WARN_ON(cq->state & (MLX4_EN_CQ_STATENAPI)); + + if (cq->state & MLX4_EN_CQ_STATEPOLL_YIELD) + rc = true; + cq->state = MLX4_EN_CQ_STATEIDLE; + spin_unlock_bh(&cq->poll_lock); + return rc; +} + +/* true if a socket is polling, even if it did not get the lock */ +static inline bool mlx4_en_cq_ll_polling(struct mlx4_en_cq *cq) +{ + WARN_ON(!(cq->state & MLX4_CQ_LOCKED)); + return cq->state & CQ_USER_PEND; +} +#else +static inline void mlx4_en_cq_init_lock(struct mlx4_en_cq *cq) +{ +} + +static inline bool mlx4_en_cq_lock_napi(struct mlx4_en_cq *cq) +{ + return true; +} + +static inline bool mlx4_en_cq_unlock_napi(struct mlx4_en_cq *cq) +{ + return false; +} + +static inline bool mlx4_en_cq_lock_poll(struct mlx4_en_cq *cq) +{ + return false; +} + +static inline bool mlx4_en_cq_unlock_poll(struct mlx4_en_cq *cq) +{ + return false; +} + +static inline bool mlx4_en_cq_ll_polling(struct mlx4_en_cq *cq) +{ + return false; +} +#endif /* CONFIG_NET_RX_BUSY_POLL */ + +#define MLX4_EN_WOL_DO_MODIFY (1ULL << 63) -int mlx4_en_rx_frags(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring, - struct mbuf *mb, struct mlx4_cqe *cqe); -void mlx4_en_flush_frags(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring); void mlx4_en_destroy_netdev(struct net_device *dev); int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, struct mlx4_en_port_profile *prof); -int mlx4_en_do_start_port(struct net_device *dev); -void mlx4_en_do_stop_port(struct net_device *dev); +int mlx4_en_start_port(struct net_device *dev); +void mlx4_en_stop_port(struct net_device *dev); void mlx4_en_free_resources(struct mlx4_en_priv *priv); int mlx4_en_alloc_resources(struct mlx4_en_priv *priv); -int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, - int entries, int ring, enum cq_type mode); -void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); -int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); +int mlx4_en_pre_config(struct mlx4_en_priv *priv); +int mlx4_en_create_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq, + int entries, int ring, enum cq_type mode, int node); +void mlx4_en_destroy_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq **pcq); +int mlx4_en_activate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq, + int cq_idx); void mlx4_en_deactivate_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); int mlx4_en_set_cq_moder(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); int mlx4_en_arm_cq(struct mlx4_en_priv *priv, struct mlx4_en_cq *cq); -void mlx4_en_poll_tx_cq(unsigned long data); void mlx4_en_tx_irq(struct mlx4_cq *mcq); u16 mlx4_en_select_queue(struct net_device *dev, struct mbuf *mb); -int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring, - u32 size, u16 stride); -void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring); +int mlx4_en_transmit(struct ifnet *dev, struct mbuf *m); +int mlx4_en_create_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring **pring, + u32 size, u16 stride, int node, int queue_idx); +void mlx4_en_destroy_tx_ring(struct mlx4_en_priv *priv, + struct mlx4_en_tx_ring **pring); int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring, - int cq); + int cq, int user_prio); void mlx4_en_deactivate_tx_ring(struct mlx4_en_priv *priv, struct mlx4_en_tx_ring *ring); +void mlx4_en_qflush(struct ifnet *dev); int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring, u32 size); + struct mlx4_en_rx_ring **pring, + u32 size, int node); void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv, - struct mlx4_en_rx_ring *ring); + struct mlx4_en_rx_ring **pring, + u32 size, u16 stride); +void mlx4_en_tx_que(void *context, int pending); +void mlx4_en_rx_que(void *context, int pending); int mlx4_en_activate_rx_rings(struct mlx4_en_priv *priv); void mlx4_en_deactivate_rx_ring(struct mlx4_en_priv *priv, struct mlx4_en_rx_ring *ring); int mlx4_en_process_rx_cq(struct net_device *dev, struct mlx4_en_cq *cq, int budget); -int mlx4_en_process_rx_cq_mb(struct net_device *dev, - struct mlx4_en_cq *cq, - int budget); -void mlx4_en_tx_que(void *context, int pending); -void mlx4_en_rx_que(void *context, int pending); +void mlx4_en_poll_tx_cq(unsigned long data); void mlx4_en_fill_qp_context(struct mlx4_en_priv *priv, int size, int stride, - int is_tx, int rss, int qpn, int cqn, - struct mlx4_qp_context *context); + int is_tx, int rss, int qpn, int cqn, int user_prio, + struct mlx4_qp_context *context); void mlx4_en_sqp_event(struct mlx4_qp *qp, enum mlx4_event event); int mlx4_en_map_buffer(struct mlx4_buf *buf); void mlx4_en_unmap_buffer(struct mlx4_buf *buf); - void mlx4_en_calc_rx_buf(struct net_device *dev); -void mlx4_en_set_prio_map(struct mlx4_en_priv *priv, u16 *prio_map, u32 ring_num); + int mlx4_en_config_rss_steer(struct mlx4_en_priv *priv); void mlx4_en_release_rss_steer(struct mlx4_en_priv *priv); +int mlx4_en_create_drop_qp(struct mlx4_en_priv *priv); +void mlx4_en_destroy_drop_qp(struct mlx4_en_priv *priv); int mlx4_en_free_tx_buf(struct net_device *dev, struct mlx4_en_tx_ring *ring); void mlx4_en_rx_irq(struct mlx4_cq *mcq); -//int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode); -int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, u8 port, u32 *vlans); -//int mlx4_SET_PORT_general(struct mlx4_dev *dev, u8 port, int mtu, -// u8 pptx, u8 pfctx, u8 pprx, u8 pfcrx); -//int mlx4_SET_PORT_qpn_calc(struct mlx4_dev *dev, u8 port, u32 base_qpn, -// u8 promisc); +int mlx4_SET_MCAST_FLTR(struct mlx4_dev *dev, u8 port, u64 mac, u64 clear, u8 mode); +int mlx4_SET_VLAN_FLTR(struct mlx4_dev *dev, struct mlx4_en_priv *priv); int mlx4_en_DUMP_ETH_STATS(struct mlx4_en_dev *mdev, u8 port, u8 reset); int mlx4_en_QUERY_PORT(struct mlx4_en_dev *mdev, u8 port); +int mlx4_en_get_vport_stats(struct mlx4_en_dev *mdev, u8 port); +void mlx4_en_create_debug_files(struct mlx4_en_priv *priv); +void mlx4_en_delete_debug_files(struct mlx4_en_priv *priv); +int mlx4_en_register_debugfs(void); +void mlx4_en_unregister_debugfs(void); + +#ifdef CONFIG_MLX4_EN_DCB +extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_ops; +extern const struct dcbnl_rtnl_ops mlx4_en_dcbnl_pfc_ops; +#endif + +int mlx4_en_setup_tc(struct net_device *dev, u8 up); + +#ifdef CONFIG_RFS_ACCEL +void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv, + struct mlx4_en_rx_ring *rx_ring); +#endif #define MLX4_EN_NUM_SELF_TEST 5 void mlx4_en_ex_selftest(struct net_device *dev, u32 *flags, u64 *buf); -u64 mlx4_en_mac_to_u64(u8 *addr); +void mlx4_en_ptp_overflow_check(struct mlx4_en_dev *mdev); + +/* + * Functions for time stamping + */ +#define SKBTX_HW_TSTAMP (1 << 0) +#define SKBTX_IN_PROGRESS (1 << 2) + +u64 mlx4_en_get_cqe_ts(struct mlx4_cqe *cqe); + +/* Functions for caching and restoring statistics */ +int mlx4_en_get_sset_count(struct net_device *dev, int sset); +void mlx4_en_restore_ethtool_stats(struct mlx4_en_priv *priv, + u64 *data); /* * Globals */ extern const struct ethtool_ops mlx4_en_ethtool_ops; + +/* + * Defines for link speed - needed by selftest + */ +#define MLX4_EN_LINK_SPEED_1G 1000 +#define MLX4_EN_LINK_SPEED_10G 10000 +#define MLX4_EN_LINK_SPEED_40G 40000 + +enum { + NETIF_MSG_DRV = 0x0001, + NETIF_MSG_PROBE = 0x0002, + NETIF_MSG_LINK = 0x0004, + NETIF_MSG_TIMER = 0x0008, + NETIF_MSG_IFDOWN = 0x0010, + NETIF_MSG_IFUP = 0x0020, + NETIF_MSG_RX_ERR = 0x0040, + NETIF_MSG_TX_ERR = 0x0080, + NETIF_MSG_TX_QUEUED = 0x0100, + NETIF_MSG_INTR = 0x0200, + NETIF_MSG_TX_DONE = 0x0400, + NETIF_MSG_RX_STATUS = 0x0800, + NETIF_MSG_PKTDATA = 0x1000, + NETIF_MSG_HW = 0x2000, + NETIF_MSG_WOL = 0x4000, +}; + + +/* + * printk / logging functions + */ + +#define en_print(level, priv, format, arg...) \ + { \ + if ((priv)->registered) \ + printk(level "%s: %s: " format, DRV_NAME, \ + (priv->dev)->if_xname, ## arg); \ + else \ + printk(level "%s: %s: Port %d: " format, \ + DRV_NAME, dev_name(&priv->mdev->pdev->dev), \ + (priv)->port, ## arg); \ + } + + +#define en_dbg(mlevel, priv, format, arg...) \ +do { \ + if (NETIF_MSG_##mlevel & priv->msg_enable) \ + en_print(KERN_DEBUG, priv, format, ##arg); \ +} while (0) +#define en_warn(priv, format, arg...) \ + en_print(KERN_WARNING, priv, format, ##arg) +#define en_err(priv, format, arg...) \ + en_print(KERN_ERR, priv, format, ##arg) +#define en_info(priv, format, arg...) \ + en_print(KERN_INFO, priv, format, ## arg) + +#define mlx4_err(mdev, format, arg...) \ + pr_err("%s %s: " format, DRV_NAME, \ + dev_name(&mdev->pdev->dev), ##arg) +#define mlx4_info(mdev, format, arg...) \ + pr_info("%s %s: " format, DRV_NAME, \ + dev_name(&mdev->pdev->dev), ##arg) +#define mlx4_warn(mdev, format, arg...) \ + pr_warning("%s %s: " format, DRV_NAME, \ + dev_name(&mdev->pdev->dev), ##arg) + #endif |
