summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--share/man/man4/cxgbe.478
-rw-r--r--sys/dev/cxgbe/adapter.h231
-rw-r--r--sys/dev/cxgbe/common/common.h35
-rw-r--r--sys/dev/cxgbe/common/t4_hw.c365
-rw-r--r--sys/dev/cxgbe/common/t4_hw.h78
-rw-r--r--sys/dev/cxgbe/firmware/t4fw_cfg.txt132
-rw-r--r--sys/dev/cxgbe/firmware/t4fw_cfg_uwire.txt503
-rw-r--r--sys/dev/cxgbe/firmware/t4fw_interface.h (renamed from sys/dev/cxgbe/common/t4fw_interface.h)640
-rw-r--r--sys/dev/cxgbe/offload.h71
-rw-r--r--sys/dev/cxgbe/osdep.h1
-rw-r--r--sys/dev/cxgbe/t4_ioctl.h27
-rw-r--r--sys/dev/cxgbe/t4_l2t.c675
-rw-r--r--sys/dev/cxgbe/t4_l2t.h12
-rw-r--r--sys/dev/cxgbe/t4_main.c3023
-rw-r--r--sys/dev/cxgbe/t4_sge.c2191
-rw-r--r--sys/modules/cxgbe/Makefile1
-rw-r--r--sys/modules/cxgbe/firmware/Makefile27
-rw-r--r--tools/tools/cxgbetool/cxgbetool.c64
18 files changed, 6282 insertions, 1872 deletions
diff --git a/share/man/man4/cxgbe.4 b/share/man/man4/cxgbe.4
index ab2979a..4bf5ac3 100644
--- a/share/man/man4/cxgbe.4
+++ b/share/man/man4/cxgbe.4
@@ -99,18 +99,29 @@ Tunables can be set at the
prompt before booting the kernel or stored in
.Xr loader.conf 5 .
.Bl -tag -width indent
-.It Va hw.cxgbe.max_ntxq_10G_port
-The maximum number of tx queues to use for a 10Gb port.
-The default value is 8.
-.It Va hw.cxgbe.max_nrxq_10G_port
-The maximum number of rx queues to use for a 10Gb port.
-The default value is 8.
-.It Va hw.cxgbe.max_ntxq_1G_port
-The maximum number of tx queues to use for a 1Gb port.
-The default value is 2.
-.It Va hw.cxgbe.max_nrxq_1G_port
-The maximum number of rx queues to use for a 1Gb port.
-The default value is 2.
+.It Va hw.cxgbe.ntxq10g
+The number of tx queues to use for a 10Gb port. The default is 16 or the number
+of CPU cores in the system, whichever is less.
+.It Va hw.cxgbe.nrxq10g
+The number of rx queues to use for a 10Gb port. The default is 8 or the number
+of CPU cores in the system, whichever is less.
+.It Va hw.cxgbe.ntxq1g
+The number of tx queues to use for a 1Gb port. The default is 4 or the number
+of CPU cores in the system, whichever is less.
+.It Va hw.cxgbe.nrxq1g
+The number of rx queues to use for a 1Gb port. The default is 2 or the number
+of CPU cores in the system, whichever is less.
+.It Va hw.cxgbe.nofldtxq10g
+The number of TOE tx queues to use for a 10Gb port. The default is 8 or the
+number of CPU cores in the system, whichever is less.
+.It Va hw.cxgbe.nofldrxq10g
+The number of TOE rx queues to use for a 10Gb port. The default is 2 or the
+number of CPU cores in the system, whichever is less.
+.It Va hw.cxgbe.nofldtxq1g
+The number of TOE tx queues to use for a 1Gb port. The default is 2 or the
+number of CPU cores in the system, whichever is less.
+.It Va hw.cxgbe.nofldrxq1g
+The number of TOE rx queues to use for a 1Gb port. The default is 1.
.It Va hw.cxgbe.holdoff_timer_idx_10G
.It Va hw.cxgbe.holdoff_timer_idx_1G
The timer index value to use to delay interrupts.
@@ -119,6 +130,8 @@ by default (all values are in microseconds) and the index selects a
value from this list.
The default value is 1 for both 10Gb and 1Gb ports, which means the
timer value is 5us.
+Different cxgbe interfaces can be assigned different values at any time via the
+dev.cxgbe.X.holdoff_tmr_idx sysctl.
.It Va hw.cxgbe.holdoff_pktc_idx_10G
.It Va hw.cxgbe.holdoff_pktc_idx_1G
The packet-count index value to use to delay interrupts.
@@ -127,6 +140,11 @@ and the index selects a value from this list.
The default value is 2 for both 10Gb and 1Gb ports, which means 16
packets (or the holdoff timer going off) before an interrupt is
generated.
+-1 disables packet counting.
+Different cxgbe interfaces can be assigned different values via the
+dev.cxgbe.X.holdoff_pktc_idx sysctl.
+This sysctl works only when the interface has never been marked up (as done by
+ifconfig up).
.It Va hw.cxgbe.qsize_txq
The size, in number of entries, of the descriptor ring used for a tx
queue.
@@ -134,10 +152,46 @@ A buf_ring of the same size is also allocated for additional
software queuing. See
.Xr ifnet 9 .
The default value is 1024.
+Different cxgbe interfaces can be assigned different values via the
+dev.cxgbe.X.qsize_txq sysctl.
+This sysctl works only when the interface has never been marked up (as done by
+ifconfig up).
.It Va hw.cxgbe.qsize_rxq
The size, in number of entries, of the descriptor ring used for an
rx queue.
The default value is 1024.
+Different cxgbe interfaces can be assigned different values via the
+dev.cxgbe.X.qsize_rxq sysctl.
+This sysctl works only when the interface has never been marked up (as done by
+ifconfig up).
+.It Va hw.cxgbe.interrupt_types
+The interrupt types that the driver is allowed to use.
+Bit 0 represents INTx (line interrupts), bit 1 MSI, bit 2 MSI-X.
+The default is 7 (all allowed).
+The driver will select the best possible type out of the allowed types by
+itself.
+.It Va hw.cxgbe.config_file
+Select a pre-packaged device configuration file.
+A configuration file contains a recipe for partitioning and configuring the
+hardware resources on the card.
+This tunable is for specialized applications only and should not be used in
+normal operation.
+The configuration profile currently in use is available in the dev.t4nex.X.cf
+and dev.t4nex.X.cfcsum sysctls.
+.It Va hw.cxgbe.linkcaps_allowed
+.It Va hw.cxgbe.niccaps_allowed
+.It Va hw.cxgbe.toecaps_allowed
+.It Va hw.cxgbe.rdmacaps_allowed
+.It Va hw.cxgbe.iscsicaps_allowed
+.It Va hw.cxgbe.fcoecaps_allowed
+Disallowing capabilities provides a hint to the driver and firmware to not
+reserve hardware resources for that feature.
+Each of these is a bit field with a bit for each sub-capability within the
+capability.
+This tunable is for specialized applications only and should not be used in
+normal operation.
+The capabilities for which hardware resources have been reserved are listed in
+dev.t4nex.X.*caps sysctls.
.El
.Sh SUPPORT
For general information and support,
diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 8624fc1..7e56932 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -31,6 +31,7 @@
#ifndef __T4_ADAPTER_H__
#define __T4_ADAPTER_H__
+#include <sys/kernel.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <sys/types.h>
@@ -46,8 +47,9 @@
#include <netinet/tcp_lro.h>
#include "offload.h"
-#include "common/t4fw_interface.h"
+#include "firmware/t4fw_interface.h"
+#define T4_CFGNAME "t4fw_cfg"
#define T4_FWNAME "t4fw"
MALLOC_DECLARE(M_CXGBE);
@@ -110,25 +112,21 @@ enum {
FW_IQ_QSIZE = 256,
FW_IQ_ESIZE = 64, /* At least 64 mandated by the firmware spec */
- INTR_IQ_QSIZE = 64,
- INTR_IQ_ESIZE = 64, /* Handles some CPLs too, do not reduce */
-
- CTRL_EQ_QSIZE = 128,
- CTRL_EQ_ESIZE = 64,
-
RX_IQ_QSIZE = 1024,
RX_IQ_ESIZE = 64, /* At least 64 so CPL_RX_PKT will fit */
- RX_FL_ESIZE = 64, /* 8 64bit addresses */
+ EQ_ESIZE = 64, /* All egress queues use this entry size */
+ RX_FL_ESIZE = EQ_ESIZE, /* 8 64bit addresses */
#if MJUMPAGESIZE != MCLBYTES
FL_BUF_SIZES = 4, /* cluster, jumbop, jumbo9k, jumbo16k */
#else
FL_BUF_SIZES = 3, /* cluster, jumbo9k, jumbo16k */
#endif
+ CTRL_EQ_QSIZE = 128,
+
TX_EQ_QSIZE = 1024,
- TX_EQ_ESIZE = 64,
TX_SGL_SEGS = 36,
TX_WR_FLITS = SGE_MAX_WR_LEN / 8
};
@@ -144,13 +142,16 @@ enum {
/* adapter flags */
FULL_INIT_DONE = (1 << 0),
FW_OK = (1 << 1),
- INTR_SHARED = (1 << 2), /* one set of intrq's for all ports */
+ INTR_DIRECT = (1 << 2), /* direct interrupts for everything */
+ MASTER_PF = (1 << 3),
+ ADAP_SYSCTL_CTX = (1 << 4),
CXGBE_BUSY = (1 << 9),
/* port flags */
DOOMED = (1 << 0),
- VI_ENABLED = (1 << 1),
+ PORT_INIT_DONE = (1 << 1),
+ PORT_SYSCTL_CTX = (1 << 2),
};
#define IS_DOOMED(pi) (pi->flags & DOOMED)
@@ -186,6 +187,12 @@ struct port_info {
int first_txq; /* index of first tx queue */
int nrxq; /* # of rx queues */
int first_rxq; /* index of first rx queue */
+#ifndef TCP_OFFLOAD_DISABLE
+ int nofldtxq; /* # of offload tx queues */
+ int first_ofld_txq; /* index of first offload tx queue */
+ int nofldrxq; /* # of offload rx queues */
+ int first_ofld_rxq; /* index of first offload rx queue */
+#endif
int tmr_idx;
int pktc_idx;
int qsize_rxq;
@@ -194,11 +201,8 @@ struct port_info {
struct link_config link_cfg;
struct port_stats stats;
- struct taskqueue *tq;
struct callout tick;
- struct sysctl_ctx_list ctx; /* lives from ifconfig up to down */
- struct sysctl_oid *oid_rxq;
- struct sysctl_oid *oid_txq;
+ struct sysctl_ctx_list ctx; /* from ifconfig up to driver detach */
uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */
};
@@ -222,17 +226,26 @@ struct tx_map {
bus_dmamap_t map;
};
+/* DMA maps used for tx */
+struct tx_maps {
+ struct tx_map *maps;
+ uint32_t map_total; /* # of DMA maps */
+ uint32_t map_pidx; /* next map to be used */
+ uint32_t map_cidx; /* reclaimed up to this index */
+ uint32_t map_avail; /* # of available maps */
+};
+
struct tx_sdesc {
uint8_t desc_used; /* # of hardware descriptors used by the WR */
uint8_t credits; /* NIC txq: # of frames sent out in the WR */
};
-typedef void (iq_intr_handler_t)(void *);
-
enum {
/* iq flags */
- IQ_ALLOCATED = (1 << 1), /* firmware resources allocated */
- IQ_STARTED = (1 << 2), /* started */
+ IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */
+ IQ_HAS_FL = (1 << 1), /* iq associated with a freelist */
+ IQ_INTR = (1 << 2), /* iq takes direct interrupt */
+ IQ_LRO_ENABLED = (1 << 3), /* iq is an eth rxq with LRO enabled */
/* iq state */
IQS_DISABLED = 0,
@@ -252,26 +265,35 @@ struct sge_iq {
uint16_t abs_id; /* absolute SGE id for the iq */
int8_t intr_pktc_idx; /* packet count threshold index */
int8_t pad0;
- iq_intr_handler_t *handler;
__be64 *desc; /* KVA of descriptor ring */
- volatile uint32_t state;
+ volatile int state;
struct adapter *adapter;
const __be64 *cdesc; /* current descriptor */
uint8_t gen; /* generation bit */
uint8_t intr_params; /* interrupt holdoff parameters */
- uint8_t intr_next; /* holdoff for next interrupt */
+ uint8_t intr_next; /* XXX: holdoff for next interrupt */
uint8_t esize; /* size (bytes) of each entry in the queue */
uint16_t qsize; /* size (# of entries) of the queue */
uint16_t cidx; /* consumer index */
- uint16_t cntxt_id; /* SGE context id for the iq */
+ uint16_t cntxt_id; /* SGE context id for the iq */
+
+ STAILQ_ENTRY(sge_iq) link;
};
enum {
+ EQ_CTRL = 1,
+ EQ_ETH = 2,
+#ifndef TCP_OFFLOAD_DISABLE
+ EQ_OFLD = 3,
+#endif
+
/* eq flags */
- EQ_ALLOCATED = (1 << 1), /* firmware resources allocated */
- EQ_STARTED = (1 << 2), /* started */
- EQ_CRFLUSHED = (1 << 3), /* expecting an update from SGE */
+ EQ_TYPEMASK = 7, /* 3 lsbits hold the type */
+ EQ_ALLOCATED = (1 << 3), /* firmware resources allocated */
+ EQ_DOOMED = (1 << 4), /* about to be destroyed */
+ EQ_CRFLUSHED = (1 << 5), /* expecting an update from SGE */
+ EQ_STALLED = (1 << 6), /* out of hw descriptors or dmamaps */
};
/*
@@ -281,10 +303,11 @@ enum {
* consumes them) but it's special enough to have its own struct (see sge_fl).
*/
struct sge_eq {
+ unsigned int flags; /* MUST be first */
+ unsigned int cntxt_id; /* SGE context id for the eq */
bus_dma_tag_t desc_tag;
bus_dmamap_t desc_map;
char lockname[16];
- unsigned int flags;
struct mtx eq_lock;
struct tx_desc *desc; /* KVA of descriptor ring */
@@ -297,9 +320,24 @@ struct sge_eq {
uint16_t pidx; /* producer idx (desc idx) */
uint16_t pending; /* # of descriptors used since last doorbell */
uint16_t iqid; /* iq that gets egr_update for the eq */
- unsigned int cntxt_id; /* SGE context id for the eq */
+ uint8_t tx_chan; /* tx channel used by the eq */
+ struct task tx_task;
+ struct callout tx_callout;
+
+ /* stats */
+
+ uint32_t egr_update; /* # of SGE_EGR_UPDATE notifications for eq */
+ uint32_t unstalled; /* recovered from stall */
+};
+
+enum {
+ FL_STARVING = (1 << 0), /* on the adapter's list of starving fl's */
+ FL_DOOMED = (1 << 1), /* about to be destroyed */
};
+#define FL_RUNNING_LOW(fl) (fl->cap - fl->needed <= fl->lowat)
+#define FL_NOT_RUNNING_LOW(fl) (fl->cap - fl->needed >= 2 * fl->lowat)
+
struct sge_fl {
bus_dma_tag_t desc_tag;
bus_dmamap_t desc_map;
@@ -307,6 +345,7 @@ struct sge_fl {
uint8_t tag_idx;
struct mtx fl_lock;
char lockname[16];
+ int flags;
__be64 *desc; /* KVA of descriptor ring, ptr to addresses */
bus_addr_t ba; /* bus address of descriptor ring */
@@ -317,8 +356,10 @@ struct sge_fl {
uint32_t cidx; /* consumer idx (buffer idx, NOT hw desc idx) */
uint32_t pidx; /* producer idx (buffer idx, NOT hw desc idx) */
uint32_t needed; /* # of buffers needed to fill up fl. */
+ uint32_t lowat; /* # of buffers <= this means fl needs help */
uint32_t pending; /* # of bufs allocated since last doorbell */
unsigned int dmamap_failed;
+ TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
};
/* txq: SGE egress queue + what's needed for Ethernet NIC */
@@ -330,14 +371,8 @@ struct sge_txq {
struct buf_ring *br; /* tx buffer ring */
struct tx_sdesc *sdesc; /* KVA of software descriptor ring */
struct mbuf *m; /* held up due to temporary resource shortage */
- struct task resume_tx;
- /* DMA maps used for tx */
- struct tx_map *maps;
- uint32_t map_total; /* # of DMA maps */
- uint32_t map_pidx; /* next map to be used */
- uint32_t map_cidx; /* reclaimed up to this index */
- uint32_t map_avail; /* # of available maps */
+ struct tx_maps txmaps;
/* stats for common events first */
@@ -354,20 +389,14 @@ struct sge_txq {
uint32_t no_dmamap; /* no DMA map to load the mbuf */
uint32_t no_desc; /* out of hardware descriptors */
- uint32_t egr_update; /* # of SGE_EGR_UPDATE notifications for txq */
} __aligned(CACHE_LINE_SIZE);
-enum {
- RXQ_LRO_ENABLED = (1 << 0)
-};
-
/* rxq: SGE ingress queue + SGE free list + miscellaneous items */
struct sge_rxq {
struct sge_iq iq; /* MUST be first */
- struct sge_fl fl;
+ struct sge_fl fl; /* MUST follow iq */
struct ifnet *ifp; /* the interface this rxq belongs to */
- unsigned int flags;
#ifdef INET
struct lro_ctrl lro; /* LRO state */
#endif
@@ -381,12 +410,28 @@ struct sge_rxq {
} __aligned(CACHE_LINE_SIZE);
-/* ctrlq: SGE egress queue + stats for control queue */
-struct sge_ctrlq {
+#ifndef TCP_OFFLOAD_DISABLE
+/* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */
+struct sge_ofld_rxq {
+ struct sge_iq iq; /* MUST be first */
+ struct sge_fl fl; /* MUST follow iq */
+} __aligned(CACHE_LINE_SIZE);
+#endif
+
+/*
+ * wrq: SGE egress queue that is given prebuilt work requests. Both the control
+ * and offload tx queues are of this type.
+ */
+struct sge_wrq {
struct sge_eq eq; /* MUST be first */
+ struct adapter *adapter;
+ struct mbuf *head; /* held up due to lack of descriptors */
+ struct mbuf *tail; /* valid only if head is valid */
+
/* stats for common events first */
+ uint64_t tx_wrs; /* # of tx work requests */
/* stats for not-that-common events */
@@ -394,20 +439,28 @@ struct sge_ctrlq {
} __aligned(CACHE_LINE_SIZE);
struct sge {
- uint16_t timer_val[SGE_NTIMERS];
- uint8_t counter_val[SGE_NCOUNTERS];
+ int timer_val[SGE_NTIMERS];
+ int counter_val[SGE_NCOUNTERS];
int fl_starve_threshold;
- int nrxq; /* total rx queues (all ports and the rest) */
- int ntxq; /* total tx queues (all ports and the rest) */
- int niq; /* total ingress queues */
- int neq; /* total egress queues */
+ int nrxq; /* total # of Ethernet rx queues */
+ int ntxq; /* total # of Ethernet tx tx queues */
+#ifndef TCP_OFFLOAD_DISABLE
+ int nofldrxq; /* total # of TOE rx queues */
+ int nofldtxq; /* total # of TOE tx queues */
+#endif
+ int niq; /* total # of ingress queues */
+ int neq; /* total # of egress queues */
struct sge_iq fwq; /* Firmware event queue */
- struct sge_ctrlq *ctrlq;/* Control queues */
- struct sge_iq *intrq; /* Interrupt queues */
+ struct sge_wrq mgmtq; /* Management queue (control queue) */
+ struct sge_wrq *ctrlq; /* Control queues */
struct sge_txq *txq; /* NIC tx queues */
struct sge_rxq *rxq; /* NIC rx queues */
+#ifndef TCP_OFFLOAD_DISABLE
+ struct sge_wrq *ofld_txq; /* TOE tx queues */
+ struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */
+#endif
uint16_t iq_start;
int eq_start;
@@ -415,7 +468,12 @@ struct sge {
struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */
};
+struct rss_header;
+typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
+ struct mbuf *);
+
struct adapter {
+ SLIST_ENTRY(adapter) link;
device_t dev;
struct cdev *cdev;
@@ -444,27 +502,47 @@ struct adapter {
struct sge sge;
+ struct taskqueue *tq[NCHAN]; /* taskqueues that flush data out */
struct port_info *port[MAX_NPORTS];
uint8_t chan_map[NCHAN];
+ uint32_t filter_mode;
+#ifndef TCP_OFFLOAD_DISABLE
+ struct uld_softc tom;
+ struct tom_tunables tt;
+#endif
struct l2t_data *l2t; /* L2 table */
struct tid_info tids;
- int registered_device_map;
int open_device_map;
+#ifndef TCP_OFFLOAD_DISABLE
+ int offload_map;
+#endif
int flags;
char fw_version[32];
+ unsigned int cfcsum;
struct adapter_params params;
struct t4_virt_res vres;
- struct sysctl_ctx_list ctx; /* from first_port_up to last_port_down */
- struct sysctl_oid *oid_fwq;
- struct sysctl_oid *oid_ctrlq;
- struct sysctl_oid *oid_intrq;
+ uint16_t linkcaps;
+ uint16_t niccaps;
+ uint16_t toecaps;
+ uint16_t rdmacaps;
+ uint16_t iscsicaps;
+ uint16_t fcoecaps;
+
+ struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */
struct mtx sc_lock;
char lockname[16];
+
+ /* Starving free lists */
+ struct mtx sfl_lock; /* same cache-line as sc_lock? but that's ok */
+ TAILQ_HEAD(, sge_fl) sfl;
+ struct callout sfl_callout;
+
+ cpl_handler_t cpl_handler[256] __aligned(CACHE_LINE_SIZE);
};
#define ADAPTER_LOCK(sc) mtx_lock(&(sc)->sc_lock)
@@ -506,11 +584,15 @@ struct adapter {
#define for_each_rxq(pi, iter, rxq) \
rxq = &pi->adapter->sge.rxq[pi->first_rxq]; \
for (iter = 0; iter < pi->nrxq; ++iter, ++rxq)
+#define for_each_ofld_txq(pi, iter, ofld_txq) \
+ ofld_txq = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq]; \
+ for (iter = 0; iter < pi->nofldtxq; ++iter, ++ofld_txq)
+#define for_each_ofld_rxq(pi, iter, ofld_rxq) \
+ ofld_rxq = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq]; \
+ for (iter = 0; iter < pi->nofldrxq; ++iter, ++ofld_rxq)
/* One for errors, one for firmware events */
#define T4_EXTRA_INTR 2
-#define NINTRQ(sc) ((sc)->intr_count > T4_EXTRA_INTR ? \
- (sc)->intr_count - T4_EXTRA_INTR : 1)
static inline uint32_t
t4_read_reg(struct adapter *sc, uint32_t reg)
@@ -589,29 +671,52 @@ static inline bool is_10G_port(const struct port_info *pi)
return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0);
}
+static inline int tx_resume_threshold(struct sge_eq *eq)
+{
+ return (eq->qsize / 4);
+}
+
/* t4_main.c */
-void cxgbe_txq_start(void *, int);
+void t4_tx_task(void *, int);
+void t4_tx_callout(void *);
int t4_os_find_pci_capability(struct adapter *, int);
int t4_os_pci_save_state(struct adapter *);
int t4_os_pci_restore_state(struct adapter *);
void t4_os_portmod_changed(const struct adapter *, int);
void t4_os_link_changed(struct adapter *, int, int);
+void t4_iterate(void (*)(struct adapter *, void *), void *);
+int t4_register_cpl_handler(struct adapter *, int, cpl_handler_t);
/* t4_sge.c */
void t4_sge_modload(void);
-void t4_sge_init(struct adapter *);
+int t4_sge_init(struct adapter *);
int t4_create_dma_tag(struct adapter *);
int t4_destroy_dma_tag(struct adapter *);
int t4_setup_adapter_queues(struct adapter *);
int t4_teardown_adapter_queues(struct adapter *);
-int t4_setup_eth_queues(struct port_info *);
-int t4_teardown_eth_queues(struct port_info *);
+int t4_setup_port_queues(struct port_info *);
+int t4_teardown_port_queues(struct port_info *);
+int t4_alloc_tx_maps(struct tx_maps *, bus_dma_tag_t, int, int);
+void t4_free_tx_maps(struct tx_maps *, bus_dma_tag_t);
void t4_intr_all(void *);
void t4_intr(void *);
void t4_intr_err(void *);
void t4_intr_evt(void *);
int t4_mgmt_tx(struct adapter *, struct mbuf *);
+int t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct mbuf *);
int t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *);
void t4_update_fl_bufsize(struct ifnet *);
+int can_resume_tx(struct sge_eq *);
+
+static inline int t4_wrq_tx(struct adapter *sc, struct sge_wrq *wrq, struct mbuf *m)
+{
+ int rc;
+
+ TXQ_LOCK(wrq);
+ rc = t4_wrq_tx_locked(sc, wrq, m);
+ TXQ_UNLOCK(wrq);
+ return (rc);
+}
+
#endif
diff --git a/sys/dev/cxgbe/common/common.h b/sys/dev/cxgbe/common/common.h
index 913be9b..81866df 100644
--- a/sys/dev/cxgbe/common/common.h
+++ b/sys/dev/cxgbe/common/common.h
@@ -42,6 +42,15 @@ enum {
enum { MEM_EDC0, MEM_EDC1, MEM_MC };
+enum {
+ MEMWIN0_APERTURE = 2048,
+ MEMWIN0_BASE = 0x1b800,
+ MEMWIN1_APERTURE = 32768,
+ MEMWIN1_BASE = 0x28000,
+ MEMWIN2_APERTURE = 65536,
+ MEMWIN2_BASE = 0x30000,
+};
+
enum dev_master { MASTER_CANT, MASTER_MAY, MASTER_MUST };
enum dev_state { DEV_STATE_UNINIT, DEV_STATE_INIT, DEV_STATE_ERR };
@@ -53,8 +62,8 @@ enum {
};
#define FW_VERSION_MAJOR 1
-#define FW_VERSION_MINOR 3
-#define FW_VERSION_MICRO 10
+#define FW_VERSION_MINOR 4
+#define FW_VERSION_MICRO 16
struct port_stats {
u64 tx_octets; /* total # of octets in good frames */
@@ -190,7 +199,6 @@ struct tp_proxy_stats {
struct tp_cpl_stats {
u32 req[4];
u32 rsp[4];
- u32 tx_err[4];
};
struct tp_rdma_stats {
@@ -214,9 +222,9 @@ struct vpd_params {
};
struct pci_params {
- unsigned int vpd_cap_addr;
- unsigned char speed;
- unsigned char width;
+ unsigned int vpd_cap_addr;
+ unsigned short speed;
+ unsigned short width;
};
/*
@@ -239,20 +247,20 @@ struct adapter_params {
unsigned int fw_vers;
unsigned int tp_vers;
- u8 api_vers[7];
unsigned short mtus[NMTUS];
unsigned short a_wnd[NCCTRL_WIN];
unsigned short b_wnd[NCCTRL_WIN];
- unsigned int mc_size; /* MC memory size */
- unsigned int nfilters; /* size of filter region */
+ unsigned int mc_size; /* MC memory size */
+ unsigned int nfilters; /* size of filter region */
unsigned int cim_la_size;
- unsigned int nports; /* # of ethernet ports */
+ /* Used as int in sysctls, do not reduce size */
+ unsigned int nports; /* # of ethernet ports */
unsigned int portvec;
- unsigned int rev; /* chip revision */
+ unsigned int rev; /* chip revision */
unsigned int offload;
unsigned int ofldq_wr_cred;
@@ -366,6 +374,9 @@ int t4_seeprom_wp(struct adapter *adapter, int enable);
int t4_read_flash(struct adapter *adapter, unsigned int addr, unsigned int nwords,
u32 *data, int byte_oriented);
int t4_load_fw(struct adapter *adapter, const u8 *fw_data, unsigned int size);
+int t4_load_boot(struct adapter *adap, const u8 *boot_data,
+ unsigned int boot_addr, unsigned int size);
+unsigned int t4_flash_cfg_addr(struct adapter *adapter);
int t4_load_cfg(struct adapter *adapter, const u8 *cfg_data, unsigned int size);
int t4_get_fw_version(struct adapter *adapter, u32 *vers);
int t4_get_tp_version(struct adapter *adapter, u32 *vers);
@@ -460,8 +471,8 @@ int t4_wol_pat_enable(struct adapter *adap, unsigned int port, unsigned int map,
int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox,
enum dev_master master, enum dev_state *state);
int t4_fw_bye(struct adapter *adap, unsigned int mbox);
-int t4_early_init(struct adapter *adap, unsigned int mbox);
int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset);
+int t4_fw_initialize(struct adapter *adap, unsigned int mbox);
int t4_query_params(struct adapter *adap, unsigned int mbox, unsigned int pf,
unsigned int vf, unsigned int nparams, const u32 *params,
u32 *val);
diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c
index dd8d0fc..48462aa 100644
--- a/sys/dev/cxgbe/common/t4_hw.c
+++ b/sys/dev/cxgbe/common/t4_hw.c
@@ -30,10 +30,10 @@ __FBSDID("$FreeBSD$");
#include "common.h"
#include "t4_regs.h"
#include "t4_regs_values.h"
-#include "t4fw_interface.h"
+#include "firmware/t4fw_interface.h"
#undef msleep
-#define msleep(x) DELAY((x) * 1000)
+#define msleep(x) pause("t4hw", (x) * hz / 1000)
/**
* t4_wait_op_done_val - wait until an operation is completed
@@ -187,7 +187,7 @@ int t4_wr_mbox_meat(struct adapter *adap, int mbox, const void *cmd, int size,
* off to larger delays to a maximum retry delay.
*/
static const int delay[] = {
- 1, 1, 3, 5, 10, 10, 20, 50, 100, 200
+ 1, 1, 3, 5, 10, 10, 20, 50, 100
};
u32 v;
@@ -625,17 +625,6 @@ enum {
SF_RD_DATA_FAST = 0xb, /* read flash */
SF_RD_ID = 0x9f, /* read ID */
SF_ERASE_SECTOR = 0xd8, /* erase sector */
-
- FW_START_SEC = 8, /* first flash sector for FW */
- FW_END_SEC = 15, /* last flash sector for FW */
- FW_IMG_START = FW_START_SEC * SF_SEC_SIZE,
- FW_MAX_SIZE = (FW_END_SEC - FW_START_SEC + 1) * SF_SEC_SIZE,
-
- FLASH_CFG_MAX_SIZE = 0x10000 , /* max size of the flash config file */
- FLASH_CFG_OFFSET = 0x1f0000,
- FLASH_CFG_START_SEC = FLASH_CFG_OFFSET / SF_SEC_SIZE,
- FPGA_FLASH_CFG_OFFSET = 0xf0000 , /* if FPGA mode, then cfg file is at 1MB - 64KB */
- FPGA_FLASH_CFG_START_SEC = FPGA_FLASH_CFG_OFFSET / SF_SEC_SIZE,
};
/**
@@ -763,12 +752,15 @@ int t4_read_flash(struct adapter *adapter, unsigned int addr,
* @addr: the start address to write
* @n: length of data to write in bytes
* @data: the data to write
+ * @byte_oriented: whether to store data as bytes or as words
*
* Writes up to a page of data (256 bytes) to the serial flash starting
* at the given address. All the data must be written to the same page.
+ * If @byte_oriented is set the write data is stored as byte stream
+ * (i.e. matches what on disk), otherwise in big-endian.
*/
static int t4_write_flash(struct adapter *adapter, unsigned int addr,
- unsigned int n, const u8 *data)
+ unsigned int n, const u8 *data, int byte_oriented)
{
int ret;
u32 buf[SF_PAGE_SIZE / 4];
@@ -788,6 +780,9 @@ static int t4_write_flash(struct adapter *adapter, unsigned int addr,
for (val = 0, i = 0; i < c; ++i)
val = (val << 8) + *data++;
+ if (!byte_oriented)
+ val = htonl(val);
+
ret = sf1_write(adapter, c, c != left, 1, val);
if (ret)
goto unlock;
@@ -799,7 +794,8 @@ static int t4_write_flash(struct adapter *adapter, unsigned int addr,
t4_write_reg(adapter, A_SF_OP, 0); /* unlock SF */
/* Read the page to verify the write succeeded */
- ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, 1);
+ ret = t4_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf,
+ byte_oriented);
if (ret)
return ret;
@@ -825,7 +821,7 @@ unlock:
int t4_get_fw_version(struct adapter *adapter, u32 *vers)
{
return t4_read_flash(adapter,
- FW_IMG_START + offsetof(struct fw_hdr, fw_ver), 1,
+ FLASH_FW_START + offsetof(struct fw_hdr, fw_ver), 1,
vers, 0);
}
@@ -838,7 +834,7 @@ int t4_get_fw_version(struct adapter *adapter, u32 *vers)
*/
int t4_get_tp_version(struct adapter *adapter, u32 *vers)
{
- return t4_read_flash(adapter, FW_IMG_START + offsetof(struct fw_hdr,
+ return t4_read_flash(adapter, FLASH_FW_START + offsetof(struct fw_hdr,
tp_microcode_ver),
1, vers, 0);
}
@@ -854,24 +850,17 @@ int t4_get_tp_version(struct adapter *adapter, u32 *vers)
*/
int t4_check_fw_version(struct adapter *adapter)
{
- u32 api_vers[2];
int ret, major, minor, micro;
ret = t4_get_fw_version(adapter, &adapter->params.fw_vers);
if (!ret)
ret = t4_get_tp_version(adapter, &adapter->params.tp_vers);
- if (!ret)
- ret = t4_read_flash(adapter,
- FW_IMG_START + offsetof(struct fw_hdr, intfver_nic),
- 2, api_vers, 1);
if (ret)
return ret;
major = G_FW_HDR_FW_VER_MAJOR(adapter->params.fw_vers);
minor = G_FW_HDR_FW_VER_MINOR(adapter->params.fw_vers);
micro = G_FW_HDR_FW_VER_MICRO(adapter->params.fw_vers);
- memcpy(adapter->params.api_vers, api_vers,
- sizeof(adapter->params.api_vers));
if (major != FW_VERSION_MAJOR) { /* major mismatch - fail */
CH_ERR(adapter, "card FW has major version %u, driver wants "
@@ -914,6 +903,21 @@ static int t4_flash_erase_sectors(struct adapter *adapter, int start, int end)
}
/**
+ * t4_flash_cfg_addr - return the address of the flash configuration file
+ * @adapter: the adapter
+ *
+ * Return the address within the flash where the Firmware Configuration
+ * File is stored.
+ */
+unsigned int t4_flash_cfg_addr(struct adapter *adapter)
+{
+ if (adapter->params.sf_size == 0x100000)
+ return FLASH_FPGA_CFG_START;
+ else
+ return FLASH_CFG_START;
+}
+
+/**
* t4_load_cfg - download config file
* @adap: the adapter
* @cfg_data: the cfg text file to write
@@ -928,17 +932,8 @@ int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size)
unsigned int flash_cfg_start_sec;
unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
- if (adap->params.sf_size == 0x100000) {
- addr = FPGA_FLASH_CFG_OFFSET;
- flash_cfg_start_sec = FPGA_FLASH_CFG_START_SEC;
- } else {
- addr = FLASH_CFG_OFFSET;
- flash_cfg_start_sec = FLASH_CFG_START_SEC;
- }
- if (!size) {
- CH_ERR(adap, "cfg file has no data\n");
- return -EINVAL;
- }
+ addr = t4_flash_cfg_addr(adap);
+ flash_cfg_start_sec = addr / SF_SEC_SIZE;
if (size > FLASH_CFG_MAX_SIZE) {
CH_ERR(adap, "cfg file too large, max is %u bytes\n",
@@ -950,7 +945,11 @@ int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size)
sf_sec_size);
ret = t4_flash_erase_sectors(adap, flash_cfg_start_sec,
flash_cfg_start_sec + i - 1);
- if (ret)
+ /*
+ * If size == 0 then we're simply erasing the FLASH sectors associated
+ * with the on-adapter Firmware Configuration File.
+ */
+ if (ret || size == 0)
goto out;
/* this will write to the flash up to SF_PAGE_SIZE at a time */
@@ -959,7 +958,7 @@ int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size)
n = size - i;
else
n = SF_PAGE_SIZE;
- ret = t4_write_flash(adap, addr, n, cfg_data);
+ ret = t4_write_flash(adap, addr, n, cfg_data, 1);
if (ret)
goto out;
@@ -969,7 +968,8 @@ int t4_load_cfg(struct adapter *adap, const u8 *cfg_data, unsigned int size)
out:
if (ret)
- CH_ERR(adap, "config file download failed %d\n", ret);
+ CH_ERR(adap, "config file %s failed %d\n",
+ (size == 0 ? "clear" : "download"), ret);
return ret;
}
@@ -1004,9 +1004,9 @@ int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size)
CH_ERR(adap, "FW image size differs from size in FW header\n");
return -EINVAL;
}
- if (size > FW_MAX_SIZE) {
+ if (size > FLASH_FW_MAX_SIZE) {
CH_ERR(adap, "FW image too large, max is %u bytes\n",
- FW_MAX_SIZE);
+ FLASH_FW_MAX_SIZE);
return -EFBIG;
}
@@ -1020,7 +1020,8 @@ int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size)
}
i = DIV_ROUND_UP(size, sf_sec_size); /* # of sectors spanned */
- ret = t4_flash_erase_sectors(adap, FW_START_SEC, FW_START_SEC + i - 1);
+ ret = t4_flash_erase_sectors(adap, FLASH_FW_START_SEC,
+ FLASH_FW_START_SEC + i - 1);
if (ret)
goto out;
@@ -1031,28 +1032,110 @@ int t4_load_fw(struct adapter *adap, const u8 *fw_data, unsigned int size)
*/
memcpy(first_page, fw_data, SF_PAGE_SIZE);
((struct fw_hdr *)first_page)->fw_ver = htonl(0xffffffff);
- ret = t4_write_flash(adap, FW_IMG_START, SF_PAGE_SIZE, first_page);
+ ret = t4_write_flash(adap, FLASH_FW_START, SF_PAGE_SIZE, first_page, 1);
if (ret)
goto out;
- addr = FW_IMG_START;
+ addr = FLASH_FW_START;
for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) {
addr += SF_PAGE_SIZE;
fw_data += SF_PAGE_SIZE;
- ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data);
+ ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, fw_data, 1);
if (ret)
goto out;
}
ret = t4_write_flash(adap,
- FW_IMG_START + offsetof(struct fw_hdr, fw_ver),
- sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver);
+ FLASH_FW_START + offsetof(struct fw_hdr, fw_ver),
+ sizeof(hdr->fw_ver), (const u8 *)&hdr->fw_ver, 1);
out:
if (ret)
CH_ERR(adap, "firmware download failed, error %d\n", ret);
return ret;
}
+/* BIOS boot header */
+typedef struct boot_header_s {
+ u8 signature[2]; /* signature */
+ u8 length; /* image length (include header) */
+ u8 offset[4]; /* initialization vector */
+ u8 reserved[19]; /* reserved */
+ u8 exheader[2]; /* offset to expansion header */
+} boot_header_t;
+
+enum {
+ BOOT_FLASH_BOOT_ADDR = 0x0,/* start address of boot image in flash */
+ BOOT_SIGNATURE = 0xaa55, /* signature of BIOS boot ROM */
+ BOOT_SIZE_INC = 512, /* image size measured in 512B chunks */
+ BOOT_MIN_SIZE = sizeof(boot_header_t), /* at least basic header */
+ BOOT_MAX_SIZE = 1024*BOOT_SIZE_INC /* 1 byte * length increment */
+};
+
+/*
+ * t4_load_boot - download boot flash
+ * @adapter: the adapter
+ * @boot_data: the boot image to write
+ * @size: image size
+ *
+ * Write the supplied boot image to the card's serial flash.
+ * The boot image has the following sections: a 28-byte header and the
+ * boot image.
+ */
+int t4_load_boot(struct adapter *adap, const u8 *boot_data,
+ unsigned int boot_addr, unsigned int size)
+{
+ int ret, addr;
+ unsigned int i;
+ unsigned int boot_sector = boot_addr * 1024;
+ unsigned int sf_sec_size = adap->params.sf_size / adap->params.sf_nsec;
+
+ /*
+ * Perform some primitive sanity testing to avoid accidentally
+ * writing garbage over the boot sectors. We ought to check for
+ * more but it's not worth it for now ...
+ */
+ if (size < BOOT_MIN_SIZE || size > BOOT_MAX_SIZE) {
+ CH_ERR(adap, "boot image too small/large\n");
+ return -EFBIG;
+ }
+
+ /*
+ * Make sure the boot image does not encroach on the firmware region
+ */
+ if ((boot_sector + size) >> 16 > FLASH_FW_START_SEC) {
+ CH_ERR(adap, "boot image encroaching on firmware region\n");
+ return -EFBIG;
+ }
+
+ i = DIV_ROUND_UP(size, sf_sec_size); /* # of sectors spanned */
+ ret = t4_flash_erase_sectors(adap, boot_sector >> 16,
+ (boot_sector >> 16) + i - 1);
+ if (ret)
+ goto out;
+
+ /*
+ * Skip over the first SF_PAGE_SIZE worth of data and write it after
+ * we finish copying the rest of the boot image. This will ensure
+ * that the BIOS boot header will only be written if the boot image
+ * was written in full.
+ */
+ addr = boot_sector;
+ for (size -= SF_PAGE_SIZE; size; size -= SF_PAGE_SIZE) {
+ addr += SF_PAGE_SIZE;
+ boot_data += SF_PAGE_SIZE;
+ ret = t4_write_flash(adap, addr, SF_PAGE_SIZE, boot_data, 0);
+ if (ret)
+ goto out;
+ }
+
+ ret = t4_write_flash(adap, boot_sector, SF_PAGE_SIZE, boot_data, 0);
+
+out:
+ if (ret)
+ CH_ERR(adap, "boot image download failed, error %d\n", ret);
+ return ret;
+}
+
/**
* t4_read_cimq_cfg - read CIM queue configuration
* @adap: the adapter
@@ -1668,7 +1751,10 @@ static void sge_intr_handler(struct adapter *adapter)
err = t4_read_reg(adapter, A_SGE_ERROR_STATS);
if (err & F_ERROR_QID_VALID) {
CH_ERR(adapter, "SGE error for queue %u\n", G_ERROR_QID(err));
- t4_write_reg(adapter, A_SGE_ERROR_STATS, F_ERROR_QID_VALID);
+ if (err & F_UNCAPTURED_ERROR)
+ CH_ERR(adapter, "SGE UNCAPTURED_ERROR set (clearing)\n");
+ t4_write_reg(adapter, A_SGE_ERROR_STATS, F_ERROR_QID_VALID |
+ F_UNCAPTURED_ERROR);
}
if (v != 0)
@@ -2261,6 +2347,7 @@ int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid,
*/
while (n > 0) {
int nq = min(n, 32);
+ int nq_packed = 0;
__be32 *qp = &cmd.iq0_to_iq2;
/*
@@ -2282,25 +2369,28 @@ int t4_config_rss_range(struct adapter *adapter, int mbox, unsigned int viid,
* Ingress Queue ID array and insert them into the command.
*/
while (nq > 0) {
- unsigned int v;
/*
* Grab up to the next 3 Ingress Queue IDs (wrapping
* around the Ingress Queue ID array if necessary) and
* insert them into the firmware RSS command at the
* current 3-tuple position within the commad.
*/
- v = V_FW_RSS_IND_TBL_CMD_IQ0(*rsp);
- if (++rsp >= rsp_end)
- rsp = rspq;
- v |= V_FW_RSS_IND_TBL_CMD_IQ1(*rsp);
- if (++rsp >= rsp_end)
- rsp = rspq;
- v |= V_FW_RSS_IND_TBL_CMD_IQ2(*rsp);
- if (++rsp >= rsp_end)
- rsp = rspq;
-
- *qp++ = htonl(v);
- nq -= 3;
+ u16 qbuf[3];
+ u16 *qbp = qbuf;
+ int nqbuf = min(3, nq);
+
+ nq -= nqbuf;
+ qbuf[0] = qbuf[1] = qbuf[2] = 0;
+ while (nqbuf && nq_packed < 32) {
+ nqbuf--;
+ nq_packed++;
+ *qbp++ = *rsp++;
+ if (rsp >= rsp_end)
+ rsp = rspq;
+ }
+ *qp++ = cpu_to_be32(V_FW_RSS_IND_TBL_CMD_IQ0(qbuf[0]) |
+ V_FW_RSS_IND_TBL_CMD_IQ1(qbuf[1]) |
+ V_FW_RSS_IND_TBL_CMD_IQ2(qbuf[2]));
}
/*
@@ -2694,8 +2784,6 @@ void t4_tp_get_cpl_stats(struct adapter *adap, struct tp_cpl_stats *st)
{
t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->req,
8, A_TP_MIB_CPL_IN_REQ_0);
- t4_read_indirect(adap, A_TP_MIB_INDEX, A_TP_MIB_DATA, st->tx_err,
- 4, A_TP_MIB_CPL_OUT_ERR_0);
}
/**
@@ -3298,6 +3386,7 @@ void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p)
t4_read_reg64(adap, PORT_REG(idx, A_MPS_PORT_STAT_##name##_L))
#define GET_STAT_COM(name) t4_read_reg64(adap, A_MPS_STAT_##name##_L)
+ p->tx_pause = GET_STAT(TX_PORT_PAUSE);
p->tx_octets = GET_STAT(TX_PORT_BYTES);
p->tx_frames = GET_STAT(TX_PORT_FRAMES);
p->tx_bcast_frames = GET_STAT(TX_PORT_BCAST);
@@ -3312,7 +3401,6 @@ void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p)
p->tx_frames_1024_1518 = GET_STAT(TX_PORT_1024B_1518B);
p->tx_frames_1519_max = GET_STAT(TX_PORT_1519B_MAX);
p->tx_drop = GET_STAT(TX_PORT_DROP);
- p->tx_pause = GET_STAT(TX_PORT_PAUSE);
p->tx_ppp0 = GET_STAT(TX_PORT_PPP0);
p->tx_ppp1 = GET_STAT(TX_PORT_PPP1);
p->tx_ppp2 = GET_STAT(TX_PORT_PPP2);
@@ -3322,6 +3410,7 @@ void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p)
p->tx_ppp6 = GET_STAT(TX_PORT_PPP6);
p->tx_ppp7 = GET_STAT(TX_PORT_PPP7);
+ p->rx_pause = GET_STAT(RX_PORT_PAUSE);
p->rx_octets = GET_STAT(RX_PORT_BYTES);
p->rx_frames = GET_STAT(RX_PORT_FRAMES);
p->rx_bcast_frames = GET_STAT(RX_PORT_BCAST);
@@ -3340,7 +3429,6 @@ void t4_get_port_stats(struct adapter *adap, int idx, struct port_stats *p)
p->rx_frames_512_1023 = GET_STAT(RX_PORT_512B_1023B);
p->rx_frames_1024_1518 = GET_STAT(RX_PORT_1024B_1518B);
p->rx_frames_1519_max = GET_STAT(RX_PORT_1519B_MAX);
- p->rx_pause = GET_STAT(RX_PORT_PAUSE);
p->rx_ppp0 = GET_STAT(RX_PORT_PPP0);
p->rx_ppp1 = GET_STAT(RX_PORT_PPP1);
p->rx_ppp2 = GET_STAT(RX_PORT_PPP2);
@@ -3683,28 +3771,114 @@ int t4_fw_hello(struct adapter *adap, unsigned int mbox, unsigned int evt_mbox,
{
int ret;
struct fw_hello_cmd c;
+ u32 v;
+ unsigned int master_mbox;
+ int retries = FW_CMD_HELLO_RETRIES;
+retry:
memset(&c, 0, sizeof(c));
INIT_CMD(c, HELLO, WRITE);
- c.err_to_mbasyncnot = htonl(
+ c.err_to_clearinit = htonl(
V_FW_HELLO_CMD_MASTERDIS(master == MASTER_CANT) |
V_FW_HELLO_CMD_MASTERFORCE(master == MASTER_MUST) |
V_FW_HELLO_CMD_MBMASTER(master == MASTER_MUST ? mbox :
M_FW_HELLO_CMD_MBMASTER) |
- V_FW_HELLO_CMD_MBASYNCNOT(evt_mbox));
+ V_FW_HELLO_CMD_MBASYNCNOT(evt_mbox) |
+ V_FW_HELLO_CMD_STAGE(FW_HELLO_CMD_STAGE_OS) |
+ F_FW_HELLO_CMD_CLEARINIT);
+ /*
+ * Issue the HELLO command to the firmware. If it's not successful
+ * but indicates that we got a "busy" or "timeout" condition, retry
+ * the HELLO until we exhaust our retry limit.
+ */
ret = t4_wr_mbox(adap, mbox, &c, sizeof(c), &c);
- if (ret == 0 && state) {
- u32 v = ntohl(c.err_to_mbasyncnot);
- if (v & F_FW_HELLO_CMD_INIT)
- *state = DEV_STATE_INIT;
- else if (v & F_FW_HELLO_CMD_ERR)
+ if (ret != FW_SUCCESS) {
+ if ((ret == -EBUSY || ret == -ETIMEDOUT) && retries-- > 0)
+ goto retry;
+ return ret;
+ }
+
+ v = ntohl(c.err_to_clearinit);
+ master_mbox = G_FW_HELLO_CMD_MBMASTER(v);
+ if (state) {
+ if (v & F_FW_HELLO_CMD_ERR)
*state = DEV_STATE_ERR;
+ else if (v & F_FW_HELLO_CMD_INIT)
+ *state = DEV_STATE_INIT;
else
*state = DEV_STATE_UNINIT;
- return G_FW_HELLO_CMD_MBMASTER(v);
}
- return ret;
+
+ /*
+ * If we're not the Master PF then we need to wait around for the
+ * Master PF Driver to finish setting up the adapter.
+ *
+ * Note that we also do this wait if we're a non-Master-capable PF and
+ * there is no current Master PF; a Master PF may show up momentarily
+ * and we wouldn't want to fail pointlessly. (This can happen when an
+ * OS loads lots of different drivers rapidly at the same time). In
+ * this case, the Master PF returned by the firmware will be
+ * M_PCIE_FW_MASTER so the test below will work ...
+ */
+ if ((v & (F_FW_HELLO_CMD_ERR|F_FW_HELLO_CMD_INIT)) == 0 &&
+ master_mbox != mbox) {
+ int waiting = FW_CMD_HELLO_TIMEOUT;
+
+ /*
+ * Wait for the firmware to either indicate an error or
+ * initialized state. If we see either of these we bail out
+ * and report the issue to the caller. If we exhaust the
+ * "hello timeout" and we haven't exhausted our retries, try
+ * again. Otherwise bail with a timeout error.
+ */
+ for (;;) {
+ u32 pcie_fw;
+
+ msleep(50);
+ waiting -= 50;
+
+ /*
+ * If neither Error nor Initialialized are indicated
+ * by the firmware keep waiting till we exhaust our
+ * timeout ... and then retry if we haven't exhausted
+ * our retries ...
+ */
+ pcie_fw = t4_read_reg(adap, A_PCIE_FW);
+ if (!(pcie_fw & (F_PCIE_FW_ERR|F_PCIE_FW_INIT))) {
+ if (waiting <= 0) {
+ if (retries-- > 0)
+ goto retry;
+
+ return -ETIMEDOUT;
+ }
+ continue;
+ }
+
+ /*
+ * We either have an Error or Initialized condition
+ * report errors preferentially.
+ */
+ if (state) {
+ if (pcie_fw & F_PCIE_FW_ERR)
+ *state = DEV_STATE_ERR;
+ else if (pcie_fw & F_PCIE_FW_INIT)
+ *state = DEV_STATE_INIT;
+ }
+
+ /*
+ * If we arrived before a Master PF was selected and
+ * there's not a valid Master PF, grab its identity
+ * for our caller.
+ */
+ if (master_mbox == M_PCIE_FW_MASTER &&
+ (pcie_fw & F_PCIE_FW_MASTER_VLD))
+ master_mbox = G_PCIE_FW_MASTER(pcie_fw);
+ break;
+ }
+ }
+
+ return master_mbox;
}
/**
@@ -3724,37 +3898,37 @@ int t4_fw_bye(struct adapter *adap, unsigned int mbox)
}
/**
- * t4_init_cmd - ask FW to initialize the device
+ * t4_fw_reset - issue a reset to FW
* @adap: the adapter
* @mbox: mailbox to use for the FW command
+ * @reset: specifies the type of reset to perform
*
- * Issues a command to FW to partially initialize the device. This
- * performs initialization that generally doesn't depend on user input.
+ * Issues a reset command of the specified type to FW.
*/
-int t4_early_init(struct adapter *adap, unsigned int mbox)
+int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset)
{
- struct fw_initialize_cmd c;
+ struct fw_reset_cmd c;
memset(&c, 0, sizeof(c));
- INIT_CMD(c, INITIALIZE, WRITE);
+ INIT_CMD(c, RESET, WRITE);
+ c.val = htonl(reset);
return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
}
/**
- * t4_fw_reset - issue a reset to FW
+ * t4_fw_initialize - ask FW to initialize the device
* @adap: the adapter
* @mbox: mailbox to use for the FW command
- * @reset: specifies the type of reset to perform
*
- * Issues a reset command of the specified type to FW.
+ * Issues a command to FW to partially initialize the device. This
+ * performs initialization that generally doesn't depend on user input.
*/
-int t4_fw_reset(struct adapter *adap, unsigned int mbox, int reset)
+int t4_fw_initialize(struct adapter *adap, unsigned int mbox)
{
- struct fw_reset_cmd c;
+ struct fw_initialize_cmd c;
memset(&c, 0, sizeof(c));
- INIT_CMD(c, RESET, WRITE);
- c.val = htonl(reset);
+ INIT_CMD(c, INITIALIZE, WRITE);
return t4_wr_mbox(adap, mbox, &c, sizeof(c), NULL);
}
@@ -4495,6 +4669,21 @@ static int __devinit get_flash_params(struct adapter *adapter)
return 0;
}
+static void __devinit set_pcie_completion_timeout(struct adapter *adapter,
+ u8 range)
+{
+ u16 val;
+ u32 pcie_cap;
+
+ pcie_cap = t4_os_find_pci_capability(adapter, PCI_CAP_ID_EXP);
+ if (pcie_cap) {
+ t4_os_pci_read_cfg2(adapter, pcie_cap + PCI_EXP_DEVCTL2, &val);
+ val &= 0xfff0;
+ val |= range ;
+ t4_os_pci_write_cfg2(adapter, pcie_cap + PCI_EXP_DEVCTL2, val);
+ }
+}
+
/**
* t4_prep_adapter - prepare SW and HW for operation
* @adapter: the adapter
@@ -4541,6 +4730,8 @@ int __devinit t4_prep_adapter(struct adapter *adapter)
adapter->params.portvec = 1;
adapter->params.vpd.cclk = 50000;
+ /* Set pci completion timeout value to 4 seconds. */
+ set_pcie_completion_timeout(adapter, 0xd);
return 0;
}
diff --git a/sys/dev/cxgbe/common/t4_hw.h b/sys/dev/cxgbe/common/t4_hw.h
index fd48aab..b93734e 100644
--- a/sys/dev/cxgbe/common/t4_hw.h
+++ b/sys/dev/cxgbe/common/t4_hw.h
@@ -182,4 +182,82 @@ struct pagepod {
#define M_PPOD_OFST 0xFFFFFFFF
#define V_PPOD_OFST(x) ((x) << S_PPOD_OFST)
+/*
+ * Flash layout.
+ */
+#define FLASH_START(start) ((start) * SF_SEC_SIZE)
+#define FLASH_MAX_SIZE(nsecs) ((nsecs) * SF_SEC_SIZE)
+
+enum {
+ /*
+ * Various Expansion-ROM boot images, etc.
+ */
+ FLASH_EXP_ROM_START_SEC = 0,
+ FLASH_EXP_ROM_NSECS = 6,
+ FLASH_EXP_ROM_START = FLASH_START(FLASH_EXP_ROM_START_SEC),
+ FLASH_EXP_ROM_MAX_SIZE = FLASH_MAX_SIZE(FLASH_EXP_ROM_NSECS),
+
+ /*
+ * iSCSI Boot Firmware Table (iBFT) and other driver-related
+ * parameters ...
+ */
+ FLASH_IBFT_START_SEC = 6,
+ FLASH_IBFT_NSECS = 1,
+ FLASH_IBFT_START = FLASH_START(FLASH_IBFT_START_SEC),
+ FLASH_IBFT_MAX_SIZE = FLASH_MAX_SIZE(FLASH_IBFT_NSECS),
+
+ /*
+ * Boot configuration data.
+ */
+ FLASH_BOOTCFG_START_SEC = 7,
+ FLASH_BOOTCFG_NSECS = 1,
+ FLASH_BOOTCFG_START = FLASH_START(FLASH_BOOTCFG_START_SEC),
+ FLASH_BOOTCFG_MAX_SIZE = FLASH_MAX_SIZE(FLASH_BOOTCFG_NSECS),
+
+ /*
+ * Location of firmware image in FLASH.
+ */
+ FLASH_FW_START_SEC = 8,
+ FLASH_FW_NSECS = 8,
+ FLASH_FW_START = FLASH_START(FLASH_FW_START_SEC),
+ FLASH_FW_MAX_SIZE = FLASH_MAX_SIZE(FLASH_FW_NSECS),
+
+ /*
+ * iSCSI persistent/crash information.
+ */
+ FLASH_ISCSI_CRASH_START_SEC = 29,
+ FLASH_ISCSI_CRASH_NSECS = 1,
+ FLASH_ISCSI_CRASH_START = FLASH_START(FLASH_ISCSI_CRASH_START_SEC),
+ FLASH_ISCSI_CRASH_MAX_SIZE = FLASH_MAX_SIZE(FLASH_ISCSI_CRASH_NSECS),
+
+ /*
+ * FCoE persistent/crash information.
+ */
+ FLASH_FCOE_CRASH_START_SEC = 30,
+ FLASH_FCOE_CRASH_NSECS = 1,
+ FLASH_FCOE_CRASH_START = FLASH_START(FLASH_FCOE_CRASH_START_SEC),
+ FLASH_FCOE_CRASH_MAX_SIZE = FLASH_MAX_SIZE(FLASH_FCOE_CRASH_NSECS),
+
+ /*
+ * Location of Firmware Configuration File in FLASH. Since the FPGA
+ * "FLASH" is smaller we need to store the Configuration File in a
+ * different location -- which will overlap the end of the firmware
+ * image if firmware ever gets that large ...
+ */
+ FLASH_CFG_START_SEC = 31,
+ FLASH_CFG_NSECS = 1,
+ FLASH_CFG_START = FLASH_START(FLASH_CFG_START_SEC),
+ FLASH_CFG_MAX_SIZE = FLASH_MAX_SIZE(FLASH_CFG_NSECS),
+
+ FLASH_FPGA_CFG_START_SEC = 15,
+ FLASH_FPGA_CFG_START = FLASH_START(FLASH_FPGA_CFG_START_SEC),
+
+ /*
+ * Sectors 32-63 are reserved for FLASH failover.
+ */
+};
+
+#undef FLASH_START
+#undef FLASH_MAX_SIZE
+
#endif /* __T4_HW_H */
diff --git a/sys/dev/cxgbe/firmware/t4fw_cfg.txt b/sys/dev/cxgbe/firmware/t4fw_cfg.txt
new file mode 100644
index 0000000..65d6f0b
--- /dev/null
+++ b/sys/dev/cxgbe/firmware/t4fw_cfg.txt
@@ -0,0 +1,132 @@
+# Firmware configuration file.
+#
+# Global limits (some are hardware limits, others are due to the firmware).
+# Also note that the firmware reserves some of these resources for its own use
+# so it's not always possible for the drivers to grab everything listed here.
+# nvi = 128 virtual interfaces
+# niqflint = 1023 ingress queues with freelists and/or interrupts
+# nethctrl = 64K Ethernet or ctrl egress queues
+# neq = 64K egress queues of all kinds, including freelists
+# nexactf = 336 MPS TCAM entries, can oversubscribe.
+#
+
+[global]
+ rss_glb_config_mode = basicvirtual
+ rss_glb_config_options = tnlmapen, hashtoeplitz, tnlalllkp
+
+ sge_timer_value = 1, 5, 10, 50, 100, 200 # usecs
+
+ # TP_SHIFT_CNT
+ reg[0x7dc0] = 0x64f8849
+
+ filterMode = fragmentation, mpshittype, protocol, vlan, port, fcoe
+
+ # TP rx and tx payload memory (% of the total EDRAM + DDR3).
+ tp_pmrx = 40
+ tp_pmtx = 60
+ tp_pmrx_pagesize = 64K
+ tp_pmtx_pagesize = 64K
+
+# PFs 0-3. These get 8 MSI/8 MSI-X vectors each. VFs are supported by
+# these 4 PFs only. Not used here at all.
+[function "0"]
+ nvf = 16
+ nvi = 1
+[function "0/*"]
+ nvi = 1
+
+[function "1"]
+ nvf = 16
+ nvi = 1
+[function "1/*"]
+ nvi = 1
+
+[function "2"]
+ nvf = 16
+ nvi = 1
+[function "2/*"]
+ nvi = 1
+
+[function "3"]
+ nvf = 16
+ nvi = 1
+[function "3/*"]
+ nvi = 1
+
+# PF4 is the resource-rich PF that the bus/nexus driver attaches to.
+# It gets 32 MSI/128 MSI-X vectors.
+[function "4"]
+ wx_caps = all
+ r_caps = all
+ nvi = 48
+ niqflint = 256
+ nethctrl = 128
+ neq = 256
+ nexactf = 300
+ cmask = all
+ pmask = all
+
+ # driver will mask off features it won't use
+ protocol = ofld
+
+ tp_l2t = 100
+
+ # TCAM has 8K cells; each region must start at a multiple of 128 cell.
+ # Each entry in these categories takes 4 cells each. nhash will use the
+ # TCAM iff there is room left (that is, the rest don't add up to 2048).
+ nroute = 32
+ nclip = 0 # needed only for IPv6 offload
+ nfilter = 1504
+ nserver = 512
+ nhash = 16384
+
+# PF5 is the SCSI Controller PF. It gets 32 MSI/40 MSI-X vectors.
+# Not used right now.
+[function "5"]
+ nvi = 1
+
+# PF6 is the FCoE Controller PF. It gets 32 MSI/40 MSI-X vectors.
+# Not used right now.
+[function "6"]
+ nvi = 1
+
+# MPS has 192K buffer space for ingress packets from the wire as well as
+# loopback path of the L2 switch.
+[port "0"]
+ dcb = none
+ bg_mem = 25
+ lpbk_mem = 25
+ hwm = 30
+ lwm = 15
+ dwm = 30
+
+[port "1"]
+ dcb = none
+ bg_mem = 25
+ lpbk_mem = 25
+ hwm = 30
+ lwm = 15
+ dwm = 30
+
+[port "2"]
+ dcb = none
+ bg_mem = 25
+ lpbk_mem = 25
+ hwm = 30
+ lwm = 15
+ dwm = 30
+
+[port "3"]
+ dcb = none
+ bg_mem = 25
+ lpbk_mem = 25
+ hwm = 30
+ lwm = 15
+ dwm = 30
+
+[fini]
+ version = 0x1
+ checksum = 0xb31cdfac
+#
+# $FreeBSD$
+#
diff --git a/sys/dev/cxgbe/firmware/t4fw_cfg_uwire.txt b/sys/dev/cxgbe/firmware/t4fw_cfg_uwire.txt
new file mode 100644
index 0000000..134d60c
--- /dev/null
+++ b/sys/dev/cxgbe/firmware/t4fw_cfg_uwire.txt
@@ -0,0 +1,503 @@
+# Chelsio T4 Factory Default configuration file.
+#
+# Copyright (C) 2010 Chelsio Communications. All rights reserved.
+#
+
+# This file provides the default, power-on configuration for 4-port T4-based
+# adapters shipped from the factory. These defaults are designed to address
+# the needs of the vast majority of T4 customers. The basic idea is to have
+# a default configuration which allows a customer to plug a T4 adapter in and
+# have it work regardless of OS, driver or application except in the most
+# unusual and/or demanding customer applications.
+#
+# Many of the T4 resources which are described by this configuration are
+# finite. This requires balancing the configuration/operation needs of
+# device drivers across OSes and a large number of customer application.
+#
+# Some of the more important resources to allocate and their constaints are:
+# 1. Virtual Interfaces: 128.
+# 2. Ingress Queues with Free Lists: 1024. PCI-E SR-IOV Virtual Functions
+# must use a power of 2 Ingress Queues.
+# 3. Egress Queues: 128K. PCI-E SR-IOV Virtual Functions must use a
+# power of 2 Egress Queues.
+# 4. MSI-X Vectors: 1088. A complication here is that the PCI-E SR-IOV
+# Virtual Functions based off of a Physical Function all get the
+# same umber of MSI-X Vectors as the base Physical Function.
+# Additionally, regardless of whether Virtual Functions are enabled or
+# not, their MSI-X "needs" are counted by the PCI-E implementation.
+# And finally, all Physical Funcations capable of supporting Virtual
+# Functions (PF0-3) must have the same number of configured TotalVFs in
+# their SR-IOV Capabilities.
+# 5. Multi-Port Support (MPS) TCAM: 336 entries to support MAC destination
+# address matching on Ingress Packets.
+#
+# Some of the important OS/Driver resource needs are:
+# 6. Some OS Drivers will manage all resources through a single Physical
+# Function (currently PF0 but it could be any Physical Function). Thus,
+# this "Unified PF" will need to have enough resources allocated to it
+# to allow for this. And because of the MSI-X resource allocation
+# constraints mentioned above, this probably means we'll either have to
+# severely limit the TotalVFs if we continue to use PF0 as the Unified PF
+# or we'll need to move the Unified PF into the PF4-7 range since those
+# Physical Functions don't have any Virtual Functions associated with
+# them.
+# 7. Some OS Drivers will manage different ports and functions (NIC,
+# storage, etc.) on different Physical Functions. For example, NIC
+# functions for ports 0-3 on PF0-3, FCoE on PF4, iSCSI on PF5, etc.
+#
+# Some of the customer application needs which need to be accommodated:
+# 8. Some customers will want to support large CPU count systems with
+# good scaling. Thus, we'll need to accommodate a number of
+# Ingress Queues and MSI-X Vectors to allow up to some number of CPUs
+# to be involved per port and per application function. For example,
+# in the case where all ports and application functions will be
+# managed via a single Unified PF and we want to accommodate scaling up
+# to 8 CPUs, we would want:
+#
+# 4 ports *
+# 3 application functions (NIC, FCoE, iSCSI) per port *
+# 8 Ingress Queue/MSI-X Vectors per application function
+#
+# for a total of 96 Ingress Queues and MSI-X Vectors on the Unified PF.
+# (Plus a few for Firmware Event Queues, etc.)
+#
+# 9. Some customers will want to use T4's PCI-E SR-IOV Capability to allow
+# Virtual Machines to directly access T4 functionality via SR-IOV
+# Virtual Functions and "PCI Device Passthrough" -- this is especially
+# true for the NIC application functionality. (Note that there is
+# currently no ability to use the TOE, FCoE, iSCSI, etc. via Virtual
+# Functions so this is in fact solely limited to NIC.)
+#
+
+
+# Global configuration settings.
+#
+[global]
+ rss_glb_config_mode = basicvirtual
+ rss_glb_config_options = tnlmapen,hashtoeplitz,tnlalllkp
+
+ # The following Scatter Gather Engine (SGE) settings assume a 4KB Host
+ # Page Size and a 64B L1 Cache Line Size. It programs the
+ # EgrStatusPageSize and IngPadBoundary to 64B and the PktShift to 2.
+ # If a Master PF Driver finds itself on a machine with different
+ # parameters, then the Master PF Driver is responsible for initializing
+ # these parameters to appropriate values.
+ #
+ # Notes:
+ # 1. The Free List Buffer Sizes below are raw and the firmware will
+ # round them up to the Ingress Padding Boundary.
+ # 2. The SGE Timer Values below are expressed below in microseconds.
+ # The firmware will convert these values to Core Clock Ticks when
+ # it processes the configuration parameters.
+ #
+ reg[0x1008] = 0x40810/0x21c70 # SGE_CONTROL
+ reg[0x100c] = 0x22222222 # SGE_HOST_PAGE_SIZE
+ reg[0x10a0] = 0x01040810 # SGE_INGRESS_RX_THRESHOLD
+ reg[0x1044] = 4096 # SGE_FL_BUFFER_SIZE0
+ reg[0x1048] = 65536 # SGE_FL_BUFFER_SIZE1
+ reg[0x104c] = 1536 # SGE_FL_BUFFER_SIZE2
+ reg[0x1050] = 9024 # SGE_FL_BUFFER_SIZE3
+ reg[0x1054] = 9216 # SGE_FL_BUFFER_SIZE4
+ reg[0x1058] = 2048 # SGE_FL_BUFFER_SIZE5
+ reg[0x105c] = 128 # SGE_FL_BUFFER_SIZE6
+ reg[0x1060] = 8192 # SGE_FL_BUFFER_SIZE7
+ reg[0x1064] = 16384 # SGE_FL_BUFFER_SIZE8
+ reg[0x10a4] = 0xa000a000/0xf000f000 # SGE_DBFIFO_STATUS
+ reg[0x10a8] = 0x2000/0x2000 # SGE_DOORBELL_CONTROL
+ sge_timer_value = 5, 10, 20, 50, 100, 200 # SGE_TIMER_VALUE* in usecs
+
+ reg[0x7dc0] = 0x64f8849 # TP_SHIFT_CNT
+
+ # Selection of tuples for LE filter lookup, fields (and widths which
+ # must sum to <= 36): { IP Fragment (1), MPS Match Type (3),
+ # IP Protocol (8), [Inner] VLAN (17), Port (3), FCoE (1) }
+ #
+ filterMode = fragmentation, mpshittype, protocol, vnic_id, port, fcoe
+
+ # Percentage of dynamic memory (in either the EDRAM or external MEM)
+ # to use for TP RX payload
+ tp_pmrx = 30
+
+ # TP RX payload page size
+ tp_pmrx_pagesize = 64K
+
+ # Percentage of dynamic memory (in either the EDRAM or external MEM)
+ # to use for TP TX payload
+ tp_pmtx = 50
+
+ # TP TX payload page size
+ tp_pmtx_pagesize = 64K
+
+# Some "definitions" to make the rest of this a bit more readable. We support
+# 4 ports, 3 functions (NIC, FCoE and iSCSI), scaling up to 8 "CPU Queue Sets"
+# per function per port ...
+#
+# NMSIX = 1088 # available MSI-X Vectors
+# NVI = 128 # available Virtual Interfaces
+# NMPSTCAM = 336 # MPS TCAM entries
+#
+# NPORTS = 4 # ports
+# NCPUS = 8 # CPUs we want to support scalably
+# NFUNCS = 3 # functions per port (NIC, FCoE, iSCSI)
+
+# Breakdown of Virtual Interface/Queue/Interrupt resources for the "Unified
+# PF" which many OS Drivers will use to manage most or all functions.
+#
+# Each Ingress Queue can use one MSI-X interrupt but some Ingress Queues can
+# use Forwarded Interrupt Ingress Queues. For these latter, an Ingress Queue
+# would be created and the Queue ID of a Forwarded Interrupt Ingress Queue
+# will be specified as the "Ingress Queue Asynchronous Destination Index."
+# Thus, the number of MSI-X Vectors assigned to the Unified PF will be less
+# than or equal to the number of Ingress Queues ...
+#
+# NVI_NIC = 4 # NIC access to NPORTS
+# NFLIQ_NIC = 32 # NIC Ingress Queues with Free Lists
+# NETHCTRL_NIC = 32 # NIC Ethernet Control/TX Queues
+# NEQ_NIC = 64 # NIC Egress Queues (FL, ETHCTRL/TX)
+# NMPSTCAM_NIC = 16 # NIC MPS TCAM Entries (NPORTS*4)
+# NMSIX_NIC = 32 # NIC MSI-X Interrupt Vectors (FLIQ)
+#
+# NVI_OFLD = 0 # Offload uses NIC function to access ports
+# NFLIQ_OFLD = 16 # Offload Ingress Queues with Free Lists
+# NETHCTRL_OFLD = 0 # Offload Ethernet Control/TX Queues
+# NEQ_OFLD = 16 # Offload Egress Queues (FL)
+# NMPSTCAM_OFLD = 0 # Offload MPS TCAM Entries (uses NIC's)
+# NMSIX_OFLD = 16 # Offload MSI-X Interrupt Vectors (FLIQ)
+#
+# NVI_RDMA = 0 # RDMA uses NIC function to access ports
+# NFLIQ_RDMA = 4 # RDMA Ingress Queues with Free Lists
+# NETHCTRL_RDMA = 0 # RDMA Ethernet Control/TX Queues
+# NEQ_RDMA = 4 # RDMA Egress Queues (FL)
+# NMPSTCAM_RDMA = 0 # RDMA MPS TCAM Entries (uses NIC's)
+# NMSIX_RDMA = 4 # RDMA MSI-X Interrupt Vectors (FLIQ)
+#
+# NEQ_WD = 128 # Wire Direct TX Queues and FLs
+# NETHCTRL_WD = 64 # Wire Direct TX Queues
+# NFLIQ_WD = 64 ` # Wire Direct Ingress Queues with Free Lists
+#
+# NVI_ISCSI = 4 # ISCSI access to NPORTS
+# NFLIQ_ISCSI = 4 # ISCSI Ingress Queues with Free Lists
+# NETHCTRL_ISCSI = 0 # ISCSI Ethernet Control/TX Queues
+# NEQ_ISCSI = 4 # ISCSI Egress Queues (FL)
+# NMPSTCAM_ISCSI = 4 # ISCSI MPS TCAM Entries (NPORTS)
+# NMSIX_ISCSI = 4 # ISCSI MSI-X Interrupt Vectors (FLIQ)
+#
+# NVI_FCOE = 4 # FCOE access to NPORTS
+# NFLIQ_FCOE = 34 # FCOE Ingress Queues with Free Lists
+# NETHCTRL_FCOE = 32 # FCOE Ethernet Control/TX Queues
+# NEQ_FCOE = 66 # FCOE Egress Queues (FL)
+# NMPSTCAM_FCOE = 32 # FCOE MPS TCAM Entries (NPORTS)
+# NMSIX_FCOE = 34 # FCOE MSI-X Interrupt Vectors (FLIQ)
+
+# Two extra Ingress Queues per function for Firmware Events and Forwarded
+# Interrupts, and two extra interrupts per function for Firmware Events (or a
+# Forwarded Interrupt Queue) and General Interrupts per function.
+#
+# NFLIQ_EXTRA = 6 # "extra" Ingress Queues 2*NFUNCS (Firmware and
+# # Forwarded Interrupts
+# NMSIX_EXTRA = 6 # extra interrupts 2*NFUNCS (Firmware and
+# # General Interrupts
+
+# Microsoft HyperV resources. The HyperV Virtual Ingress Queues will have
+# their interrupts forwarded to another set of Forwarded Interrupt Queues.
+#
+# NVI_HYPERV = 16 # VMs we want to support
+# NVIIQ_HYPERV = 2 # Virtual Ingress Queues with Free Lists per VM
+# NFLIQ_HYPERV = 40 # VIQs + NCPUS Forwarded Interrupt Queues
+# NEQ_HYPERV = 32 # VIQs Free Lists
+# NMPSTCAM_HYPERV = 16 # MPS TCAM Entries (NVI_HYPERV)
+# NMSIX_HYPERV = 8 # NCPUS Forwarded Interrupt Queues
+
+# Adding all of the above Unified PF resource needs together: (NIC + OFLD +
+# RDMA + ISCSI + FCOE + EXTRA + HYPERV)
+#
+# NVI_UNIFIED = 28
+# NFLIQ_UNIFIED = 106
+# NETHCTRL_UNIFIED = 32
+# NEQ_UNIFIED = 124
+# NMPSTCAM_UNIFIED = 40
+#
+# The sum of all the MSI-X resources above is 74 MSI-X Vectors but we'll round
+# that up to 128 to make sure the Unified PF doesn't run out of resources.
+#
+# NMSIX_UNIFIED = 128
+#
+# The Storage PFs could need up to NPORTS*NCPUS + NMSIX_EXTRA MSI-X Vectors
+# which is 34 but they're probably safe with 32.
+#
+# NMSIX_STORAGE = 32
+
+# Note: The UnifiedPF is PF4 which doesn't have any Virtual Functions
+# associated with it. Thus, the MSI-X Vector allocations we give to the
+# UnifiedPF aren't inherited by any Virtual Functions. As a result we can
+# provision many more Virtual Functions than we can if the UnifiedPF were
+# one of PF0-3.
+#
+
+# All of the below PCI-E parameters are actually stored in various *_init.txt
+# files. We include them below essentially as comments.
+#
+# For PF0-3 we assign 8 vectors each for NIC Ingress Queues of the associated
+# ports 0-3.
+#
+# For PF4, the Unified PF, we give it an MSI-X Table Size as outlined above.
+#
+# For PF5-6 we assign enough MSI-X Vectors to support FCoE and iSCSI
+# storage applications across all four possible ports.
+#
+# Additionally, since the UnifiedPF isn't one of the per-port Physical
+# Functions, we give the UnifiedPF and the PF0-3 Physical Functions
+# different PCI Device IDs which will allow Unified and Per-Port Drivers
+# to directly select the type of Physical Function to which they wish to be
+# attached.
+#
+# Note that the actual values used for the PCI-E Intelectual Property will be
+# 1 less than those below since that's the way it "counts" things. For
+# readability, we use the number we actually mean ...
+#
+# PF0_INT = 8 # NCPUS
+# PF1_INT = 8 # NCPUS
+# PF2_INT = 8 # NCPUS
+# PF3_INT = 8 # NCPUS
+# PF0_3_INT = 32 # PF0_INT + PF1_INT + PF2_INT + PF3_INT
+#
+# PF4_INT = 128 # NMSIX_UNIFIED
+# PF5_INT = 32 # NMSIX_STORAGE
+# PF6_INT = 32 # NMSIX_STORAGE
+# PF7_INT = 0 # Nothing Assigned
+# PF4_7_INT = 192 # PF4_INT + PF5_INT + PF6_INT + PF7_INT
+#
+# PF0_7_INT = 224 # PF0_3_INT + PF4_7_INT
+#
+# With the above we can get 17 VFs/PF0-3 (limited by 336 MPS TCAM entries)
+# but we'll lower that to 16 to make our total 64 and a nice power of 2 ...
+#
+# NVF = 16
+
+# For those OSes which manage different ports on different PFs, we need
+# only enough resources to support a single port's NIC application functions
+# on PF0-3. The below assumes that we're only doing NIC with NCPUS "Queue
+# Sets" for ports 0-3. The FCoE and iSCSI functions for such OSes will be
+# managed on the "storage PFs" (see below).
+#
+[function "0"]
+ nvf = 16 # NVF on this function
+ wx_caps = all # write/execute permissions for all commands
+ r_caps = all # read permissions for all commands
+ nvi = 1 # 1 port
+ niqflint = 8 # NCPUS "Queue Sets"
+ nethctrl = 8 # NCPUS "Queue Sets"
+ neq = 16 # niqflint + nethctrl Egress Queues
+ nexactf = 8 # number of exact MPSTCAM MAC filters
+ cmask = all # access to all channels
+ pmask = 0x1 # access to only one port
+
+[function "1"]
+ nvf = 16 # NVF on this function
+ wx_caps = all # write/execute permissions for all commands
+ r_caps = all # read permissions for all commands
+ nvi = 1 # 1 port
+ niqflint = 8 # NCPUS "Queue Sets"
+ nethctrl = 8 # NCPUS "Queue Sets"
+ neq = 16 # niqflint + nethctrl Egress Queues
+ nexactf = 8 # number of exact MPSTCAM MAC filters
+ cmask = all # access to all channels
+ pmask = 0x2 # access to only one port
+
+[function "2"]
+ nvf = 16 # NVF on this function
+ wx_caps = all # write/execute permissions for all commands
+ r_caps = all # read permissions for all commands
+ nvi = 1 # 1 port
+ niqflint = 8 # NCPUS "Queue Sets"
+ nethctrl = 8 # NCPUS "Queue Sets"
+ neq = 16 # niqflint + nethctrl Egress Queues
+ nexactf = 8 # number of exact MPSTCAM MAC filters
+ cmask = all # access to all channels
+ pmask = 0x4 # access to only one port
+
+[function "3"]
+ nvf = 16 # NVF on this function
+ wx_caps = all # write/execute permissions for all commands
+ r_caps = all # read permissions for all commands
+ nvi = 1 # 1 port
+ niqflint = 8 # NCPUS "Queue Sets"
+ nethctrl = 8 # NCPUS "Queue Sets"
+ neq = 16 # niqflint + nethctrl Egress Queues
+ nexactf = 8 # number of exact MPSTCAM MAC filters
+ cmask = all # access to all channels
+ pmask = 0x8 # access to only one port
+
+# Some OS Drivers manage all application functions for all ports via PF4.
+# Thus we need to provide a large number of resources here. For Egress
+# Queues we need to account for both TX Queues as well as Free List Queues
+# (because the host is responsible for producing Free List Buffers for the
+# hardware to consume).
+#
+[function "4"]
+ wx_caps = all # write/execute permissions for all commands
+ r_caps = all # read permissions for all commands
+ nvi = 28 # NVI_UNIFIED
+ niqflint = 170 # NFLIQ_UNIFIED + NLFIQ_WD
+ nethctrl = 96 # NETHCTRL_UNIFIED + NETHCTRL_WD
+ neq = 252 # NEQ_UNIFIED + NEQ_WD
+ nexactf = 40 # NMPSTCAM_UNIFIED
+ cmask = all # access to all channels
+ pmask = all # access to all four ports ...
+ nroute = 32 # number of routing region entries
+ nclip = 32 # number of clip region entries
+ nfilter = 768 # number of filter region entries
+ nserver = 256 # number of server region entries
+ nhash = 0 # number of hash region entries
+ protocol = nic_vm, ofld, rddp, rdmac, iscsi_initiator_pdu, iscsi_target_pdu
+ tp_l2t = 100
+ tp_ddp = 2
+ tp_ddp_iscsi = 2
+ tp_stag = 2
+ tp_pbl = 5
+ tp_rq = 7
+
+# We have FCoE and iSCSI storage functions on PF5 and PF6 each of which may
+# need to have Virtual Interfaces on each of the four ports with up to NCPUS
+# "Queue Sets" each.
+#
+[function "5"]
+ wx_caps = all # write/execute permissions for all commands
+ r_caps = all # read permissions for all commands
+ nvi = 4 # NPORTS
+ niqflint = 34 # NPORTS*NCPUS + NMSIX_EXTRA
+ nethctrl = 32 # NPORTS*NCPUS
+ neq = 64 # NPORTS*NCPUS * 2 (FL, ETHCTRL/TX)
+ nexactf = 4 # NPORTS
+ cmask = all # access to all channels
+ pmask = all # access to all four ports ...
+
+[function "6"]
+ wx_caps = all # write/execute permissions for all commands
+ r_caps = all # read permissions for all commands
+ nvi = 4 # NPORTS
+ niqflint = 34 # NPORTS*NCPUS + NMSIX_EXTRA
+ nethctrl = 32 # NPORTS*NCPUS
+ neq = 66 # NPORTS*NCPUS * 2 (FL, ETHCTRL/TX) + 2 (EXTRA)
+ nexactf = 32 # NPORTS + adding 28 exact entries for FCoE
+ # which is OK since < MIN(SUM PF0..3, PF4)
+ # and we never load PF0..3 and PF4 concurrently
+ cmask = all # access to all channels
+ pmask = all # access to all four ports ...
+ nhash = 0
+ protocol = fcoe_initiator
+ tp_ddp = 2
+ fcoe_nfcf = 16
+ fcoe_nvnp = 32
+ fcoe_nssn = 1024
+
+# For Virtual functions, we only allow NIC functionality and we only allow
+# access to one port (1 << PF). Note that because of limitations in the
+# Scatter Gather Engine (SGE) hardware which checks writes to VF KDOORBELL
+# and GTS registers, the number of Ingress and Egress Queues must be a power
+# of 2.
+#
+[function "0/*"] # NVF
+ wx_caps = 0x82 # DMAQ | VF
+ r_caps = 0x86 # DMAQ | VF | PORT
+ nvi = 1 # 1 port
+ niqflint = 4 # 2 "Queue Sets" + NXIQ
+ nethctrl = 2 # 2 "Queue Sets"
+ neq = 4 # 2 "Queue Sets" * 2
+ nexactf = 4
+ cmask = all # access to all channels
+ pmask = 0x1 # access to only one port ...
+
+[function "1/*"] # NVF
+ wx_caps = 0x82 # DMAQ | VF
+ r_caps = 0x86 # DMAQ | VF | PORT
+ nvi = 1 # 1 port
+ niqflint = 4 # 2 "Queue Sets" + NXIQ
+ nethctrl = 2 # 2 "Queue Sets"
+ neq = 4 # 2 "Queue Sets" * 2
+ nexactf = 4
+ cmask = all # access to all channels
+ pmask = 0x2 # access to only one port ...
+
+[function "2/*"] # NVF
+ wx_caps = 0x82 # DMAQ | VF
+ r_caps = 0x86 # DMAQ | VF | PORT
+ nvi = 1 # 1 port
+ niqflint = 4 # 2 "Queue Sets" + NXIQ
+ nethctrl = 2 # 2 "Queue Sets"
+ neq = 4 # 2 "Queue Sets" * 2
+ nexactf = 4
+ cmask = all # access to all channels
+ pmask = 0x4 # access to only one port ...
+
+[function "3/*"] # NVF
+ wx_caps = 0x82 # DMAQ | VF
+ r_caps = 0x86 # DMAQ | VF | PORT
+ nvi = 1 # 1 port
+ niqflint = 4 # 2 "Queue Sets" + NXIQ
+ nethctrl = 2 # 2 "Queue Sets"
+ neq = 4 # 2 "Queue Sets" * 2
+ nexactf = 4
+ cmask = all # access to all channels
+ pmask = 0x8 # access to only one port ...
+
+# MPS features a 196608 bytes ingress buffer that is used for ingress buffering
+# for packets from the wire as well as the loopback path of the L2 switch. The
+# folling params control how the buffer memory is distributed and the L2 flow
+# control settings:
+#
+# bg_mem: %-age of mem to use for port/buffer group
+# lpbk_mem: %-age of port/bg mem to use for loopback
+# hwm: high watermark; bytes available when starting to send pause
+# frames (in units of 0.1 MTU)
+# lwm: low watermark; bytes remaining when sending 'unpause' frame
+# (in inuits of 0.1 MTU)
+# dwm: minimum delta between high and low watermark (in units of 100
+# Bytes)
+#
+[port "0"]
+ dcb = ppp, dcbx # configure for DCB PPP and enable DCBX offload
+ bg_mem = 25
+ lpbk_mem = 25
+ hwm = 30
+ lwm = 15
+ dwm = 30
+
+[port "1"]
+ dcb = ppp, dcbx
+ bg_mem = 25
+ lpbk_mem = 25
+ hwm = 30
+ lwm = 15
+ dwm = 30
+
+[port "2"]
+ dcb = ppp, dcbx
+ bg_mem = 25
+ lpbk_mem = 25
+ hwm = 30
+ lwm = 15
+ dwm = 30
+
+[port "3"]
+ dcb = ppp, dcbx
+ bg_mem = 25
+ lpbk_mem = 25
+ hwm = 30
+ lwm = 15
+ dwm = 30
+
+[fini]
+ version = 0x14250007
+ checksum = 0xfcbadefb
+
+# Total resources used by above allocations:
+# Virtual Interfaces: 104
+# Ingress Queues/w Free Lists and Interrupts: 526
+# Egress Queues: 702
+# MPS TCAM Entries: 336
+# MSI-X Vectors: 736
+# Virtual Functions: 64
+#
+# $FreeBSD$
+#
diff --git a/sys/dev/cxgbe/common/t4fw_interface.h b/sys/dev/cxgbe/firmware/t4fw_interface.h
index 3eb9615..6d259a5 100644
--- a/sys/dev/cxgbe/common/t4fw_interface.h
+++ b/sys/dev/cxgbe/firmware/t4fw_interface.h
@@ -37,16 +37,23 @@
enum fw_retval {
FW_SUCCESS = 0, /* completed sucessfully */
FW_EPERM = 1, /* operation not permitted */
+ FW_ENOENT = 2, /* no such file or directory */
FW_EIO = 5, /* input/output error; hw bad */
- FW_ENOEXEC = 8, /* Exec format error; inv microcode */
+ FW_ENOEXEC = 8, /* exec format error; inv microcode */
FW_EAGAIN = 11, /* try again */
FW_ENOMEM = 12, /* out of memory */
FW_EFAULT = 14, /* bad address; fw bad */
FW_EBUSY = 16, /* resource busy */
- FW_EEXIST = 17, /* File exists */
+ FW_EEXIST = 17, /* file exists */
FW_EINVAL = 22, /* invalid argument */
+ FW_ENOSPC = 28, /* no space left on device */
FW_ENOSYS = 38, /* functionality not implemented */
FW_EPROTO = 71, /* protocol error */
+ FW_EADDRINUSE = 98, /* address already in use */
+ FW_EADDRNOTAVAIL = 99, /* cannot assigned requested address */
+ FW_ENETDOWN = 100, /* network is down */
+ FW_ENETUNREACH = 101, /* network is unreachable */
+ FW_ENOBUFS = 105, /* no buffer space available */
FW_ETIMEDOUT = 110, /* timeout */
FW_EINPROGRESS = 115, /* fw internal */
FW_SCSI_ABORT_REQUESTED = 128, /* */
@@ -62,6 +69,8 @@ enum fw_retval {
FW_ERR_RDEV_IMPL_LOGO = 138, /* */
FW_SCSI_UNDER_FLOW_ERR = 139, /* */
FW_SCSI_OVER_FLOW_ERR = 140, /* */
+ FW_SCSI_DDP_ERR = 141, /* DDP error*/
+ FW_SCSI_TASK_ERR = 142, /* No SCSI tasks available */
};
/******************************************************************************
@@ -89,7 +98,7 @@ enum fw_wr_opcodes {
FW_RI_INV_LSTAG_WR = 0x1a,
FW_RI_WR = 0x0d,
FW_ISCSI_NODE_WR = 0x4a,
- FW_LASTC2E_WR = 0x4b
+ FW_LASTC2E_WR = 0x50
};
/*
@@ -512,8 +521,14 @@ struct fw_eth_tx_pkt_wr {
__be64 r3;
};
+#define S_FW_ETH_TX_PKT_WR_IMMDLEN 0
+#define M_FW_ETH_TX_PKT_WR_IMMDLEN 0x1ff
+#define V_FW_ETH_TX_PKT_WR_IMMDLEN(x) ((x) << S_FW_ETH_TX_PKT_WR_IMMDLEN)
+#define G_FW_ETH_TX_PKT_WR_IMMDLEN(x) \
+ (((x) >> S_FW_ETH_TX_PKT_WR_IMMDLEN) & M_FW_ETH_TX_PKT_WR_IMMDLEN)
+
struct fw_eth_tx_pkts_wr {
- __be32 op_immdlen;
+ __be32 op_pkd;
__be32 equiq_to_len16;
__be32 r3;
__be16 plen;
@@ -537,7 +552,7 @@ enum fw_flowc_mnem {
FW_FLOWC_MNEM_RCVNXT,
FW_FLOWC_MNEM_SNDBUF,
FW_FLOWC_MNEM_MSS,
- FW_FLOWC_MEM_TXDATAPLEN_MAX,
+ FW_FLOWC_MNEM_TXDATAPLEN_MAX,
};
struct fw_flowc_mnemval {
@@ -1469,22 +1484,129 @@ struct fw_ri_wr {
#define G_FW_RI_WR_P2PTYPE(x) \
(((x) >> S_FW_RI_WR_P2PTYPE) & M_FW_RI_WR_P2PTYPE)
-#ifdef FOISCSI
+/******************************************************************************
+ * S C S I W O R K R E Q U E S T s
+ **********************************************/
+
+
+/******************************************************************************
+ * F O i S C S I W O R K R E Q U E S T s
+ **********************************************/
+
+#define ISCSI_NAME_MAX_LEN 224
+#define ISCSI_ALIAS_MAX_LEN 224
+
+enum session_type {
+ ISCSI_SESSION_DISCOVERY = 0,
+ ISCSI_SESSION_NORMAL,
+};
+
+enum digest_val {
+ DIGEST_NONE = 0,
+ DIGEST_CRC32,
+ DIGEST_BOTH,
+};
+
+enum fw_iscsi_subops {
+ NODE_ONLINE = 1,
+ SESS_ONLINE,
+ CONN_ONLINE,
+ NODE_OFFLINE,
+ SESS_OFFLINE,
+ CONN_OFFLINE,
+ NODE_STATS,
+ SESS_STATS,
+ CONN_STATS,
+ UPDATE_IOHANDLE,
+};
+
+struct fw_iscsi_node_attr {
+ __u8 name_len;
+ __u8 node_name[ISCSI_NAME_MAX_LEN];
+ __u8 alias_len;
+ __u8 node_alias[ISCSI_ALIAS_MAX_LEN];
+};
+
+struct fw_iscsi_sess_attr {
+ __u8 sess_type;
+ __u8 seq_inorder;
+ __u8 pdu_inorder;
+ __u8 immd_data_en;
+ __u8 init_r2t_en;
+ __u8 erl;
+ __be16 max_conn;
+ __be16 max_r2t;
+ __be16 time2wait;
+ __be16 time2retain;
+ __be32 max_burst;
+ __be32 first_burst;
+};
+
+struct fw_iscsi_conn_attr {
+ __u8 hdr_digest;
+ __u8 data_digest;
+ __be32 max_rcv_dsl;
+ __be16 dst_port;
+ __be32 dst_addr;
+ __be16 src_port;
+ __be32 src_addr;
+ __be32 ping_tmo;
+};
+
+struct fw_iscsi_node_stats {
+ __be16 sess_count;
+ __be16 chap_fail_count;
+ __be16 login_count;
+ __be16 r1;
+};
+
+struct fw_iscsi_sess_stats {
+ __be32 rxbytes;
+ __be32 txbytes;
+ __be32 scmd_count;
+ __be32 read_cmds;
+ __be32 write_cmds;
+ __be32 read_bytes;
+ __be32 write_bytes;
+ __be32 scsi_err_count;
+ __be32 scsi_rst_count;
+ __be32 iscsi_tmf_count;
+ __be32 conn_count;
+};
+
+struct fw_iscsi_conn_stats {
+ __be32 txbytes;
+ __be32 rxbytes;
+ __be32 dataout;
+ __be32 datain;
+};
+
struct fw_iscsi_node_wr {
__u8 opcode;
__u8 subop;
- __u8 node_attr_to_compl;
- __u8 len16;
- __u8 status;
- __u8 r2;
__be16 immd_len;
+ __be32 flowid_len16;
__be64 cookie;
+ __u8 node_attr_to_compl;
+ __u8 status;
+ __be16 r1;
__be32 node_id;
__be32 ctrl_handle;
__be32 io_handle;
- __be32 r3;
};
+#define S_FW_ISCSI_NODE_WR_FLOWID 8
+#define M_FW_ISCSI_NODE_WR_FLOWID 0xfffff
+#define V_FW_ISCSI_NODE_WR_FLOWID(x) ((x) << S_FW_ISCSI_NODE_WR_FLOWID)
+#define G_FW_ISCSI_NODE_WR_FLOWID(x) \
+ (((x) >> S_FW_ISCSI_NODE_WR_FLOWID) & M_FW_ISCSI_NODE_WR_FLOWID)
+
+#define S_FW_ISCSI_NODE_WR_LEN16 0
+#define M_FW_ISCSI_NODE_WR_LEN16 0xff
+#define V_FW_ISCSI_NODE_WR_LEN16(x) ((x) << S_FW_ISCSI_NODE_WR_LEN16)
+#define G_FW_ISCSI_NODE_WR_LEN16(x) \
+ (((x) >> S_FW_ISCSI_NODE_WR_LEN16) & M_FW_ISCSI_NODE_WR_LEN16)
+
#define S_FW_ISCSI_NODE_WR_NODE_ATTR 7
#define M_FW_ISCSI_NODE_WR_NODE_ATTR 0x1
#define V_FW_ISCSI_NODE_WR_NODE_ATTR(x) ((x) << S_FW_ISCSI_NODE_WR_NODE_ATTR)
@@ -1527,7 +1649,109 @@ struct fw_iscsi_node_wr {
(((x) >> S_FW_ISCSI_NODE_WR_COMPL) & M_FW_ISCSI_NODE_WR_COMPL)
#define F_FW_ISCSI_NODE_WR_COMPL V_FW_ISCSI_NODE_WR_COMPL(1U)
-#endif
+#define FW_ISCSI_NODE_INVALID_ID 0xffffffff
+
+struct fw_scsi_iscsi_data {
+ __u8 r0;
+ __u8 fbit_to_tattr;
+ __be16 r2;
+ __be32 r3;
+ __u8 lun[8];
+ __be32 r4;
+ __be32 dlen;
+ __be32 r5;
+ __be32 r6;
+ __u8 cdb[16];
+};
+
+#define S_FW_SCSI_ISCSI_DATA_FBIT 7
+#define M_FW_SCSI_ISCSI_DATA_FBIT 0x1
+#define V_FW_SCSI_ISCSI_DATA_FBIT(x) ((x) << S_FW_SCSI_ISCSI_DATA_FBIT)
+#define G_FW_SCSI_ISCSI_DATA_FBIT(x) \
+ (((x) >> S_FW_SCSI_ISCSI_DATA_FBIT) & M_FW_SCSI_ISCSI_DATA_FBIT)
+#define F_FW_SCSI_ISCSI_DATA_FBIT V_FW_SCSI_ISCSI_DATA_FBIT(1U)
+
+#define S_FW_SCSI_ISCSI_DATA_RBIT 6
+#define M_FW_SCSI_ISCSI_DATA_RBIT 0x1
+#define V_FW_SCSI_ISCSI_DATA_RBIT(x) ((x) << S_FW_SCSI_ISCSI_DATA_RBIT)
+#define G_FW_SCSI_ISCSI_DATA_RBIT(x) \
+ (((x) >> S_FW_SCSI_ISCSI_DATA_RBIT) & M_FW_SCSI_ISCSI_DATA_RBIT)
+#define F_FW_SCSI_ISCSI_DATA_RBIT V_FW_SCSI_ISCSI_DATA_RBIT(1U)
+
+#define S_FW_SCSI_ISCSI_DATA_WBIT 5
+#define M_FW_SCSI_ISCSI_DATA_WBIT 0x1
+#define V_FW_SCSI_ISCSI_DATA_WBIT(x) ((x) << S_FW_SCSI_ISCSI_DATA_WBIT)
+#define G_FW_SCSI_ISCSI_DATA_WBIT(x) \
+ (((x) >> S_FW_SCSI_ISCSI_DATA_WBIT) & M_FW_SCSI_ISCSI_DATA_WBIT)
+#define F_FW_SCSI_ISCSI_DATA_WBIT V_FW_SCSI_ISCSI_DATA_WBIT(1U)
+
+#define S_FW_SCSI_ISCSI_DATA_TATTR 0
+#define M_FW_SCSI_ISCSI_DATA_TATTR 0x7
+#define V_FW_SCSI_ISCSI_DATA_TATTR(x) ((x) << S_FW_SCSI_ISCSI_DATA_TATTR)
+#define G_FW_SCSI_ISCSI_DATA_TATTR(x) \
+ (((x) >> S_FW_SCSI_ISCSI_DATA_TATTR) & M_FW_SCSI_ISCSI_DATA_TATTR)
+
+#define FW_SCSI_ISCSI_DATA_TATTR_UNTAGGED 0
+#define FW_SCSI_ISCSI_DATA_TATTR_SIMPLE 1
+#define FW_SCSI_ISCSI_DATA_TATTR_ORDERED 2
+#define FW_SCSI_ISCSI_DATA_TATTR_HEADOQ 3
+#define FW_SCSI_ISCSI_DATA_TATTR_ACA 4
+
+#define FW_SCSI_ISCSI_TMF_OP 0x02
+#define FW_SCSI_ISCSI_ABORT_FUNC 0x01
+#define FW_SCSI_ISCSI_LUN_RESET_FUNC 0x05
+#define FW_SCSI_ISCSI_RESERVED_TAG 0xffffffff
+
+struct fw_scsi_iscsi_rsp {
+ __u8 r0;
+ __u8 sbit_to_uflow;
+ __u8 response;
+ __u8 status;
+ __be32 r4;
+ __u8 r5[32];
+ __be32 bidir_res_cnt;
+ __be32 res_cnt;
+ __u8 sense_data[128];
+};
+
+#define S_FW_SCSI_ISCSI_RSP_SBIT 7
+#define M_FW_SCSI_ISCSI_RSP_SBIT 0x1
+#define V_FW_SCSI_ISCSI_RSP_SBIT(x) ((x) << S_FW_SCSI_ISCSI_RSP_SBIT)
+#define G_FW_SCSI_ISCSI_RSP_SBIT(x) \
+ (((x) >> S_FW_SCSI_ISCSI_RSP_SBIT) & M_FW_SCSI_ISCSI_RSP_SBIT)
+#define F_FW_SCSI_ISCSI_RSP_SBIT V_FW_SCSI_ISCSI_RSP_SBIT(1U)
+
+#define S_FW_SCSI_ISCSI_RSP_BIDIR_OFLOW 4
+#define M_FW_SCSI_ISCSI_RSP_BIDIR_OFLOW 0x1
+#define V_FW_SCSI_ISCSI_RSP_BIDIR_OFLOW(x) \
+ ((x) << S_FW_SCSI_ISCSI_RSP_BIDIR_OFLOW)
+#define G_FW_SCSI_ISCSI_RSP_BIDIR_OFLOW(x) \
+ (((x) >> S_FW_SCSI_ISCSI_RSP_BIDIR_OFLOW) & \
+ M_FW_SCSI_ISCSI_RSP_BIDIR_OFLOW)
+#define F_FW_SCSI_ISCSI_RSP_BIDIR_OFLOW V_FW_SCSI_ISCSI_RSP_BIDIR_OFLOW(1U)
+
+#define S_FW_SCSI_ISCSI_RSP_BIDIR_UFLOW 3
+#define M_FW_SCSI_ISCSI_RSP_BIDIR_UFLOW 0x1
+#define V_FW_SCSI_ISCSI_RSP_BIDIR_UFLOW(x) \
+ ((x) << S_FW_SCSI_ISCSI_RSP_BIDIR_UFLOW)
+#define G_FW_SCSI_ISCSI_RSP_BIDIR_UFLOW(x) \
+ (((x) >> S_FW_SCSI_ISCSI_RSP_BIDIR_UFLOW) & \
+ M_FW_SCSI_ISCSI_RSP_BIDIR_UFLOW)
+#define F_FW_SCSI_ISCSI_RSP_BIDIR_UFLOW V_FW_SCSI_ISCSI_RSP_BIDIR_UFLOW(1U)
+
+#define S_FW_SCSI_ISCSI_RSP_OFLOW 2
+#define M_FW_SCSI_ISCSI_RSP_OFLOW 0x1
+#define V_FW_SCSI_ISCSI_RSP_OFLOW(x) ((x) << S_FW_SCSI_ISCSI_RSP_OFLOW)
+#define G_FW_SCSI_ISCSI_RSP_OFLOW(x) \
+ (((x) >> S_FW_SCSI_ISCSI_RSP_OFLOW) & M_FW_SCSI_ISCSI_RSP_OFLOW)
+#define F_FW_SCSI_ISCSI_RSP_OFLOW V_FW_SCSI_ISCSI_RSP_OFLOW(1U)
+
+#define S_FW_SCSI_ISCSI_RSP_UFLOW 1
+#define M_FW_SCSI_ISCSI_RSP_UFLOW 0x1
+#define V_FW_SCSI_ISCSI_RSP_UFLOW(x) ((x) << S_FW_SCSI_ISCSI_RSP_UFLOW)
+#define G_FW_SCSI_ISCSI_RSP_UFLOW(x) \
+ (((x) >> S_FW_SCSI_ISCSI_RSP_UFLOW) & M_FW_SCSI_ISCSI_RSP_UFLOW)
+#define F_FW_SCSI_ISCSI_RSP_UFLOW V_FW_SCSI_ISCSI_RSP_UFLOW(1U)
/******************************************************************************
* C O M M A N D s
@@ -1543,6 +1767,16 @@ struct fw_iscsi_node_wr {
*/
#define FW_CMD_MAX_TIMEOUT 10000
+/*
+ * If a host driver does a HELLO and discovers that there's already a MASTER
+ * selected, we may have to wait for that MASTER to finish issuing RESET,
+ * configuration and INITIALIZE commands. Also, there's a possibility that
+ * our own HELLO may get lost if it happens right as the MASTER is issuign a
+ * RESET command, so we need to be willing to make a few retries of our HELLO.
+ */
+#define FW_CMD_HELLO_TIMEOUT (3 * FW_CMD_MAX_TIMEOUT)
+#define FW_CMD_HELLO_RETRIES 3
+
enum fw_cmd_opcodes {
FW_LDST_CMD = 0x01,
FW_RESET_CMD = 0x03,
@@ -1575,10 +1809,11 @@ enum fw_cmd_opcodes {
FW_SCHED_CMD = 0x24,
FW_DEVLOG_CMD = 0x25,
FW_NETIF_CMD = 0x26,
+ FW_WATCHDOG_CMD = 0x27,
+ FW_CLIP_CMD = 0x28,
FW_LASTC2E_CMD = 0x40,
FW_ERROR_CMD = 0x80,
FW_DEBUG_CMD = 0x81,
-
};
enum fw_cmd_cap {
@@ -1696,7 +1931,7 @@ struct fw_ldst_cmd {
} addrval;
struct fw_ldst_idctxt {
__be32 physid;
- __be32 msg_pkd;
+ __be32 msg_ctxtflush;
__be32 ctxt_data7;
__be32 ctxt_data6;
__be32 ctxt_data5;
@@ -1769,6 +2004,13 @@ struct fw_ldst_cmd {
(((x) >> S_FW_LDST_CMD_MSG) & M_FW_LDST_CMD_MSG)
#define F_FW_LDST_CMD_MSG V_FW_LDST_CMD_MSG(1U)
+#define S_FW_LDST_CMD_CTXTFLUSH 30
+#define M_FW_LDST_CMD_CTXTFLUSH 0x1
+#define V_FW_LDST_CMD_CTXTFLUSH(x) ((x) << S_FW_LDST_CMD_CTXTFLUSH)
+#define G_FW_LDST_CMD_CTXTFLUSH(x) \
+ (((x) >> S_FW_LDST_CMD_CTXTFLUSH) & M_FW_LDST_CMD_CTXTFLUSH)
+#define F_FW_LDST_CMD_CTXTFLUSH V_FW_LDST_CMD_CTXTFLUSH(1U)
+
#define S_FW_LDST_CMD_PADDR 8
#define M_FW_LDST_CMD_PADDR 0x1f
#define V_FW_LDST_CMD_PADDR(x) ((x) << S_FW_LDST_CMD_PADDR)
@@ -1852,13 +2094,27 @@ struct fw_reset_cmd {
__be32 op_to_write;
__be32 retval_len16;
__be32 val;
- __be32 r3;
+ __be32 halt_pkd;
+};
+
+#define S_FW_RESET_CMD_HALT 31
+#define M_FW_RESET_CMD_HALT 0x1
+#define V_FW_RESET_CMD_HALT(x) ((x) << S_FW_RESET_CMD_HALT)
+#define G_FW_RESET_CMD_HALT(x) \
+ (((x) >> S_FW_RESET_CMD_HALT) & M_FW_RESET_CMD_HALT)
+#define F_FW_RESET_CMD_HALT V_FW_RESET_CMD_HALT(1U)
+
+enum {
+ FW_HELLO_CMD_STAGE_OS = 0,
+ FW_HELLO_CMD_STAGE_PREOS0 = 1,
+ FW_HELLO_CMD_STAGE_PREOS1 = 2,
+ FW_HELLO_CMD_STAGE_POSTOS = 3,
};
struct fw_hello_cmd {
__be32 op_to_write;
__be32 retval_len16;
- __be32 err_to_mbasyncnot;
+ __be32 err_to_clearinit;
__be32 fwrev;
};
@@ -1909,6 +2165,19 @@ struct fw_hello_cmd {
#define G_FW_HELLO_CMD_MBASYNCNOT(x) \
(((x) >> S_FW_HELLO_CMD_MBASYNCNOT) & M_FW_HELLO_CMD_MBASYNCNOT)
+#define S_FW_HELLO_CMD_STAGE 17
+#define M_FW_HELLO_CMD_STAGE 0x7
+#define V_FW_HELLO_CMD_STAGE(x) ((x) << S_FW_HELLO_CMD_STAGE)
+#define G_FW_HELLO_CMD_STAGE(x) \
+ (((x) >> S_FW_HELLO_CMD_STAGE) & M_FW_HELLO_CMD_STAGE)
+
+#define S_FW_HELLO_CMD_CLEARINIT 16
+#define M_FW_HELLO_CMD_CLEARINIT 0x1
+#define V_FW_HELLO_CMD_CLEARINIT(x) ((x) << S_FW_HELLO_CMD_CLEARINIT)
+#define G_FW_HELLO_CMD_CLEARINIT(x) \
+ (((x) >> S_FW_HELLO_CMD_CLEARINIT) & M_FW_HELLO_CMD_CLEARINIT)
+#define F_FW_HELLO_CMD_CLEARINIT V_FW_HELLO_CMD_CLEARINIT(1U)
+
struct fw_bye_cmd {
__be32 op_to_write;
__be32 retval_len16;
@@ -1989,6 +2258,8 @@ enum fw_caps_config_nic {
FW_CAPS_CONFIG_NIC = 0x00000001,
FW_CAPS_CONFIG_NIC_VM = 0x00000002,
FW_CAPS_CONFIG_NIC_IDS = 0x00000004,
+ FW_CAPS_CONFIG_NIC_UM = 0x00000008,
+ FW_CAPS_CONFIG_NIC_UM_ISGL = 0x00000010,
};
enum fw_caps_config_toe {
@@ -2015,9 +2286,16 @@ enum fw_caps_config_fcoe {
FW_CAPS_CONFIG_FCOE_CTRL_OFLD = 0x00000004,
};
+enum fw_memtype_cf {
+ FW_MEMTYPE_CF_EDC0 = 0x0,
+ FW_MEMTYPE_CF_EDC1 = 0x1,
+ FW_MEMTYPE_CF_EXTMEM = 0x2,
+ FW_MEMTYPE_CF_FLASH = 0x4,
+};
+
struct fw_caps_config_cmd {
__be32 op_to_write;
- __be32 retval_len16;
+ __be32 cfvalid_to_len16;
__be32 r2;
__be32 hwmbitmap;
__be16 nbmcaps;
@@ -2030,9 +2308,33 @@ struct fw_caps_config_cmd {
__be16 r4;
__be16 iscsicaps;
__be16 fcoecaps;
- __be32 r5;
- __be64 r6;
-};
+ __be32 cfcsum;
+ __be32 finiver;
+ __be32 finicsum;
+};
+
+#define S_FW_CAPS_CONFIG_CMD_CFVALID 27
+#define M_FW_CAPS_CONFIG_CMD_CFVALID 0x1
+#define V_FW_CAPS_CONFIG_CMD_CFVALID(x) ((x) << S_FW_CAPS_CONFIG_CMD_CFVALID)
+#define G_FW_CAPS_CONFIG_CMD_CFVALID(x) \
+ (((x) >> S_FW_CAPS_CONFIG_CMD_CFVALID) & M_FW_CAPS_CONFIG_CMD_CFVALID)
+#define F_FW_CAPS_CONFIG_CMD_CFVALID V_FW_CAPS_CONFIG_CMD_CFVALID(1U)
+
+#define S_FW_CAPS_CONFIG_CMD_MEMTYPE_CF 24
+#define M_FW_CAPS_CONFIG_CMD_MEMTYPE_CF 0x7
+#define V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(x) \
+ ((x) << S_FW_CAPS_CONFIG_CMD_MEMTYPE_CF)
+#define G_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(x) \
+ (((x) >> S_FW_CAPS_CONFIG_CMD_MEMTYPE_CF) & \
+ M_FW_CAPS_CONFIG_CMD_MEMTYPE_CF)
+
+#define S_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF 16
+#define M_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF 0xff
+#define V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(x) \
+ ((x) << S_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF)
+#define G_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(x) \
+ (((x) >> S_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF) & \
+ M_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF)
/*
* params command mnemonics
@@ -2056,15 +2358,17 @@ enum fw_params_param_dev {
* Lookup Engine
*/
FW_PARAMS_PARAM_DEV_FLOWC_BUFFIFO_SZ = 0x03,
- FW_PARAMS_PARAM_DEV_INTVER_NIC = 0x04,
- FW_PARAMS_PARAM_DEV_INTVER_VNIC = 0x05,
- FW_PARAMS_PARAM_DEV_INTVER_OFLD = 0x06,
- FW_PARAMS_PARAM_DEV_INTVER_RI = 0x07,
- FW_PARAMS_PARAM_DEV_INTVER_ISCSIPDU = 0x08,
- FW_PARAMS_PARAM_DEV_INTVER_ISCSI = 0x09,
- FW_PARAMS_PARAM_DEV_INTVER_FCOE = 0x0A,
+ FW_PARAMS_PARAM_DEV_INTFVER_NIC = 0x04,
+ FW_PARAMS_PARAM_DEV_INTFVER_VNIC = 0x05,
+ FW_PARAMS_PARAM_DEV_INTFVER_OFLD = 0x06,
+ FW_PARAMS_PARAM_DEV_INTFVER_RI = 0x07,
+ FW_PARAMS_PARAM_DEV_INTFVER_ISCSIPDU = 0x08,
+ FW_PARAMS_PARAM_DEV_INTFVER_ISCSI = 0x09,
+ FW_PARAMS_PARAM_DEV_INTFVER_FCOE = 0x0A,
FW_PARAMS_PARAM_DEV_FWREV = 0x0B,
FW_PARAMS_PARAM_DEV_TPREV = 0x0C,
+ FW_PARAMS_PARAM_DEV_CF = 0x0D,
+ FW_PARAMS_PARAM_DEV_BYPASS = 0x0E,
};
/*
@@ -2119,6 +2423,23 @@ enum fw_params_param_dmaq {
FW_PARAMS_PARAM_DMAQ_EQ_SCHEDCLASS_ETH = 0x12,
};
+/*
+ * dev bypass parameters; actions and modes
+ */
+enum fw_params_param_dev_bypass {
+
+ /* actions
+ */
+ FW_PARAMS_PARAM_DEV_BYPASS_PFAIL = 0x00,
+ FW_PARAMS_PARAM_DEV_BYPASS_CURRENT = 0x01,
+
+ /* modes
+ */
+ FW_PARAMS_PARAM_DEV_BYPASS_NORMAL = 0x00,
+ FW_PARAMS_PARAM_DEV_BYPASS_DROP = 0x1,
+ FW_PARAMS_PARAM_DEV_BYPASS_BYPASS = 0x2,
+};
+
#define S_FW_PARAMS_MNEM 24
#define M_FW_PARAMS_MNEM 0xff
#define V_FW_PARAMS_MNEM(x) ((x) << S_FW_PARAMS_MNEM)
@@ -2271,6 +2592,7 @@ struct fw_pfvf_cmd {
#define V_FW_PFVF_CMD_NETHCTRL(x) ((x) << S_FW_PFVF_CMD_NETHCTRL)
#define G_FW_PFVF_CMD_NETHCTRL(x) \
(((x) >> S_FW_PFVF_CMD_NETHCTRL) & M_FW_PFVF_CMD_NETHCTRL)
+
/*
* ingress queue type; the first 1K ingress queues can have associated 0,
* 1 or 2 free lists and an interrupt, all other ingress queues lack these
@@ -3518,6 +3840,7 @@ struct fw_eq_ofld_cmd {
#define V_FW_EQ_OFLD_CMD_EQSIZE(x) ((x) << S_FW_EQ_OFLD_CMD_EQSIZE)
#define G_FW_EQ_OFLD_CMD_EQSIZE(x) \
(((x) >> S_FW_EQ_OFLD_CMD_EQSIZE) & M_FW_EQ_OFLD_CMD_EQSIZE)
+
/* Macros for VIID parsing:
VIID - [10:8] PFN, [7] VI Valid, [6:0] VI number */
#define S_FW_VIID_PFN 8
@@ -4081,8 +4404,10 @@ enum fw_port_action {
FW_PORT_ACTION_L2_WOL_MODE_EN = 0x0012,
FW_PORT_ACTION_LPBK_TO_NORMAL = 0x0020,
FW_PORT_ACTION_L1_SS_LPBK_ASIC = 0x0021,
+ FW_PORT_ACTION_MAC_LPBK = 0x0022,
FW_PORT_ACTION_L1_WS_LPBK_ASIC = 0x0023,
FW_PORT_ACTION_L1_EXT_LPBK = 0x0026,
+ FW_PORT_ACTION_PCS_LPBK = 0x0028,
FW_PORT_ACTION_PHY_RESET = 0x0040,
FW_PORT_ACTION_PMA_RESET = 0x0041,
FW_PORT_ACTION_PCS_RESET = 0x0042,
@@ -4164,7 +4489,8 @@ struct fw_port_cmd {
struct fw_port_dcb_pgrate {
__u8 type;
__u8 apply_pkd;
- __u8 r10_lo[6];
+ __u8 r10_lo[5];
+ __u8 num_tcs_supported;
__u8 pgrate[8];
} pgrate;
struct fw_port_dcb_priorate {
@@ -4181,11 +4507,12 @@ struct fw_port_cmd {
} pfc;
struct fw_port_app_priority {
__u8 type;
- __u8 r10_lo[3];
- __u8 prio;
- __u8 sel;
+ __u8 r10[2];
+ __u8 idx;
+ __u8 user_prio_map;
+ __u8 sel_field;
__be16 protocolid;
- __u8 r12[8];
+ __be64 r12;
} app_priority;
} dcb;
} u;
@@ -4337,20 +4664,6 @@ struct fw_port_cmd {
(((x) >> S_FW_PORT_CMD_APPLY) & M_FW_PORT_CMD_APPLY)
#define F_FW_PORT_CMD_APPLY V_FW_PORT_CMD_APPLY(1U)
-#define S_FW_PORT_CMD_APPLY 7
-#define M_FW_PORT_CMD_APPLY 0x1
-#define V_FW_PORT_CMD_APPLY(x) ((x) << S_FW_PORT_CMD_APPLY)
-#define G_FW_PORT_CMD_APPLY(x) \
- (((x) >> S_FW_PORT_CMD_APPLY) & M_FW_PORT_CMD_APPLY)
-#define F_FW_PORT_CMD_APPLY V_FW_PORT_CMD_APPLY(1U)
-
-#define S_FW_PORT_CMD_APPLY 7
-#define M_FW_PORT_CMD_APPLY 0x1
-#define V_FW_PORT_CMD_APPLY(x) ((x) << S_FW_PORT_CMD_APPLY)
-#define G_FW_PORT_CMD_APPLY(x) \
- (((x) >> S_FW_PORT_CMD_APPLY) & M_FW_PORT_CMD_APPLY)
-#define F_FW_PORT_CMD_APPLY V_FW_PORT_CMD_APPLY(1U)
-
/*
* These are configured into the VPD and hence tools that generate
* VPD may use this enumeration.
@@ -4383,6 +4696,7 @@ enum fw_port_module_type {
FW_PORT_MOD_TYPE_TWINAX_PASSIVE = 0x4,
FW_PORT_MOD_TYPE_TWINAX_ACTIVE = 0x5,
FW_PORT_MOD_TYPE_LRM = 0x6,
+ FW_PORT_MOD_TYPE_ERROR = M_FW_PORT_CMD_MODTYPE - 3,
FW_PORT_MOD_TYPE_UNKNOWN = M_FW_PORT_CMD_MODTYPE - 2,
FW_PORT_MOD_TYPE_NOTSUPPORTED = M_FW_PORT_CMD_MODTYPE - 1,
FW_PORT_MOD_TYPE_NONE = M_FW_PORT_CMD_MODTYPE
@@ -5189,15 +5503,12 @@ struct fw_rss_vi_config_cmd {
#define F_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN \
V_FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN(1U)
-#define S_FW_RSS_VI_CONFIG_CMD_UDPEN 0
-#define M_FW_RSS_VI_CONFIG_CMD_UDPEN 0x1
-#define V_FW_RSS_VI_CONFIG_CMD_UDPEN(x) \
- ((x) << S_FW_RSS_VI_CONFIG_CMD_UDPEN)
-#define G_FW_RSS_VI_CONFIG_CMD_UDPEN(x) \
- (((x) >> S_FW_RSS_VI_CONFIG_CMD_UDPEN) & \
- M_FW_RSS_VI_CONFIG_CMD_UDPEN)
-#define F_FW_RSS_VI_CONFIG_CMD_UDPEN \
- V_FW_RSS_VI_CONFIG_CMD_UDPEN(1U)
+#define S_FW_RSS_VI_CONFIG_CMD_UDPEN 0
+#define M_FW_RSS_VI_CONFIG_CMD_UDPEN 0x1
+#define V_FW_RSS_VI_CONFIG_CMD_UDPEN(x) ((x) << S_FW_RSS_VI_CONFIG_CMD_UDPEN)
+#define G_FW_RSS_VI_CONFIG_CMD_UDPEN(x) \
+ (((x) >> S_FW_RSS_VI_CONFIG_CMD_UDPEN) & M_FW_RSS_VI_CONFIG_CMD_UDPEN)
+#define F_FW_RSS_VI_CONFIG_CMD_UDPEN V_FW_RSS_VI_CONFIG_CMD_UDPEN(1U)
enum fw_sched_sc {
FW_SCHED_SC_CONFIG = 0,
@@ -5352,103 +5663,97 @@ struct fw_devlog_cmd {
M_FW_DEVLOG_CMD_MEMADDR16_DEVLOG)
struct fw_netif_cmd {
- __be32 op_portid;
- __be32 retval_to_len16;
- __be32 add_to_ipv4gw;
- __be32 vlanid_mtuval;
+ __be32 op_to_ipv4gw;
+ __be32 retval_len16;
+ __be32 netifi_ifadridx;
+ __be32 portid_to_mtuval;
__be32 gwaddr;
__be32 addr;
__be32 nmask;
__be32 bcaddr;
};
-#define S_FW_NETIF_CMD_PORTID 0
-#define M_FW_NETIF_CMD_PORTID 0xf
-#define V_FW_NETIF_CMD_PORTID(x) ((x) << S_FW_NETIF_CMD_PORTID)
-#define G_FW_NETIF_CMD_PORTID(x) \
- (((x) >> S_FW_NETIF_CMD_PORTID) & M_FW_NETIF_CMD_PORTID)
-
-#define S_FW_NETIF_CMD_RETVAL 24
-#define M_FW_NETIF_CMD_RETVAL 0xff
-#define V_FW_NETIF_CMD_RETVAL(x) ((x) << S_FW_NETIF_CMD_RETVAL)
-#define G_FW_NETIF_CMD_RETVAL(x) \
- (((x) >> S_FW_NETIF_CMD_RETVAL) & M_FW_NETIF_CMD_RETVAL)
-
-#define S_FW_NETIF_CMD_IFIDX 16
-#define M_FW_NETIF_CMD_IFIDX 0xff
-#define V_FW_NETIF_CMD_IFIDX(x) ((x) << S_FW_NETIF_CMD_IFIDX)
-#define G_FW_NETIF_CMD_IFIDX(x) \
- (((x) >> S_FW_NETIF_CMD_IFIDX) & M_FW_NETIF_CMD_IFIDX)
-
-#define S_FW_NETIF_CMD_LEN16 0
-#define M_FW_NETIF_CMD_LEN16 0xff
-#define V_FW_NETIF_CMD_LEN16(x) ((x) << S_FW_NETIF_CMD_LEN16)
-#define G_FW_NETIF_CMD_LEN16(x) \
- (((x) >> S_FW_NETIF_CMD_LEN16) & M_FW_NETIF_CMD_LEN16)
-
-#define S_FW_NETIF_CMD_ADD 31
+#define S_FW_NETIF_CMD_ADD 20
#define M_FW_NETIF_CMD_ADD 0x1
#define V_FW_NETIF_CMD_ADD(x) ((x) << S_FW_NETIF_CMD_ADD)
#define G_FW_NETIF_CMD_ADD(x) \
(((x) >> S_FW_NETIF_CMD_ADD) & M_FW_NETIF_CMD_ADD)
#define F_FW_NETIF_CMD_ADD V_FW_NETIF_CMD_ADD(1U)
-#define S_FW_NETIF_CMD_LINK 30
+#define S_FW_NETIF_CMD_LINK 19
#define M_FW_NETIF_CMD_LINK 0x1
#define V_FW_NETIF_CMD_LINK(x) ((x) << S_FW_NETIF_CMD_LINK)
#define G_FW_NETIF_CMD_LINK(x) \
(((x) >> S_FW_NETIF_CMD_LINK) & M_FW_NETIF_CMD_LINK)
#define F_FW_NETIF_CMD_LINK V_FW_NETIF_CMD_LINK(1U)
-#define S_FW_NETIF_CMD_VLAN 29
+#define S_FW_NETIF_CMD_VLAN 18
#define M_FW_NETIF_CMD_VLAN 0x1
#define V_FW_NETIF_CMD_VLAN(x) ((x) << S_FW_NETIF_CMD_VLAN)
#define G_FW_NETIF_CMD_VLAN(x) \
(((x) >> S_FW_NETIF_CMD_VLAN) & M_FW_NETIF_CMD_VLAN)
#define F_FW_NETIF_CMD_VLAN V_FW_NETIF_CMD_VLAN(1U)
-#define S_FW_NETIF_CMD_MTU 28
+#define S_FW_NETIF_CMD_MTU 17
#define M_FW_NETIF_CMD_MTU 0x1
#define V_FW_NETIF_CMD_MTU(x) ((x) << S_FW_NETIF_CMD_MTU)
#define G_FW_NETIF_CMD_MTU(x) \
(((x) >> S_FW_NETIF_CMD_MTU) & M_FW_NETIF_CMD_MTU)
#define F_FW_NETIF_CMD_MTU V_FW_NETIF_CMD_MTU(1U)
-#define S_FW_NETIF_CMD_DHCP 27
+#define S_FW_NETIF_CMD_DHCP 16
#define M_FW_NETIF_CMD_DHCP 0x1
#define V_FW_NETIF_CMD_DHCP(x) ((x) << S_FW_NETIF_CMD_DHCP)
#define G_FW_NETIF_CMD_DHCP(x) \
(((x) >> S_FW_NETIF_CMD_DHCP) & M_FW_NETIF_CMD_DHCP)
#define F_FW_NETIF_CMD_DHCP V_FW_NETIF_CMD_DHCP(1U)
-#define S_FW_NETIF_CMD_IPV4BCADDR 3
+#define S_FW_NETIF_CMD_IPV4BCADDR 15
#define M_FW_NETIF_CMD_IPV4BCADDR 0x1
#define V_FW_NETIF_CMD_IPV4BCADDR(x) ((x) << S_FW_NETIF_CMD_IPV4BCADDR)
#define G_FW_NETIF_CMD_IPV4BCADDR(x) \
(((x) >> S_FW_NETIF_CMD_IPV4BCADDR) & M_FW_NETIF_CMD_IPV4BCADDR)
#define F_FW_NETIF_CMD_IPV4BCADDR V_FW_NETIF_CMD_IPV4BCADDR(1U)
-#define S_FW_NETIF_CMD_IPV4NMASK 2
+#define S_FW_NETIF_CMD_IPV4NMASK 14
#define M_FW_NETIF_CMD_IPV4NMASK 0x1
#define V_FW_NETIF_CMD_IPV4NMASK(x) ((x) << S_FW_NETIF_CMD_IPV4NMASK)
#define G_FW_NETIF_CMD_IPV4NMASK(x) \
(((x) >> S_FW_NETIF_CMD_IPV4NMASK) & M_FW_NETIF_CMD_IPV4NMASK)
#define F_FW_NETIF_CMD_IPV4NMASK V_FW_NETIF_CMD_IPV4NMASK(1U)
-#define S_FW_NETIF_CMD_IPV4ADDR 1
+#define S_FW_NETIF_CMD_IPV4ADDR 13
#define M_FW_NETIF_CMD_IPV4ADDR 0x1
#define V_FW_NETIF_CMD_IPV4ADDR(x) ((x) << S_FW_NETIF_CMD_IPV4ADDR)
#define G_FW_NETIF_CMD_IPV4ADDR(x) \
(((x) >> S_FW_NETIF_CMD_IPV4ADDR) & M_FW_NETIF_CMD_IPV4ADDR)
#define F_FW_NETIF_CMD_IPV4ADDR V_FW_NETIF_CMD_IPV4ADDR(1U)
-#define S_FW_NETIF_CMD_IPV4GW 0
+#define S_FW_NETIF_CMD_IPV4GW 12
#define M_FW_NETIF_CMD_IPV4GW 0x1
#define V_FW_NETIF_CMD_IPV4GW(x) ((x) << S_FW_NETIF_CMD_IPV4GW)
#define G_FW_NETIF_CMD_IPV4GW(x) \
(((x) >> S_FW_NETIF_CMD_IPV4GW) & M_FW_NETIF_CMD_IPV4GW)
#define F_FW_NETIF_CMD_IPV4GW V_FW_NETIF_CMD_IPV4GW(1U)
+#define S_FW_NETIF_CMD_NETIFI 8
+#define M_FW_NETIF_CMD_NETIFI 0xffffff
+#define V_FW_NETIF_CMD_NETIFI(x) ((x) << S_FW_NETIF_CMD_NETIFI)
+#define G_FW_NETIF_CMD_NETIFI(x) \
+ (((x) >> S_FW_NETIF_CMD_NETIFI) & M_FW_NETIF_CMD_NETIFI)
+
+#define S_FW_NETIF_CMD_IFADRIDX 0
+#define M_FW_NETIF_CMD_IFADRIDX 0xff
+#define V_FW_NETIF_CMD_IFADRIDX(x) ((x) << S_FW_NETIF_CMD_IFADRIDX)
+#define G_FW_NETIF_CMD_IFADRIDX(x) \
+ (((x) >> S_FW_NETIF_CMD_IFADRIDX) & M_FW_NETIF_CMD_IFADRIDX)
+
+#define S_FW_NETIF_CMD_PORTID 28
+#define M_FW_NETIF_CMD_PORTID 0xf
+#define V_FW_NETIF_CMD_PORTID(x) ((x) << S_FW_NETIF_CMD_PORTID)
+#define G_FW_NETIF_CMD_PORTID(x) \
+ (((x) >> S_FW_NETIF_CMD_PORTID) & M_FW_NETIF_CMD_PORTID)
+
#define S_FW_NETIF_CMD_VLANID 16
#define M_FW_NETIF_CMD_VLANID 0xfff
#define V_FW_NETIF_CMD_VLANID(x) ((x) << S_FW_NETIF_CMD_VLANID)
@@ -5461,6 +5766,42 @@ struct fw_netif_cmd {
#define G_FW_NETIF_CMD_MTUVAL(x) \
(((x) >> S_FW_NETIF_CMD_MTUVAL) & M_FW_NETIF_CMD_MTUVAL)
+enum fw_watchdog_actions {
+ FW_WATCHDOG_ACTION_FLR = 0x1,
+ FW_WATCHDOG_ACTION_BYPASS = 0x2,
+};
+
+#define FW_WATCHDOG_MAX_TIMEOUT_SECS 60
+
+struct fw_watchdog_cmd {
+ __be32 op_to_write;
+ __be32 retval_len16;
+ __be32 timeout;
+ __be32 actions;
+};
+
+struct fw_clip_cmd {
+ __be32 op_to_write;
+ __be32 alloc_to_len16;
+ __be64 ip_hi;
+ __be64 ip_lo;
+ __be32 r4[2];
+};
+
+#define S_FW_CLIP_CMD_ALLOC 31
+#define M_FW_CLIP_CMD_ALLOC 0x1
+#define V_FW_CLIP_CMD_ALLOC(x) ((x) << S_FW_CLIP_CMD_ALLOC)
+#define G_FW_CLIP_CMD_ALLOC(x) \
+ (((x) >> S_FW_CLIP_CMD_ALLOC) & M_FW_CLIP_CMD_ALLOC)
+#define F_FW_CLIP_CMD_ALLOC V_FW_CLIP_CMD_ALLOC(1U)
+
+#define S_FW_CLIP_CMD_FREE 30
+#define M_FW_CLIP_CMD_FREE 0x1
+#define V_FW_CLIP_CMD_FREE(x) ((x) << S_FW_CLIP_CMD_FREE)
+#define G_FW_CLIP_CMD_FREE(x) \
+ (((x) >> S_FW_CLIP_CMD_FREE) & M_FW_CLIP_CMD_FREE)
+#define F_FW_CLIP_CMD_FREE V_FW_CLIP_CMD_FREE(1U)
+
enum fw_error_type {
FW_ERROR_TYPE_EXCEPTION = 0x0,
FW_ERROR_TYPE_HWMODULE = 0x1,
@@ -5570,6 +5911,94 @@ struct fw_debug_cmd {
#define G_FW_DEBUG_CMD_TYPE(x) \
(((x) >> S_FW_DEBUG_CMD_TYPE) & M_FW_DEBUG_CMD_TYPE)
+
+/******************************************************************************
+ * P C I E F W R E G I S T E R
+ **************************************/
+
+/**
+ * Register definitions for the PCIE_FW register which the firmware uses
+ * to retain status across RESETs. This register should be considered
+ * as a READ-ONLY register for Host Software and only to be used to
+ * track firmware initialization/error state, etc.
+ */
+#define S_PCIE_FW_ERR 31
+#define M_PCIE_FW_ERR 0x1
+#define V_PCIE_FW_ERR(x) ((x) << S_PCIE_FW_ERR)
+#define G_PCIE_FW_ERR(x) (((x) >> S_PCIE_FW_ERR) & M_PCIE_FW_ERR)
+#define F_PCIE_FW_ERR V_PCIE_FW_ERR(1U)
+
+#define S_PCIE_FW_INIT 30
+#define M_PCIE_FW_INIT 0x1
+#define V_PCIE_FW_INIT(x) ((x) << S_PCIE_FW_INIT)
+#define G_PCIE_FW_INIT(x) (((x) >> S_PCIE_FW_INIT) & M_PCIE_FW_INIT)
+#define F_PCIE_FW_INIT V_PCIE_FW_INIT(1U)
+
+#define S_PCIE_FW_HALT 29
+#define M_PCIE_FW_HALT 0x1
+#define V_PCIE_FW_HALT(x) ((x) << S_PCIE_FW_HALT)
+#define G_PCIE_FW_HALT(x) (((x) >> S_PCIE_FW_HALT) & M_PCIE_FW_HALT)
+#define F_PCIE_FW_HALT V_PCIE_FW_HALT(1U)
+
+#define S_PCIE_FW_STAGE 21
+#define M_PCIE_FW_STAGE 0x7
+#define V_PCIE_FW_STAGE(x) ((x) << S_PCIE_FW_STAGE)
+#define G_PCIE_FW_STAGE(x) (((x) >> S_PCIE_FW_STAGE) & M_PCIE_FW_STAGE)
+
+#define S_PCIE_FW_ASYNCNOT_VLD 20
+#define M_PCIE_FW_ASYNCNOT_VLD 0x1
+#define V_PCIE_FW_ASYNCNOT_VLD(x) \
+ ((x) << S_PCIE_FW_ASYNCNOT_VLD)
+#define G_PCIE_FW_ASYNCNOT_VLD(x) \
+ (((x) >> S_PCIE_FW_ASYNCNOT_VLD) & M_PCIE_FW_ASYNCNOT_VLD)
+#define F_PCIE_FW_ASYNCNOT_VLD V_PCIE_FW_ASYNCNOT_VLD(1U)
+
+#define S_PCIE_FW_ASYNCNOTINT 19
+#define M_PCIE_FW_ASYNCNOTINT 0x1
+#define V_PCIE_FW_ASYNCNOTINT(x) \
+ ((x) << S_PCIE_FW_ASYNCNOTINT)
+#define G_PCIE_FW_ASYNCNOTINT(x) \
+ (((x) >> S_PCIE_FW_ASYNCNOTINT) & M_PCIE_FW_ASYNCNOTINT)
+#define F_PCIE_FW_ASYNCNOTINT V_PCIE_FW_ASYNCNOTINT(1U)
+
+#define S_PCIE_FW_ASYNCNOT 16
+#define M_PCIE_FW_ASYNCNOT 0x7
+#define V_PCIE_FW_ASYNCNOT(x) ((x) << S_PCIE_FW_ASYNCNOT)
+#define G_PCIE_FW_ASYNCNOT(x) \
+ (((x) >> S_PCIE_FW_ASYNCNOT) & M_PCIE_FW_ASYNCNOT)
+
+#define S_PCIE_FW_MASTER_VLD 15
+#define M_PCIE_FW_MASTER_VLD 0x1
+#define V_PCIE_FW_MASTER_VLD(x) ((x) << S_PCIE_FW_MASTER_VLD)
+#define G_PCIE_FW_MASTER_VLD(x) \
+ (((x) >> S_PCIE_FW_MASTER_VLD) & M_PCIE_FW_MASTER_VLD)
+#define F_PCIE_FW_MASTER_VLD V_PCIE_FW_MASTER_VLD(1U)
+
+#define S_PCIE_FW_MASTER 12
+#define M_PCIE_FW_MASTER 0x7
+#define V_PCIE_FW_MASTER(x) ((x) << S_PCIE_FW_MASTER)
+#define G_PCIE_FW_MASTER(x) (((x) >> S_PCIE_FW_MASTER) & M_PCIE_FW_MASTER)
+
+#define S_PCIE_FW_RESET_VLD 11
+#define M_PCIE_FW_RESET_VLD 0x1
+#define V_PCIE_FW_RESET_VLD(x) ((x) << S_PCIE_FW_RESET_VLD)
+#define G_PCIE_FW_RESET_VLD(x) \
+ (((x) >> S_PCIE_FW_RESET_VLD) & M_PCIE_FW_RESET_VLD)
+#define F_PCIE_FW_RESET_VLD V_PCIE_FW_RESET_VLD(1U)
+
+#define S_PCIE_FW_RESET 8
+#define M_PCIE_FW_RESET 0x7
+#define V_PCIE_FW_RESET(x) ((x) << S_PCIE_FW_RESET)
+#define G_PCIE_FW_RESET(x) \
+ (((x) >> S_PCIE_FW_RESET) & M_PCIE_FW_RESET)
+
+#define S_PCIE_FW_REGISTERED 0
+#define M_PCIE_FW_REGISTERED 0xff
+#define V_PCIE_FW_REGISTERED(x) ((x) << S_PCIE_FW_REGISTERED)
+#define G_PCIE_FW_REGISTERED(x) \
+ (((x) >> S_PCIE_FW_REGISTERED) & M_PCIE_FW_REGISTERED)
+
+
/******************************************************************************
* B I N A R Y H E A D E R F O R M A T
**********************************************/
@@ -5579,7 +6008,7 @@ struct fw_debug_cmd {
*/
struct fw_hdr {
__u8 ver;
- __u8 reserved1;
+ __u8 chip; /* terminator chip family */
__be16 len512; /* bin length in units of 512-bytes */
__be32 fw_ver; /* firmware version */
__be32 tp_microcode_ver; /* tcp processor microcode version */
@@ -5591,7 +6020,16 @@ struct fw_hdr {
__u8 intfver_iscsi;
__u8 intfver_fcoe;
__u8 reserved2;
- __be32 reserved3[27];
+ __u32 reserved3;
+ __u32 reserved4;
+ __u32 reserved5;
+ __be32 flags;
+ __be32 reserved6[23];
+};
+
+enum fw_hdr_chip {
+ FW_HDR_CHIP_T4,
+ FW_HDR_CHIP_T5
};
#define S_FW_HDR_FW_VER_MAJOR 24
@@ -5622,4 +6060,18 @@ struct fw_hdr {
#define G_FW_HDR_FW_VER_BUILD(x) \
(((x) >> S_FW_HDR_FW_VER_BUILD) & M_FW_HDR_FW_VER_BUILD)
+enum fw_hdr_intfver {
+ FW_HDR_INTFVER_NIC = 0x00,
+ FW_HDR_INTFVER_VNIC = 0x00,
+ FW_HDR_INTFVER_OFLD = 0x00,
+ FW_HDR_INTFVER_RI = 0x00,
+ FW_HDR_INTFVER_ISCSIPDU = 0x00,
+ FW_HDR_INTFVER_ISCSI = 0x00,
+ FW_HDR_INTFVER_FCOE = 0x00,
+};
+
+enum fw_hdr_flags {
+ FW_HDR_FLAGS_RESET_HALT = 0x00000001,
+};
+
#endif /* _T4FW_INTERFACE_H_ */
diff --git a/sys/dev/cxgbe/offload.h b/sys/dev/cxgbe/offload.h
index f31b840..f6ada9d 100644
--- a/sys/dev/cxgbe/offload.h
+++ b/sys/dev/cxgbe/offload.h
@@ -31,15 +31,18 @@
#ifndef __T4_OFFLOAD_H__
#define __T4_OFFLOAD_H__
-/* CPL message priority levels */
-enum {
- CPL_PRIORITY_DATA = 0, /* data messages */
- CPL_PRIORITY_SETUP = 1, /* connection setup messages */
- CPL_PRIORITY_TEARDOWN = 0, /* connection teardown messages */
- CPL_PRIORITY_LISTEN = 1, /* listen start/stop messages */
- CPL_PRIORITY_ACK = 1, /* RX ACK messages */
- CPL_PRIORITY_CONTROL = 1 /* control messages */
-};
+/* XXX: flagrant misuse of mbuf fields (during tx by TOM) */
+#define MBUF_EQ(m) (*((void **)(&(m)->m_pkthdr.rcvif)))
+/* These have to work for !M_PKTHDR so we use a field from m_hdr. */
+#define MBUF_TX_CREDITS(m) ((m)->m_hdr.pad[0])
+#define MBUF_DMA_MAPPED(m) ((m)->m_hdr.pad[1])
+
+#define INIT_ULPTX_WR(w, wrlen, atomic, tid) do { \
+ (w)->wr.wr_hi = htonl(V_FW_WR_OP(FW_ULPTX_WR) | V_FW_WR_ATOMIC(atomic)); \
+ (w)->wr.wr_mid = htonl(V_FW_WR_LEN16(DIV_ROUND_UP(wrlen, 16)) | \
+ V_FW_WR_FLOWID(tid)); \
+ (w)->wr.wr_lo = cpu_to_be64(0); \
+} while (0)
#define INIT_TP_WR(w, tid) do { \
(w)->wr.wr_hi = htonl(V_FW_WR_OP(FW_TP_WR) | \
@@ -49,13 +52,19 @@ enum {
(w)->wr.wr_lo = cpu_to_be64(0); \
} while (0)
+#define INIT_TP_WR_MIT_CPL(w, cpl, tid) do { \
+ INIT_TP_WR(w, tid); \
+ OPCODE_TID(w) = htonl(MK_OPCODE_TID(cpl, tid)); \
+} while (0)
+
/*
* Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
*/
#define MAX_ATIDS 8192U
-struct serv_entry {
+union serv_entry {
void *data;
+ union serv_entry *next;
};
union aopen_entry {
@@ -71,8 +80,7 @@ struct tid_info {
void **tid_tab;
unsigned int ntids;
- struct serv_entry *stid_tab;
- unsigned long *stid_bmap;
+ union serv_entry *stid_tab;
unsigned int nstids;
unsigned int stid_base;
@@ -84,10 +92,15 @@ struct tid_info {
unsigned int ftid_base;
unsigned int ftids_in_use;
+ struct mtx atid_lock;
union aopen_entry *afree;
unsigned int atids_in_use;
+ struct mtx stid_lock;
+ union serv_entry *sfree;
unsigned int stids_in_use;
+
+ unsigned int tids_in_use;
};
struct t4_range {
@@ -101,6 +114,40 @@ struct t4_virt_res { /* virtualized HW resources */
struct t4_range stag;
struct t4_range rq;
struct t4_range pbl;
+ struct t4_range qp;
+ struct t4_range cq;
+ struct t4_range ocq;
};
+#ifndef TCP_OFFLOAD_DISABLE
+enum {
+ ULD_TOM = 1,
+};
+
+struct adapter;
+struct port_info;
+struct uld_info {
+ SLIST_ENTRY(uld_info) link;
+ int refcount;
+ int uld_id;
+ int (*attach)(struct adapter *, void **);
+ int (*detach)(void *);
+};
+
+struct uld_softc {
+ struct uld_info *uld;
+ void *softc;
+};
+
+struct tom_tunables {
+ int sndbuf;
+ int ddp;
+ int indsz;
+ int ddp_thres;
+};
+
+int t4_register_uld(struct uld_info *);
+int t4_unregister_uld(struct uld_info *);
+#endif
+
#endif
diff --git a/sys/dev/cxgbe/osdep.h b/sys/dev/cxgbe/osdep.h
index 85a8206..40ed61b 100644
--- a/sys/dev/cxgbe/osdep.h
+++ b/sys/dev/cxgbe/osdep.h
@@ -124,6 +124,7 @@ typedef boolean_t bool;
#define PCI_EXP_LNKSTA PCIR_EXPRESS_LINK_STA
#define PCI_EXP_LNKSTA_CLS PCIM_LINK_STA_SPEED
#define PCI_EXP_LNKSTA_NLW PCIM_LINK_STA_WIDTH
+#define PCI_EXP_DEVCTL2 0x28
static inline int
ilog2(long x)
diff --git a/sys/dev/cxgbe/t4_ioctl.h b/sys/dev/cxgbe/t4_ioctl.h
index ecc2c3d..2a3fa39 100644
--- a/sys/dev/cxgbe/t4_ioctl.h
+++ b/sys/dev/cxgbe/t4_ioctl.h
@@ -47,6 +47,8 @@ enum {
T4_SET_FILTER, /* program a filter */
T4_DEL_FILTER, /* delete a filter */
T4_GET_SGE_CONTEXT, /* get SGE context for a queue */
+ T4_LOAD_FW, /* flash firmware */
+ T4_GET_MEM, /* read memory */
};
struct t4_reg {
@@ -62,6 +64,11 @@ struct t4_regdump {
uint32_t *data;
};
+struct t4_data {
+ uint32_t len;
+ uint8_t *data;
+};
+
/*
* A hardware filter is some valid combination of these.
*/
@@ -73,8 +80,8 @@ struct t4_regdump {
#define T4_FILTER_IP_DPORT 0x20 /* Destination IP port */
#define T4_FILTER_FCoE 0x40 /* Fibre Channel over Ethernet packet */
#define T4_FILTER_PORT 0x80 /* Physical ingress port */
-#define T4_FILTER_OVLAN 0x100 /* Outer VLAN ID */
-#define T4_FILTER_IVLAN 0x200 /* Inner VLAN ID */
+#define T4_FILTER_VNIC 0x100 /* VNIC id or outer VLAN */
+#define T4_FILTER_VLAN 0x200 /* VLAN ID */
#define T4_FILTER_IP_TOS 0x400 /* IPv4 TOS/IPv6 Traffic Class */
#define T4_FILTER_IP_PROTO 0x800 /* IP protocol */
#define T4_FILTER_ETH_TYPE 0x1000 /* Ethernet Type */
@@ -131,8 +138,8 @@ struct t4_filter_tuple {
* is used to select the global mode and all filters are limited to the
* set of fields allowed by the global mode.
*/
- uint16_t ovlan; /* outer VLAN */
- uint16_t ivlan; /* inner VLAN */
+ uint16_t vnic; /* VNIC id or outer VLAN tag */
+ uint16_t vlan; /* VLAN tag */
uint16_t ethtype; /* Ethernet type */
uint8_t tos; /* TOS/Traffic Type */
uint8_t proto; /* protocol type */
@@ -141,8 +148,8 @@ struct t4_filter_tuple {
uint32_t matchtype:3; /* MPS match type */
uint32_t frag:1; /* fragmentation extension header */
uint32_t macidx:9; /* exact match MAC index */
- uint32_t ivlan_vld:1; /* inner VLAN valid */
- uint32_t ovlan_vld:1; /* outer VLAN valid */
+ uint32_t vlan_vld:1; /* VLAN valid */
+ uint32_t vnic_vld:1; /* VNIC id/outer VLAN tag valid */
};
struct t4_filter_specification {
@@ -199,6 +206,12 @@ struct t4_sge_context {
uint32_t data[T4_SGE_CONTEXT_SIZE / 4];
};
+struct t4_mem_range {
+ uint32_t addr;
+ uint32_t len;
+ uint32_t *data;
+};
+
#define CHELSIO_T4_GETREG _IOWR('f', T4_GETREG, struct t4_reg)
#define CHELSIO_T4_SETREG _IOW('f', T4_SETREG, struct t4_reg)
#define CHELSIO_T4_REGDUMP _IOWR('f', T4_REGDUMP, struct t4_regdump)
@@ -209,4 +222,6 @@ struct t4_sge_context {
#define CHELSIO_T4_DEL_FILTER _IOW('f', T4_DEL_FILTER, struct t4_filter)
#define CHELSIO_T4_GET_SGE_CONTEXT _IOWR('f', T4_GET_SGE_CONTEXT, \
struct t4_sge_context)
+#define CHELSIO_T4_LOAD_FW _IOW('f', T4_LOAD_FW, struct t4_data)
+#define CHELSIO_T4_GET_MEM _IOW('f', T4_GET_MEM, struct t4_mem_range)
#endif
diff --git a/sys/dev/cxgbe/t4_l2t.c b/sys/dev/cxgbe/t4_l2t.c
index 31197b8..be206c1 100644
--- a/sys/dev/cxgbe/t4_l2t.c
+++ b/sys/dev/cxgbe/t4_l2t.c
@@ -37,7 +37,9 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/rwlock.h>
#include <sys/socket.h>
+#include <sys/sbuf.h>
#include <net/if.h>
+#include <net/if_types.h>
#include <net/ethernet.h>
#include <net/if_vlan_var.h>
#include <net/if_dl.h>
@@ -50,9 +52,26 @@ __FBSDID("$FreeBSD$");
#include "common/common.h"
#include "common/jhash.h"
#include "common/t4_msg.h"
-#include "offload.h"
#include "t4_l2t.h"
+/*
+ * Module locking notes: There is a RW lock protecting the L2 table as a
+ * whole plus a spinlock per L2T entry. Entry lookups and allocations happen
+ * under the protection of the table lock, individual entry changes happen
+ * while holding that entry's spinlock. The table lock nests outside the
+ * entry locks. Allocations of new entries take the table lock as writers so
+ * no other lookups can happen while allocating new entries. Entry updates
+ * take the table lock as readers so multiple entries can be updated in
+ * parallel. An L2T entry can be dropped by decrementing its reference count
+ * and therefore can happen in parallel with entry allocation but no entry
+ * can change state or increment its ref count during allocation as both of
+ * these perform lookups.
+ *
+ * Note: We do not take refereces to ifnets in this module because both
+ * the TOE and the sockets already hold references to the interfaces and the
+ * lifetime of an L2T entry is fully contained in the lifetime of the TOE.
+ */
+
/* identifies sync vs async L2T_WRITE_REQs */
#define S_SYNC_WR 12
#define V_SYNC_WR(x) ((x) << S_SYNC_WR)
@@ -76,34 +95,251 @@ struct l2t_data {
struct l2t_entry l2tab[L2T_SIZE];
};
+static int do_l2t_write_rpl(struct sge_iq *, const struct rss_header *,
+ struct mbuf *);
+
+#define VLAN_NONE 0xfff
+#define SA(x) ((struct sockaddr *)(x))
+#define SIN(x) ((struct sockaddr_in *)(x))
+#define SINADDR(x) (SIN(x)->sin_addr.s_addr)
+
/*
- * Module locking notes: There is a RW lock protecting the L2 table as a
- * whole plus a spinlock per L2T entry. Entry lookups and allocations happen
- * under the protection of the table lock, individual entry changes happen
- * while holding that entry's spinlock. The table lock nests outside the
- * entry locks. Allocations of new entries take the table lock as writers so
- * no other lookups can happen while allocating new entries. Entry updates
- * take the table lock as readers so multiple entries can be updated in
- * parallel. An L2T entry can be dropped by decrementing its reference count
- * and therefore can happen in parallel with entry allocation but no entry
- * can change state or increment its ref count during allocation as both of
- * these perform lookups.
- *
- * Note: We do not take refereces to ifnets in this module because both
- * the TOE and the sockets already hold references to the interfaces and the
- * lifetime of an L2T entry is fully contained in the lifetime of the TOE.
+ * Allocate a free L2T entry. Must be called with l2t_data.lock held.
*/
+static struct l2t_entry *
+alloc_l2e(struct l2t_data *d)
+{
+ struct l2t_entry *end, *e, **p;
+
+ rw_assert(&d->lock, RA_WLOCKED);
+
+ if (!atomic_load_acq_int(&d->nfree))
+ return (NULL);
+
+ /* there's definitely a free entry */
+ for (e = d->rover, end = &d->l2tab[L2T_SIZE]; e != end; ++e)
+ if (atomic_load_acq_int(&e->refcnt) == 0)
+ goto found;
+
+ for (e = d->l2tab; atomic_load_acq_int(&e->refcnt); ++e) ;
+found:
+ d->rover = e + 1;
+ atomic_subtract_int(&d->nfree, 1);
+
+ /*
+ * The entry we found may be an inactive entry that is
+ * presently in the hash table. We need to remove it.
+ */
+ if (e->state < L2T_STATE_SWITCHING) {
+ for (p = &d->l2tab[e->hash].first; *p; p = &(*p)->next) {
+ if (*p == e) {
+ *p = e->next;
+ e->next = NULL;
+ break;
+ }
+ }
+ }
+
+ e->state = L2T_STATE_UNUSED;
+ return (e);
+}
+
+/*
+ * Write an L2T entry. Must be called with the entry locked.
+ * The write may be synchronous or asynchronous.
+ */
+static int
+write_l2e(struct adapter *sc, struct l2t_entry *e, int sync)
+{
+ struct mbuf *m;
+ struct cpl_l2t_write_req *req;
+
+ mtx_assert(&e->lock, MA_OWNED);
+
+ if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
+ return (ENOMEM);
+
+ req = mtod(m, struct cpl_l2t_write_req *);
+ m->m_pkthdr.len = m->m_len = sizeof(*req);
+
+ INIT_TP_WR(req, 0);
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, e->idx |
+ V_SYNC_WR(sync) | V_TID_QID(sc->sge.fwq.abs_id)));
+ req->params = htons(V_L2T_W_PORT(e->lport) | V_L2T_W_NOREPLY(!sync));
+ req->l2t_idx = htons(e->idx);
+ req->vlan = htons(e->vlan);
+ memcpy(req->dst_mac, e->dmac, sizeof(req->dst_mac));
+
+ t4_mgmt_tx(sc, m);
+
+ if (sync && e->state != L2T_STATE_SWITCHING)
+ e->state = L2T_STATE_SYNC_WRITE;
+
+ return (0);
+}
+
+/*
+ * Allocate an L2T entry for use by a switching rule. Such need to be
+ * explicitly freed and while busy they are not on any hash chain, so normal
+ * address resolution updates do not see them.
+ */
+struct l2t_entry *
+t4_l2t_alloc_switching(struct l2t_data *d)
+{
+ struct l2t_entry *e;
+
+ rw_rlock(&d->lock);
+ e = alloc_l2e(d);
+ if (e) {
+ mtx_lock(&e->lock); /* avoid race with t4_l2t_free */
+ e->state = L2T_STATE_SWITCHING;
+ atomic_store_rel_int(&e->refcnt, 1);
+ mtx_unlock(&e->lock);
+ }
+ rw_runlock(&d->lock);
+ return e;
+}
+
+/*
+ * Sets/updates the contents of a switching L2T entry that has been allocated
+ * with an earlier call to @t4_l2t_alloc_switching.
+ */
+int
+t4_l2t_set_switching(struct adapter *sc, struct l2t_entry *e, uint16_t vlan,
+ uint8_t port, uint8_t *eth_addr)
+{
+ int rc;
+
+ e->vlan = vlan;
+ e->lport = port;
+ memcpy(e->dmac, eth_addr, ETHER_ADDR_LEN);
+ mtx_lock(&e->lock);
+ rc = write_l2e(sc, e, 0);
+ mtx_unlock(&e->lock);
+ return (rc);
+}
+
+int
+t4_init_l2t(struct adapter *sc, int flags)
+{
+ int i;
+ struct l2t_data *d;
+
+ d = malloc(sizeof(*d), M_CXGBE, M_ZERO | flags);
+ if (!d)
+ return (ENOMEM);
+
+ d->rover = d->l2tab;
+ atomic_store_rel_int(&d->nfree, L2T_SIZE);
+ rw_init(&d->lock, "L2T");
+
+ for (i = 0; i < L2T_SIZE; i++) {
+ d->l2tab[i].idx = i;
+ d->l2tab[i].state = L2T_STATE_UNUSED;
+ mtx_init(&d->l2tab[i].lock, "L2T_E", NULL, MTX_DEF);
+ atomic_store_rel_int(&d->l2tab[i].refcnt, 0);
+ }
+
+ sc->l2t = d;
+ t4_register_cpl_handler(sc, CPL_L2T_WRITE_RPL, do_l2t_write_rpl);
+
+ return (0);
+}
+
+int
+t4_free_l2t(struct l2t_data *d)
+{
+ int i;
+
+ for (i = 0; i < L2T_SIZE; i++)
+ mtx_destroy(&d->l2tab[i].lock);
+ rw_destroy(&d->lock);
+ free(d, M_CXGBE);
+
+ return (0);
+}
+
static inline unsigned int
vlan_prio(const struct l2t_entry *e)
{
return e->vlan >> 13;
}
+static char
+l2e_state(const struct l2t_entry *e)
+{
+ switch (e->state) {
+ case L2T_STATE_VALID: return 'V'; /* valid, fast-path entry */
+ case L2T_STATE_STALE: return 'S'; /* needs revalidation, but usable */
+ case L2T_STATE_SYNC_WRITE: return 'W';
+ case L2T_STATE_RESOLVING: return e->arpq_head ? 'A' : 'R';
+ case L2T_STATE_SWITCHING: return 'X';
+ default: return 'U';
+ }
+}
+
+int
+sysctl_l2t(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct l2t_data *l2t = sc->l2t;
+ struct l2t_entry *e;
+ struct sbuf *sb;
+ int rc, i, header = 0;
+ char ip[60];
+
+ if (l2t == NULL)
+ return (ENXIO);
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ e = &l2t->l2tab[0];
+ for (i = 0; i < L2T_SIZE; i++, e++) {
+ mtx_lock(&e->lock);
+ if (e->state == L2T_STATE_UNUSED)
+ goto skip;
+
+ if (header == 0) {
+ sbuf_printf(sb, " Idx IP address "
+ "Ethernet address VLAN/P LP State Users Port");
+ header = 1;
+ }
+ if (e->state == L2T_STATE_SWITCHING || e->v6)
+ ip[0] = 0;
+ else
+ snprintf(ip, sizeof(ip), "%s",
+ inet_ntoa(*(struct in_addr *)&e->addr[0]));
+
+ /* XXX: accessing lle probably not safe? */
+ sbuf_printf(sb, "\n%4u %-15s %02x:%02x:%02x:%02x:%02x:%02x %4d"
+ " %u %2u %c %5u %s",
+ e->idx, ip, e->dmac[0], e->dmac[1], e->dmac[2],
+ e->dmac[3], e->dmac[4], e->dmac[5],
+ e->vlan & 0xfff, vlan_prio(e), e->lport,
+ l2e_state(e), atomic_load_acq_int(&e->refcnt),
+ e->lle ? e->lle->lle_tbl->llt_ifp->if_xname : "");
+skip:
+ mtx_unlock(&e->lock);
+ }
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+#ifndef TCP_OFFLOAD_DISABLE
static inline void
l2t_hold(struct l2t_data *d, struct l2t_entry *e)
{
if (atomic_fetchadd_int(&e->refcnt, 1) == 0) /* 0 -> 1 transition */
- atomic_add_int(&d->nfree, -1);
+ atomic_subtract_int(&d->nfree, 1);
}
/*
@@ -154,38 +390,6 @@ addreq(const struct l2t_entry *e, const uint32_t *addr)
}
/*
- * Write an L2T entry. Must be called with the entry locked (XXX: really?).
- * The write may be synchronous or asynchronous.
- */
-static int
-write_l2e(struct adapter *sc, struct l2t_entry *e, int sync)
-{
- struct mbuf *m;
- struct cpl_l2t_write_req *req;
-
- if ((m = m_gethdr(M_NOWAIT, MT_DATA)) == NULL)
- return (ENOMEM);
-
- req = mtod(m, struct cpl_l2t_write_req *);
- m->m_pkthdr.len = m->m_len = sizeof(*req);
-
- INIT_TP_WR(req, 0);
- OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, e->idx |
- V_SYNC_WR(sync) | V_TID_QID(sc->sge.fwq.abs_id)));
- req->params = htons(V_L2T_W_PORT(e->lport) | V_L2T_W_NOREPLY(!sync));
- req->l2t_idx = htons(e->idx);
- req->vlan = htons(e->vlan);
- memcpy(req->dst_mac, e->dmac, sizeof(req->dst_mac));
-
- t4_mgmt_tx(sc, m);
-
- if (sync && e->state != L2T_STATE_SWITCHING)
- e->state = L2T_STATE_SYNC_WRITE;
-
- return (0);
-}
-
-/*
* Add a packet to an L2T entry's queue of packets awaiting resolution.
* Must be called with the entry's lock held.
*/
@@ -194,53 +398,133 @@ arpq_enqueue(struct l2t_entry *e, struct mbuf *m)
{
mtx_assert(&e->lock, MA_OWNED);
- m->m_next = NULL;
+ KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt not NULL", __func__));
if (e->arpq_head)
- e->arpq_tail->m_next = m;
+ e->arpq_tail->m_nextpkt = m;
else
e->arpq_head = m;
e->arpq_tail = m;
}
-/*
- * Allocate a free L2T entry. Must be called with l2t_data.lock held.
- */
-static struct l2t_entry *
-alloc_l2e(struct l2t_data *d)
+static inline void
+send_pending(struct adapter *sc, struct l2t_entry *e)
{
- struct l2t_entry *end, *e, **p;
+ struct mbuf *m, *next;
- rw_assert(&d->lock, RA_WLOCKED);
+ mtx_assert(&e->lock, MA_OWNED);
- if (!atomic_load_acq_int(&d->nfree))
- return (NULL);
+ for (m = e->arpq_head; m; m = next) {
+ next = m->m_nextpkt;
+ m->m_nextpkt = NULL;
+ t4_wrq_tx(sc, MBUF_EQ(m), m);
+ }
+ e->arpq_head = e->arpq_tail = NULL;
+}
- /* there's definitely a free entry */
- for (e = d->rover, end = &d->l2tab[L2T_SIZE]; e != end; ++e)
- if (atomic_load_acq_int(&e->refcnt) == 0)
- goto found;
+#ifdef INET
+/*
+ * Looks up and fills up an l2t_entry's lle. We grab all the locks that we need
+ * ourself, and update e->state at the end if e->lle was successfully filled.
+ *
+ * The lle passed in comes from arpresolve and is ignored as it does not appear
+ * to be of much use.
+ */
+static int
+l2t_fill_lle(struct adapter *sc, struct l2t_entry *e, struct llentry *unused)
+{
+ int rc = 0;
+ struct sockaddr_in sin;
+ struct ifnet *ifp = e->ifp;
+ struct llentry *lle;
- for (e = d->l2tab; atomic_load_acq_int(&e->refcnt); ++e) ;
-found:
- d->rover = e + 1;
- atomic_add_int(&d->nfree, -1);
+ bzero(&sin, sizeof(struct sockaddr_in));
+ if (e->v6)
+ panic("%s: IPv6 L2 resolution not supported yet.", __func__);
+
+ sin.sin_family = AF_INET;
+ sin.sin_len = sizeof(struct sockaddr_in);
+ memcpy(&sin.sin_addr, e->addr, sizeof(struct sockaddr_in));
+
+ mtx_assert(&e->lock, MA_NOTOWNED);
+ KASSERT(e->addr && ifp, ("%s: bad prep before call", __func__));
+
+ IF_AFDATA_LOCK(ifp);
+ lle = lla_lookup(LLTABLE(ifp), LLE_EXCLUSIVE, SA(&sin));
+ IF_AFDATA_UNLOCK(ifp);
+ if (!LLE_IS_VALID(lle))
+ return (ENOMEM);
+ if (!(lle->la_flags & LLE_VALID)) {
+ rc = EINVAL;
+ goto done;
+ }
+
+ LLE_ADDREF(lle);
+
+ mtx_lock(&e->lock);
+ if (e->state == L2T_STATE_RESOLVING) {
+ KASSERT(e->lle == NULL, ("%s: lle already valid", __func__));
+ e->lle = lle;
+ memcpy(e->dmac, &lle->ll_addr, ETHER_ADDR_LEN);
+ write_l2e(sc, e, 1);
+ } else {
+ KASSERT(e->lle == lle, ("%s: lle changed", __func__));
+ LLE_REMREF(lle);
+ }
+ mtx_unlock(&e->lock);
+done:
+ LLE_WUNLOCK(lle);
+ return (rc);
+}
+#endif
- /*
- * The entry we found may be an inactive entry that is
- * presently in the hash table. We need to remove it.
- */
- if (e->state < L2T_STATE_SWITCHING) {
- for (p = &d->l2tab[e->hash].first; *p; p = &(*p)->next) {
- if (*p == e) {
- *p = e->next;
- e->next = NULL;
- break;
- }
+int
+t4_l2t_send(struct adapter *sc, struct mbuf *m, struct l2t_entry *e)
+{
+#ifndef INET
+ return (EINVAL);
+#else
+ struct llentry *lle = NULL;
+ struct sockaddr_in sin;
+ struct ifnet *ifp = e->ifp;
+
+ if (e->v6)
+ panic("%s: IPv6 L2 resolution not supported yet.", __func__);
+
+ bzero(&sin, sizeof(struct sockaddr_in));
+ sin.sin_family = AF_INET;
+ sin.sin_len = sizeof(struct sockaddr_in);
+ memcpy(&sin.sin_addr, e->addr, sizeof(struct sockaddr_in));
+
+again:
+ switch (e->state) {
+ case L2T_STATE_STALE: /* entry is stale, kick off revalidation */
+ if (arpresolve(ifp, NULL, NULL, SA(&sin), e->dmac, &lle) == 0)
+ l2t_fill_lle(sc, e, lle);
+
+ /* Fall through */
+
+ case L2T_STATE_VALID: /* fast-path, send the packet on */
+ return t4_wrq_tx(sc, MBUF_EQ(m), m);
+
+ case L2T_STATE_RESOLVING:
+ case L2T_STATE_SYNC_WRITE:
+ mtx_lock(&e->lock);
+ if (e->state != L2T_STATE_SYNC_WRITE &&
+ e->state != L2T_STATE_RESOLVING) {
+ /* state changed by the time we got here */
+ mtx_unlock(&e->lock);
+ goto again;
}
+ arpq_enqueue(e, m);
+ mtx_unlock(&e->lock);
+
+ if (e->state == L2T_STATE_RESOLVING &&
+ arpresolve(ifp, NULL, NULL, SA(&sin), e->dmac, &lle) == 0)
+ l2t_fill_lle(sc, e, lle);
}
- e->state = L2T_STATE_UNUSED;
- return e;
+ return (0);
+#endif
}
/*
@@ -287,75 +571,214 @@ t4_l2t_release(struct l2t_entry *e)
t4_l2e_free(e);
}
+static int
+do_l2t_write_rpl(struct sge_iq *iq, const struct rss_header *rss,
+ struct mbuf *m)
+{
+ struct adapter *sc = iq->adapter;
+ const struct cpl_l2t_write_rpl *rpl = (const void *)(rss + 1);
+ unsigned int tid = GET_TID(rpl);
+ unsigned int idx = tid & (L2T_SIZE - 1);
+
+ if (__predict_false(rpl->status != CPL_ERR_NONE)) {
+ log(LOG_ERR,
+ "Unexpected L2T_WRITE_RPL status %u for entry %u\n",
+ rpl->status, idx);
+ return (EINVAL);
+ }
+
+ if (tid & F_SYNC_WR) {
+ struct l2t_entry *e = &sc->l2t->l2tab[idx];
+
+ mtx_lock(&e->lock);
+ if (e->state != L2T_STATE_SWITCHING) {
+ send_pending(sc, e);
+ e->state = L2T_STATE_VALID;
+ }
+ mtx_unlock(&e->lock);
+ }
+
+ return (0);
+}
+
/*
- * Allocate an L2T entry for use by a switching rule. Such need to be
- * explicitly freed and while busy they are not on any hash chain, so normal
- * address resolution updates do not see them.
+ * Reuse an L2T entry that was previously used for the same next hop.
+ */
+static void
+reuse_entry(struct l2t_entry *e)
+{
+ struct llentry *lle;
+
+ mtx_lock(&e->lock); /* avoid race with t4_l2t_free */
+ lle = e->lle;
+ if (lle) {
+ KASSERT(lle->la_flags & LLE_VALID,
+ ("%s: invalid lle stored in l2t_entry", __func__));
+
+ if (lle->la_expire >= time_uptime)
+ e->state = L2T_STATE_STALE;
+ else
+ e->state = L2T_STATE_VALID;
+ } else
+ e->state = L2T_STATE_RESOLVING;
+ mtx_unlock(&e->lock);
+}
+
+/*
+ * The TOE wants an L2 table entry that it can use to reach the next hop over
+ * the specified port. Produce such an entry - create one if needed.
+ *
+ * Note that the ifnet could be a pseudo-device like if_vlan, if_lagg, etc. on
+ * top of the real cxgbe interface.
*/
struct l2t_entry *
-t4_l2t_alloc_switching(struct l2t_data *d)
+t4_l2t_get(struct port_info *pi, struct ifnet *ifp, struct sockaddr *sa)
{
struct l2t_entry *e;
+ struct l2t_data *d = pi->adapter->l2t;
+ int addr_len;
+ uint32_t *addr;
+ int hash;
+ struct sockaddr_in6 *sin6;
+ unsigned int smt_idx = pi->port_id;
+
+ if (sa->sa_family == AF_INET) {
+ addr = (uint32_t *)&SINADDR(sa);
+ addr_len = sizeof(SINADDR(sa));
+ } else if (sa->sa_family == AF_INET6) {
+ sin6 = (struct sockaddr_in6 *)sa;
+ addr = (uint32_t *)&sin6->sin6_addr.s6_addr;
+ addr_len = sizeof(sin6->sin6_addr.s6_addr);
+ } else
+ return (NULL);
- rw_rlock(&d->lock);
+ hash = addr_hash(addr, addr_len, ifp->if_index);
+
+ rw_wlock(&d->lock);
+ for (e = d->l2tab[hash].first; e; e = e->next) {
+ if (!addreq(e, addr) && e->ifp == ifp && e->smt_idx == smt_idx){
+ l2t_hold(d, e);
+ if (atomic_load_acq_int(&e->refcnt) == 1)
+ reuse_entry(e);
+ goto done;
+ }
+ }
+
+ /* Need to allocate a new entry */
e = alloc_l2e(d);
if (e) {
mtx_lock(&e->lock); /* avoid race with t4_l2t_free */
- e->state = L2T_STATE_SWITCHING;
+ e->state = L2T_STATE_RESOLVING;
+ memcpy(e->addr, addr, addr_len);
+ e->ifindex = ifp->if_index;
+ e->smt_idx = smt_idx;
+ e->ifp = ifp;
+ e->hash = hash;
+ e->lport = pi->lport;
+ e->v6 = (addr_len == 16);
+ e->lle = NULL;
atomic_store_rel_int(&e->refcnt, 1);
+ if (ifp->if_type == IFT_L2VLAN)
+ VLAN_TAG(ifp, &e->vlan);
+ else
+ e->vlan = VLAN_NONE;
+ e->next = d->l2tab[hash].first;
+ d->l2tab[hash].first = e;
mtx_unlock(&e->lock);
}
- rw_runlock(&d->lock);
+done:
+ rw_wunlock(&d->lock);
return e;
}
/*
- * Sets/updates the contents of a switching L2T entry that has been allocated
- * with an earlier call to @t4_l2t_alloc_switching.
+ * Called when the host's neighbor layer makes a change to some entry that is
+ * loaded into the HW L2 table.
*/
-int
-t4_l2t_set_switching(struct adapter *sc, struct l2t_entry *e, uint16_t vlan,
- uint8_t port, uint8_t *eth_addr)
+void
+t4_l2t_update(struct adapter *sc, struct llentry *lle)
{
- e->vlan = vlan;
- e->lport = port;
- memcpy(e->dmac, eth_addr, ETHER_ADDR_LEN);
- return write_l2e(sc, e, 0);
-}
+ struct l2t_entry *e;
+ struct l2t_data *d = sc->l2t;
+ struct sockaddr *sa = L3_ADDR(lle);
+ struct llentry *old_lle = NULL;
+ uint32_t *addr = (uint32_t *)&SINADDR(sa);
+ struct ifnet *ifp = lle->lle_tbl->llt_ifp;
+ int hash = addr_hash(addr, sizeof(*addr), ifp->if_index);
+
+ KASSERT(d != NULL, ("%s: no L2 table", __func__));
+ LLE_WLOCK_ASSERT(lle);
+ KASSERT(lle->la_flags & LLE_VALID || lle->la_flags & LLE_DELETED,
+ ("%s: entry neither valid nor deleted.", __func__));
-struct l2t_data *
-t4_init_l2t(int flags)
-{
- int i;
- struct l2t_data *d;
+ rw_rlock(&d->lock);
+ for (e = d->l2tab[hash].first; e; e = e->next) {
+ if (!addreq(e, addr) && e->ifp == ifp) {
+ mtx_lock(&e->lock);
+ if (atomic_load_acq_int(&e->refcnt))
+ goto found;
+ e->state = L2T_STATE_STALE;
+ mtx_unlock(&e->lock);
+ break;
+ }
+ }
+ rw_runlock(&d->lock);
- d = malloc(sizeof(*d), M_CXGBE, M_ZERO | flags);
- if (!d)
- return (NULL);
+ /* The TOE has no interest in this LLE */
+ return;
- d->rover = d->l2tab;
- atomic_store_rel_int(&d->nfree, L2T_SIZE);
- rw_init(&d->lock, "L2T");
+ found:
+ rw_runlock(&d->lock);
- for (i = 0; i < L2T_SIZE; i++) {
- d->l2tab[i].idx = i;
- d->l2tab[i].state = L2T_STATE_UNUSED;
- mtx_init(&d->l2tab[i].lock, "L2T_E", NULL, MTX_DEF);
- atomic_store_rel_int(&d->l2tab[i].refcnt, 0);
- }
+ if (atomic_load_acq_int(&e->refcnt)) {
- return (d);
-}
+ /* Entry is referenced by at least 1 offloaded connection. */
-int
-t4_free_l2t(struct l2t_data *d)
-{
- int i;
+ /* Handle deletes first */
+ if (lle->la_flags & LLE_DELETED) {
+ if (lle == e->lle) {
+ e->lle = NULL;
+ e->state = L2T_STATE_RESOLVING;
+ LLE_REMREF(lle);
+ }
+ goto done;
+ }
- for (i = 0; i < L2T_SIZE; i++)
- mtx_destroy(&d->l2tab[i].lock);
- rw_destroy(&d->lock);
- free(d, M_CXGBE);
+ if (lle != e->lle) {
+ old_lle = e->lle;
+ LLE_ADDREF(lle);
+ e->lle = lle;
+ }
- return (0);
+ if (e->state == L2T_STATE_RESOLVING ||
+ memcmp(e->dmac, &lle->ll_addr, ETHER_ADDR_LEN)) {
+
+ /* unresolved -> resolved; or dmac changed */
+
+ memcpy(e->dmac, &lle->ll_addr, ETHER_ADDR_LEN);
+ write_l2e(sc, e, 1);
+ } else {
+
+ /* +ve reinforcement of a valid or stale entry */
+
+ }
+
+ e->state = L2T_STATE_VALID;
+
+ } else {
+ /*
+ * Entry was used previously but is unreferenced right now.
+ * e->lle has been released and NULL'd out by t4_l2t_free, or
+ * l2t_release is about to call t4_l2t_free and do that.
+ *
+ * Either way this is of no interest to us.
+ */
+ }
+
+done:
+ mtx_unlock(&e->lock);
+ if (old_lle)
+ LLE_FREE(old_lle);
}
+
+#endif
diff --git a/sys/dev/cxgbe/t4_l2t.h b/sys/dev/cxgbe/t4_l2t.h
index c5520c6..8004c9e 100644
--- a/sys/dev/cxgbe/t4_l2t.h
+++ b/sys/dev/cxgbe/t4_l2t.h
@@ -54,18 +54,26 @@ struct l2t_entry {
struct mbuf *arpq_head; /* list of mbufs awaiting resolution */
struct mbuf *arpq_tail;
struct mtx lock;
- volatile uint32_t refcnt; /* entry reference count */
+ volatile int refcnt; /* entry reference count */
uint16_t hash; /* hash bucket the entry is on */
uint8_t v6; /* whether entry is for IPv6 */
uint8_t lport; /* associated offload logical port */
uint8_t dmac[ETHER_ADDR_LEN]; /* next hop's MAC address */
};
-struct l2t_data *t4_init_l2t(int);
+int t4_init_l2t(struct adapter *, int);
int t4_free_l2t(struct l2t_data *);
struct l2t_entry *t4_l2t_alloc_switching(struct l2t_data *);
int t4_l2t_set_switching(struct adapter *, struct l2t_entry *, uint16_t,
uint8_t, uint8_t *);
void t4_l2t_release(struct l2t_entry *);
+int sysctl_l2t(SYSCTL_HANDLER_ARGS);
+
+#ifndef TCP_OFFLOAD_DISABLE
+struct l2t_entry *t4_l2t_get(struct port_info *, struct ifnet *,
+ struct sockaddr *);
+int t4_l2t_send(struct adapter *, struct mbuf *, struct l2t_entry *);
+void t4_l2t_update(struct adapter *, struct llentry *);
+#endif
#endif /* __T4_L2T_H */
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index adca421..37a4a7c 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -55,12 +55,10 @@ __FBSDID("$FreeBSD$");
#include <net/if_dl.h>
#include <net/if_vlan_var.h>
-#include "common/t4_hw.h"
#include "common/common.h"
#include "common/t4_msg.h"
#include "common/t4_regs.h"
#include "common/t4_regs_values.h"
-#include "common/t4fw_interface.h"
#include "t4_ioctl.h"
#include "t4_l2t.h"
@@ -122,115 +120,130 @@ static void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4 Ethernet driver and services");
-/*
- * Tunables.
- */
-static SYSCTL_NODE(_hw, OID_AUTO, cxgbe, CTLFLAG_RD, 0,
- "cxgbe driver parameters");
-
-static int force_firmware_install = 0;
-TUNABLE_INT("hw.cxgbe.force_firmware_install", &force_firmware_install);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, force_firmware_install, CTLFLAG_RDTUN,
- &force_firmware_install, 0, "install firmware on every attach.");
+static struct mtx t4_list_lock;
+static SLIST_HEAD(, adapter) t4_list;
+#ifndef TCP_OFFLOAD_DISABLE
+static struct mtx t4_uld_list_lock;
+static SLIST_HEAD(, uld_info) t4_uld_list;
+#endif
/*
- * Holdoff timer and packet counter values.
+ * Tunables. See tweak_tunables() too.
*/
-static unsigned int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200};
-static unsigned int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
/*
- * Max # of tx and rx queues to use for each 10G and 1G port.
+ * Number of queues for tx and rx, 10G and 1G, NIC and offload.
*/
-static unsigned int max_ntxq_10g = 8;
-TUNABLE_INT("hw.cxgbe.max_ntxq_10G_port", &max_ntxq_10g);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, max_ntxq_10G_port, CTLFLAG_RDTUN,
- &max_ntxq_10g, 0, "maximum number of tx queues per 10G port.");
-
-static unsigned int max_nrxq_10g = 8;
-TUNABLE_INT("hw.cxgbe.max_nrxq_10G_port", &max_nrxq_10g);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, max_nrxq_10G_port, CTLFLAG_RDTUN,
- &max_nrxq_10g, 0, "maximum number of rxq's (per 10G port).");
-
-static unsigned int max_ntxq_1g = 2;
-TUNABLE_INT("hw.cxgbe.max_ntxq_1G_port", &max_ntxq_1g);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, max_ntxq_1G_port, CTLFLAG_RDTUN,
- &max_ntxq_1g, 0, "maximum number of tx queues per 1G port.");
-
-static unsigned int max_nrxq_1g = 2;
-TUNABLE_INT("hw.cxgbe.max_nrxq_1G_port", &max_nrxq_1g);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, max_nrxq_1G_port, CTLFLAG_RDTUN,
- &max_nrxq_1g, 0, "maximum number of rxq's (per 1G port).");
+#define NTXQ_10G 16
+static int t4_ntxq10g = -1;
+TUNABLE_INT("hw.cxgbe.ntxq10g", &t4_ntxq10g);
+
+#define NRXQ_10G 8
+static int t4_nrxq10g = -1;
+TUNABLE_INT("hw.cxgbe.nrxq10g", &t4_nrxq10g);
+
+#define NTXQ_1G 4
+static int t4_ntxq1g = -1;
+TUNABLE_INT("hw.cxgbe.ntxq1g", &t4_ntxq1g);
+
+#define NRXQ_1G 2
+static int t4_nrxq1g = -1;
+TUNABLE_INT("hw.cxgbe.nrxq1g", &t4_nrxq1g);
+
+#ifndef TCP_OFFLOAD_DISABLE
+#define NOFLDTXQ_10G 8
+static int t4_nofldtxq10g = -1;
+TUNABLE_INT("hw.cxgbe.nofldtxq10g", &t4_nofldtxq10g);
+
+#define NOFLDRXQ_10G 2
+static int t4_nofldrxq10g = -1;
+TUNABLE_INT("hw.cxgbe.nofldrxq10g", &t4_nofldrxq10g);
+
+#define NOFLDTXQ_1G 2
+static int t4_nofldtxq1g = -1;
+TUNABLE_INT("hw.cxgbe.nofldtxq1g", &t4_nofldtxq1g);
+
+#define NOFLDRXQ_1G 1
+static int t4_nofldrxq1g = -1;
+TUNABLE_INT("hw.cxgbe.nofldrxq1g", &t4_nofldrxq1g);
+#endif
/*
* Holdoff parameters for 10G and 1G ports.
*/
-static unsigned int tmr_idx_10g = 1;
-TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &tmr_idx_10g);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx_10G, CTLFLAG_RDTUN,
- &tmr_idx_10g, 0,
- "default timer index for interrupt holdoff (10G ports).");
-
-static int pktc_idx_10g = 2;
-TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &pktc_idx_10g);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx_10G, CTLFLAG_RDTUN,
- &pktc_idx_10g, 0,
- "default pkt counter index for interrupt holdoff (10G ports).");
-
-static unsigned int tmr_idx_1g = 1;
-TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &tmr_idx_1g);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, holdoff_timer_idx_1G, CTLFLAG_RDTUN,
- &tmr_idx_1g, 0,
- "default timer index for interrupt holdoff (1G ports).");
-
-static int pktc_idx_1g = 2;
-TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &pktc_idx_1g);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, holdoff_pktc_idx_1G, CTLFLAG_RDTUN,
- &pktc_idx_1g, 0,
- "default pkt counter index for interrupt holdoff (1G ports).");
+#define TMR_IDX_10G 1
+static int t4_tmr_idx_10g = TMR_IDX_10G;
+TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_10G", &t4_tmr_idx_10g);
+
+#define PKTC_IDX_10G 2
+static int t4_pktc_idx_10g = PKTC_IDX_10G;
+TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_10G", &t4_pktc_idx_10g);
+
+#define TMR_IDX_1G 1
+static int t4_tmr_idx_1g = TMR_IDX_1G;
+TUNABLE_INT("hw.cxgbe.holdoff_timer_idx_1G", &t4_tmr_idx_1g);
+
+#define PKTC_IDX_1G 2
+static int t4_pktc_idx_1g = PKTC_IDX_1G;
+TUNABLE_INT("hw.cxgbe.holdoff_pktc_idx_1G", &t4_pktc_idx_1g);
/*
* Size (# of entries) of each tx and rx queue.
*/
-static unsigned int qsize_txq = TX_EQ_QSIZE;
-TUNABLE_INT("hw.cxgbe.qsize_txq", &qsize_txq);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, qsize_txq, CTLFLAG_RDTUN,
- &qsize_txq, 0, "default queue size of NIC tx queues.");
+static unsigned int t4_qsize_txq = TX_EQ_QSIZE;
+TUNABLE_INT("hw.cxgbe.qsize_txq", &t4_qsize_txq);
+
+static unsigned int t4_qsize_rxq = RX_IQ_QSIZE;
+TUNABLE_INT("hw.cxgbe.qsize_rxq", &t4_qsize_rxq);
-static unsigned int qsize_rxq = RX_IQ_QSIZE;
-TUNABLE_INT("hw.cxgbe.qsize_rxq", &qsize_rxq);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, qsize_rxq, CTLFLAG_RDTUN,
- &qsize_rxq, 0, "default queue size of NIC rx queues.");
+/*
+ * Interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively).
+ */
+static int t4_intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
+TUNABLE_INT("hw.cxgbe.interrupt_types", &t4_intr_types);
/*
- * Interrupt types allowed.
+ * Configuration file.
*/
-static int intr_types = INTR_MSIX | INTR_MSI | INTR_INTX;
-TUNABLE_INT("hw.cxgbe.interrupt_types", &intr_types);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, interrupt_types, CTLFLAG_RDTUN, &intr_types, 0,
- "interrupt types allowed (bits 0, 1, 2 = INTx, MSI, MSI-X respectively)");
+static char t4_cfg_file[32] = "default";
+TUNABLE_STR("hw.cxgbe.config_file", t4_cfg_file, sizeof(t4_cfg_file));
/*
- * Force the driver to use the same set of interrupts for all ports.
+ * ASIC features that will be used. Disable the ones you don't want so that the
+ * chip resources aren't wasted on features that will not be used.
*/
-static int intr_shared = 0;
-TUNABLE_INT("hw.cxgbe.interrupts_shared", &intr_shared);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, interrupts_shared, CTLFLAG_RDTUN,
- &intr_shared, 0, "interrupts shared between all ports");
+static int t4_linkcaps_allowed = 0; /* No DCBX, PPP, etc. by default */
+TUNABLE_INT("hw.cxgbe.linkcaps_allowed", &t4_linkcaps_allowed);
+
+static int t4_niccaps_allowed = FW_CAPS_CONFIG_NIC;
+TUNABLE_INT("hw.cxgbe.niccaps_allowed", &t4_niccaps_allowed);
+
+static int t4_toecaps_allowed = FW_CAPS_CONFIG_TOE;
+TUNABLE_INT("hw.cxgbe.toecaps_allowed", &t4_toecaps_allowed);
+
+static int t4_rdmacaps_allowed = 0;
+TUNABLE_INT("hw.cxgbe.rdmacaps_allowed", &t4_rdmacaps_allowed);
-static unsigned int filter_mode = HW_TPL_FR_MT_PR_IV_P_FC;
-TUNABLE_INT("hw.cxgbe.filter_mode", &filter_mode);
-SYSCTL_UINT(_hw_cxgbe, OID_AUTO, filter_mode, CTLFLAG_RDTUN,
- &filter_mode, 0, "default global filter mode.");
+static int t4_iscsicaps_allowed = 0;
+TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
+
+static int t4_fcoecaps_allowed = 0;
+TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
struct intrs_and_queues {
int intr_type; /* INTx, MSI, or MSI-X */
int nirq; /* Number of vectors */
- int intr_shared; /* Interrupts shared between all ports */
+ int intr_flags;
int ntxq10g; /* # of NIC txq's for each 10G port */
int nrxq10g; /* # of NIC rxq's for each 10G port */
int ntxq1g; /* # of NIC txq's for each 1G port */
int nrxq1g; /* # of NIC rxq's for each 1G port */
+#ifndef TCP_OFFLOAD_DISABLE
+ int nofldtxq10g; /* # of TOE txq's for each 10G port */
+ int nofldrxq10g; /* # of TOE rxq's for each 10G port */
+ int nofldtxq1g; /* # of TOE txq's for each 1G port */
+ int nofldrxq1g; /* # of TOE rxq's for each 1G port */
+#endif
};
struct filter_entry {
@@ -244,15 +257,6 @@ struct filter_entry {
};
enum {
- MEMWIN0_APERTURE = 2048,
- MEMWIN0_BASE = 0x1b800,
- MEMWIN1_APERTURE = 32768,
- MEMWIN1_BASE = 0x28000,
- MEMWIN2_APERTURE = 65536,
- MEMWIN2_BASE = 0x30000,
-};
-
-enum {
XGMAC_MTU = (1 << 0),
XGMAC_PROMISC = (1 << 1),
XGMAC_ALLMULTI = (1 << 2),
@@ -268,9 +272,11 @@ static void setup_memwin(struct adapter *);
static int cfg_itype_and_nqueues(struct adapter *, int, int,
struct intrs_and_queues *);
static int prep_firmware(struct adapter *);
-static int get_devlog_params(struct adapter *, struct devlog_params *);
-static int get_capabilities(struct adapter *, struct fw_caps_config_cmd *);
-static int get_params(struct adapter *, struct fw_caps_config_cmd *);
+static int upload_config_file(struct adapter *, const struct firmware *,
+ uint32_t *, uint32_t *);
+static int partition_resources(struct adapter *, const struct firmware *);
+static int get_params__pre_init(struct adapter *);
+static int get_params__post_init(struct adapter *);
static void t4_set_desc(struct adapter *);
static void build_medialist(struct port_info *);
static int update_mac_settings(struct port_info *, int);
@@ -278,24 +284,46 @@ static int cxgbe_init_locked(struct port_info *);
static int cxgbe_init_synchronized(struct port_info *);
static int cxgbe_uninit_locked(struct port_info *);
static int cxgbe_uninit_synchronized(struct port_info *);
-static int first_port_up(struct adapter *);
-static int last_port_down(struct adapter *);
+static int adapter_full_init(struct adapter *);
+static int adapter_full_uninit(struct adapter *);
+static int port_full_init(struct port_info *);
+static int port_full_uninit(struct port_info *);
+static void quiesce_eq(struct adapter *, struct sge_eq *);
+static void quiesce_iq(struct adapter *, struct sge_iq *);
+static void quiesce_fl(struct adapter *, struct sge_fl *);
static int t4_alloc_irq(struct adapter *, struct irq *, int rid,
- iq_intr_handler_t *, void *, char *);
+ driver_intr_t *, void *, char *);
static int t4_free_irq(struct adapter *, struct irq *);
static void reg_block_dump(struct adapter *, uint8_t *, unsigned int,
unsigned int);
static void t4_get_regs(struct adapter *, struct t4_regdump *, uint8_t *);
static void cxgbe_tick(void *);
+static int cpl_not_handled(struct sge_iq *, const struct rss_header *,
+ struct mbuf *);
static int t4_sysctls(struct adapter *);
static int cxgbe_sysctls(struct port_info *);
static int sysctl_int_array(SYSCTL_HANDLER_ARGS);
+static int sysctl_bitfield(SYSCTL_HANDLER_ARGS);
static int sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS);
static int sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS);
static int sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS);
static int sysctl_qsize_txq(SYSCTL_HANDLER_ARGS);
static int sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS);
+static int sysctl_cctrl(SYSCTL_HANDLER_ARGS);
+static int sysctl_cpl_stats(SYSCTL_HANDLER_ARGS);
+static int sysctl_ddp_stats(SYSCTL_HANDLER_ARGS);
static int sysctl_devlog(SYSCTL_HANDLER_ARGS);
+static int sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS);
+static int sysctl_hw_sched(SYSCTL_HANDLER_ARGS);
+static int sysctl_lb_stats(SYSCTL_HANDLER_ARGS);
+static int sysctl_meminfo(SYSCTL_HANDLER_ARGS);
+static int sysctl_path_mtus(SYSCTL_HANDLER_ARGS);
+static int sysctl_pm_stats(SYSCTL_HANDLER_ARGS);
+static int sysctl_rdma_stats(SYSCTL_HANDLER_ARGS);
+static int sysctl_tcp_stats(SYSCTL_HANDLER_ARGS);
+static int sysctl_tids(SYSCTL_HANDLER_ARGS);
+static int sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS);
+static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
static inline void txq_start(struct ifnet *, struct sge_txq *);
static uint32_t fconf_to_mode(uint32_t);
static uint32_t mode_to_fconf(uint32_t);
@@ -309,8 +337,15 @@ static int del_filter(struct adapter *, struct t4_filter *);
static void clear_filter(struct filter_entry *);
static int set_filter_wr(struct adapter *, int);
static int del_filter_wr(struct adapter *, int);
-void filter_rpl(struct adapter *, const struct cpl_set_tcb_rpl *);
+static int filter_rpl(struct sge_iq *, const struct rss_header *,
+ struct mbuf *);
static int get_sge_context(struct adapter *, struct t4_sge_context *);
+static int read_card_mem(struct adapter *, struct t4_mem_range *);
+#ifndef TCP_OFFLOAD_DISABLE
+static int toe_capability(struct port_info *, int);
+static int activate_uld(struct adapter *, int, struct uld_softc *);
+static int deactivate_uld(struct uld_softc *);
+#endif
static int t4_mod_event(module_t, int, void *);
struct t4_pciids {
@@ -332,6 +367,11 @@ struct t4_pciids {
{0x440a, 4, "Chelsio T404-BT"},
};
+#ifndef TCP_OFFLOAD_DISABLE
+/* This is used in service_iq() to get to the fl associated with an iq. */
+CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
+#endif
+
static int
t4_probe(device_t dev)
{
@@ -358,10 +398,11 @@ t4_attach(device_t dev)
{
struct adapter *sc;
int rc = 0, i, n10g, n1g, rqidx, tqidx;
- struct fw_caps_config_cmd caps;
- uint32_t p, v;
struct intrs_and_queues iaq;
struct sge *s;
+#ifndef TCP_OFFLOAD_DISABLE
+ int ofld_rqidx, ofld_tqidx;
+#endif
sc = device_get_softc(dev);
sc->dev = dev;
@@ -370,6 +411,8 @@ t4_attach(device_t dev)
pci_enable_busmaster(dev);
if (pci_find_cap(dev, PCIY_EXPRESS, &i) == 0) {
+ uint32_t v;
+
pci_set_max_read_req(dev, 4096);
v = pci_read_config(dev, i + PCIR_EXPRESS_DEVICE_CTL, 2);
v |= PCIM_EXP_CTL_RELAXED_ORD_ENABLE;
@@ -379,12 +422,22 @@ t4_attach(device_t dev)
snprintf(sc->lockname, sizeof(sc->lockname), "%s",
device_get_nameunit(dev));
mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
+ mtx_lock(&t4_list_lock);
+ SLIST_INSERT_HEAD(&t4_list, sc, link);
+ mtx_unlock(&t4_list_lock);
+
+ mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
+ TAILQ_INIT(&sc->sfl);
+ callout_init(&sc->sfl_callout, CALLOUT_MPSAFE);
rc = map_bars(sc);
if (rc != 0)
goto done; /* error message displayed already */
memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
+ for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++)
+ sc->cpl_handler[i] = cpl_not_handled;
+ t4_register_cpl_handler(sc, CPL_SET_TCB_RPL, filter_rpl);
/* Prepare the adapter for operation */
rc = -t4_prep_adapter(sc);
@@ -393,107 +446,75 @@ t4_attach(device_t dev)
goto done;
}
- /* Do this really early */
+ /*
+ * Do this really early, with the memory windows set up even before the
+ * character device. The userland tool's register i/o and mem read
+ * will work even in "recovery mode".
+ */
+ setup_memwin(sc);
sc->cdev = make_dev(&t4_cdevsw, device_get_unit(dev), UID_ROOT,
GID_WHEEL, 0600, "%s", device_get_nameunit(dev));
sc->cdev->si_drv1 = sc;
+ /* Go no further if recovery mode has been requested. */
+ if (TUNABLE_INT_FETCH("hw.cxgbe.sos", &i) && i != 0) {
+ device_printf(dev, "recovery mode.\n");
+ goto done;
+ }
+
/* Prepare the firmware for operation */
rc = prep_firmware(sc);
if (rc != 0)
goto done; /* error message displayed already */
- /* Read firmware devlog parameters */
- (void) get_devlog_params(sc, &sc->params.devlog);
-
- /* Get device capabilities and select which ones we'll use */
- rc = get_capabilities(sc, &caps);
- if (rc != 0) {
- device_printf(dev,
- "failed to initialize adapter capabilities: %d.\n", rc);
- goto done;
- }
+ rc = get_params__pre_init(sc);
+ if (rc != 0)
+ goto done; /* error message displayed already */
- /* Choose the global RSS mode. */
- rc = -t4_config_glbl_rss(sc, sc->mbox,
- FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
- F_FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
- F_FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ |
- F_FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
- if (rc != 0) {
- device_printf(dev,
- "failed to select global RSS mode: %d.\n", rc);
- goto done;
- }
+ rc = t4_sge_init(sc);
+ if (rc != 0)
+ goto done; /* error message displayed already */
- /* These are total (sum of all ports) limits for a bus driver */
- rc = -t4_cfg_pfvf(sc, sc->mbox, sc->pf, 0,
- 128, /* max # of egress queues */
- 64, /* max # of egress Ethernet or control queues */
- 64, /* max # of ingress queues with fl/interrupt */
- 0, /* max # of ingress queues without interrupt */
- 0, /* PCIe traffic class */
- 4, /* max # of virtual interfaces */
- M_FW_PFVF_CMD_CMASK, M_FW_PFVF_CMD_PMASK, 16,
- FW_CMD_CAP_PF, FW_CMD_CAP_PF);
- if (rc != 0) {
- device_printf(dev,
- "failed to configure pf/vf resources: %d.\n", rc);
- goto done;
+ if (sc->flags & MASTER_PF) {
+ /* get basic stuff going */
+ rc = -t4_fw_initialize(sc, sc->mbox);
+ if (rc != 0) {
+ device_printf(dev, "early init failed: %d.\n", rc);
+ goto done;
+ }
}
- /* Need this before sge_init */
- for (i = 0; i < SGE_NTIMERS; i++)
- sc->sge.timer_val[i] = min(intr_timer[i], 200U);
- for (i = 0; i < SGE_NCOUNTERS; i++)
- sc->sge.counter_val[i] = min(intr_pktcount[i], M_THRESHOLD_0);
-
- /* Also need the cooked value of cclk before sge_init */
- p = (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) |
- V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_CCLK));
- rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &p, &v);
- if (rc != 0) {
- device_printf(sc->dev,
- "failed to obtain core clock value: %d.\n", rc);
- goto done;
- }
- sc->params.vpd.cclk = v;
+ rc = get_params__post_init(sc);
+ if (rc != 0)
+ goto done; /* error message displayed already */
- t4_sge_init(sc);
+ if (sc->flags & MASTER_PF) {
- t4_set_filter_mode(sc, filter_mode);
- t4_set_reg_field(sc, A_TP_GLOBAL_CONFIG,
- V_FIVETUPLELOOKUP(M_FIVETUPLELOOKUP),
- V_FIVETUPLELOOKUP(M_FIVETUPLELOOKUP));
- t4_tp_wr_bits_indirect(sc, A_TP_INGRESS_CONFIG, F_CSUM_HAS_PSEUDO_HDR,
- F_LOOKUPEVERYPKT);
+ /* final tweaks to some settings */
- /* get basic stuff going */
- rc = -t4_early_init(sc, sc->mbox);
- if (rc != 0) {
- device_printf(dev, "early init failed: %d.\n", rc);
- goto done;
+ t4_load_mtus(sc, sc->params.mtus, sc->params.a_wnd,
+ sc->params.b_wnd);
+ t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
+ t4_set_reg_field(sc, A_TP_PARA_REG3, F_TUNNELCNGDROP0 |
+ F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | F_TUNNELCNGDROP3, 0);
+ t4_set_reg_field(sc, A_TP_PARA_REG5,
+ V_INDICATESIZE(M_INDICATESIZE) |
+ F_REARMDDPOFFSET | F_RESETDDPOFFSET,
+ V_INDICATESIZE(M_INDICATESIZE) |
+ F_REARMDDPOFFSET | F_RESETDDPOFFSET);
+ } else {
+ /*
+ * XXX: Verify that we can live with whatever the master driver
+ * has done so far, and hope that it doesn't change any global
+ * setting from underneath us in the future.
+ */
}
- rc = get_params(sc, &caps);
- if (rc != 0)
- goto done; /* error message displayed already */
-
- /* These are finalized by FW initialization, load their values now */
- v = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
- sc->params.tp.tre = G_TIMERRESOLUTION(v);
- sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(v);
- t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
-
- /* tweak some settings */
- t4_write_reg(sc, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) |
- V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) |
- V_KEEPALIVEMAXR1(4) | V_KEEPALIVEMAXR2(9));
- t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
- t4_set_reg_field(sc, A_TP_PARA_REG3, F_TUNNELCNGDROP0 |
- F_TUNNELCNGDROP1 | F_TUNNELCNGDROP2 | F_TUNNELCNGDROP3, 0);
+ t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1,
+ A_TP_VLAN_PRI_MAP);
- setup_memwin(sc);
+ for (i = 0; i < NCHAN; i++)
+ sc->params.tp.tx_modq[i] = i;
rc = t4_create_dma_tag(sc);
if (rc != 0)
@@ -532,31 +553,18 @@ t4_attach(device_t dev)
if (is_10G_port(pi)) {
n10g++;
- pi->tmr_idx = tmr_idx_10g;
- pi->pktc_idx = pktc_idx_10g;
+ pi->tmr_idx = t4_tmr_idx_10g;
+ pi->pktc_idx = t4_pktc_idx_10g;
} else {
n1g++;
- pi->tmr_idx = tmr_idx_1g;
- pi->pktc_idx = pktc_idx_1g;
+ pi->tmr_idx = t4_tmr_idx_1g;
+ pi->pktc_idx = t4_pktc_idx_1g;
}
pi->xact_addr_filt = -1;
- pi->qsize_rxq = max(qsize_rxq, 128);
- while (pi->qsize_rxq & 7)
- pi->qsize_rxq++;
- pi->qsize_txq = max(qsize_txq, 128);
-
- if (pi->qsize_rxq != qsize_rxq) {
- device_printf(dev,
- "using %d instead of %d as the rx queue size.\n",
- pi->qsize_rxq, qsize_rxq);
- }
- if (pi->qsize_txq != qsize_txq) {
- device_printf(dev,
- "using %d instead of %d as the tx queue size.\n",
- pi->qsize_txq, qsize_txq);
- }
+ pi->qsize_rxq = t4_qsize_rxq;
+ pi->qsize_txq = t4_qsize_txq;
pi->dev = device_add_child(dev, "cxgbe", -1);
if (pi->dev == NULL) {
@@ -566,14 +574,6 @@ t4_attach(device_t dev)
goto done;
}
device_set_softc(pi->dev, pi);
-
- setbit(&sc->registered_device_map, i);
- }
-
- if (sc->registered_device_map == 0) {
- device_printf(dev, "no usable ports\n");
- rc = ENXIO;
- goto done;
}
/*
@@ -585,20 +585,31 @@ t4_attach(device_t dev)
sc->intr_type = iaq.intr_type;
sc->intr_count = iaq.nirq;
+ sc->flags |= iaq.intr_flags;
s = &sc->sge;
s->nrxq = n10g * iaq.nrxq10g + n1g * iaq.nrxq1g;
s->ntxq = n10g * iaq.ntxq10g + n1g * iaq.ntxq1g;
s->neq = s->ntxq + s->nrxq; /* the free list in an rxq is an eq */
- s->neq += sc->params.nports; /* control queues, 1 per port */
+ s->neq += sc->params.nports + 1;/* ctrl queues: 1 per port + 1 mgmt */
s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */
- if (iaq.intr_shared)
- sc->flags |= INTR_SHARED;
- s->niq += NINTRQ(sc); /* interrupt queues */
- s->intrq = malloc(NINTRQ(sc) * sizeof(struct sge_iq), M_CXGBE,
- M_ZERO | M_WAITOK);
- s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_ctrlq), M_CXGBE,
+#ifndef TCP_OFFLOAD_DISABLE
+ if (is_offload(sc)) {
+
+ s->nofldrxq = n10g * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g;
+ s->nofldtxq = n10g * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g;
+ s->neq += s->nofldtxq + s->nofldrxq;
+ s->niq += s->nofldrxq;
+
+ s->ofld_rxq = malloc(s->nofldrxq * sizeof(struct sge_ofld_rxq),
+ M_CXGBE, M_ZERO | M_WAITOK);
+ s->ofld_txq = malloc(s->nofldtxq * sizeof(struct sge_wrq),
+ M_CXGBE, M_ZERO | M_WAITOK);
+ }
+#endif
+
+ s->ctrlq = malloc(sc->params.nports * sizeof(struct sge_wrq), M_CXGBE,
M_ZERO | M_WAITOK);
s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
M_ZERO | M_WAITOK);
@@ -612,15 +623,16 @@ t4_attach(device_t dev)
sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
M_ZERO | M_WAITOK);
- sc->l2t = t4_init_l2t(M_WAITOK);
-
- t4_sysctls(sc);
+ t4_init_l2t(sc, M_WAITOK);
/*
* Second pass over the ports. This time we know the number of rx and
* tx queues that each port should get.
*/
rqidx = tqidx = 0;
+#ifndef TCP_OFFLOAD_DISABLE
+ ofld_rqidx = ofld_tqidx = 0;
+#endif
for_each_port(sc, i) {
struct port_info *pi = sc->port[i];
@@ -628,13 +640,33 @@ t4_attach(device_t dev)
continue;
pi->first_rxq = rqidx;
- pi->nrxq = is_10G_port(pi) ? iaq.nrxq10g : iaq.nrxq1g;
-
pi->first_txq = tqidx;
- pi->ntxq = is_10G_port(pi) ? iaq.ntxq10g : iaq.ntxq1g;
+ if (is_10G_port(pi)) {
+ pi->nrxq = iaq.nrxq10g;
+ pi->ntxq = iaq.ntxq10g;
+ } else {
+ pi->nrxq = iaq.nrxq1g;
+ pi->ntxq = iaq.ntxq1g;
+ }
rqidx += pi->nrxq;
tqidx += pi->ntxq;
+
+#ifndef TCP_OFFLOAD_DISABLE
+ if (is_offload(sc)) {
+ pi->first_ofld_rxq = ofld_rqidx;
+ pi->first_ofld_txq = ofld_tqidx;
+ if (is_10G_port(pi)) {
+ pi->nofldrxq = iaq.nofldrxq10g;
+ pi->nofldtxq = iaq.nofldtxq10g;
+ } else {
+ pi->nofldrxq = iaq.nofldrxq1g;
+ pi->nofldtxq = iaq.nofldtxq1g;
+ }
+ ofld_rqidx += pi->nofldrxq;
+ ofld_tqidx += pi->nofldtxq;
+ }
+#endif
}
rc = bus_generic_attach(dev);
@@ -644,17 +676,27 @@ t4_attach(device_t dev)
goto done;
}
-#ifdef INVARIANTS
device_printf(dev,
- "%p, %d ports (0x%x), %d intr_type, %d intr_count\n",
- sc, sc->params.nports, sc->params.portvec,
- sc->intr_type, sc->intr_count);
-#endif
+ "PCIe x%d, %d ports, %d %s interrupt%s, %d eq, %d iq\n",
+ sc->params.pci.width, sc->params.nports, sc->intr_count,
+ sc->intr_type == INTR_MSIX ? "MSI-X" :
+ (sc->intr_type == INTR_MSI ? "MSI" : "INTx"),
+ sc->intr_count > 1 ? "s" : "", sc->sge.neq, sc->sge.niq);
+
t4_set_desc(sc);
done:
+ if (rc != 0 && sc->cdev) {
+ /* cdev was created and so cxgbetool works; recover that way. */
+ device_printf(dev,
+ "error during attach, adapter is now in recovery mode.\n");
+ rc = 0;
+ }
+
if (rc != 0)
t4_detach(dev);
+ else
+ t4_sysctls(sc);
return (rc);
}
@@ -667,14 +709,25 @@ t4_detach(device_t dev)
{
struct adapter *sc;
struct port_info *pi;
- int i;
+ int i, rc;
sc = device_get_softc(dev);
- if (sc->cdev)
+ if (sc->flags & FULL_INIT_DONE)
+ t4_intr_disable(sc);
+
+ if (sc->cdev) {
destroy_dev(sc->cdev);
+ sc->cdev = NULL;
+ }
+
+ rc = bus_generic_detach(dev);
+ if (rc) {
+ device_printf(dev,
+ "failed to detach child devices: %d\n", rc);
+ return (rc);
+ }
- bus_generic_detach(dev);
for (i = 0; i < MAX_NPORTS; i++) {
pi = sc->port[i];
if (pi) {
@@ -687,6 +740,9 @@ t4_detach(device_t dev)
}
}
+ if (sc->flags & FULL_INIT_DONE)
+ adapter_full_uninit(sc);
+
if (sc->flags & FW_OK)
t4_fw_bye(sc, sc->mbox);
@@ -704,16 +760,27 @@ t4_detach(device_t dev)
if (sc->l2t)
t4_free_l2t(sc->l2t);
+#ifndef TCP_OFFLOAD_DISABLE
+ free(sc->sge.ofld_rxq, M_CXGBE);
+ free(sc->sge.ofld_txq, M_CXGBE);
+#endif
free(sc->irq, M_CXGBE);
free(sc->sge.rxq, M_CXGBE);
free(sc->sge.txq, M_CXGBE);
free(sc->sge.ctrlq, M_CXGBE);
- free(sc->sge.intrq, M_CXGBE);
free(sc->sge.iqmap, M_CXGBE);
free(sc->sge.eqmap, M_CXGBE);
free(sc->tids.ftid_tab, M_CXGBE);
t4_destroy_dma_tag(sc);
- mtx_destroy(&sc->sc_lock);
+ if (mtx_initialized(&sc->sc_lock)) {
+ mtx_lock(&t4_list_lock);
+ SLIST_REMOVE(&t4_list, sc, adapter, link);
+ mtx_unlock(&t4_list_lock);
+ mtx_destroy(&sc->sc_lock);
+ }
+
+ if (mtx_initialized(&sc->sfl_lock))
+ mtx_destroy(&sc->sfl_lock);
bzero(sc, sizeof(*sc));
@@ -727,7 +794,7 @@ cxgbe_probe(device_t dev)
char buf[128];
struct port_info *pi = device_get_softc(dev);
- snprintf(buf, sizeof(buf), "Port %d", pi->port_id);
+ snprintf(buf, sizeof(buf), "port %d", pi->port_id);
device_set_desc_copy(dev, buf);
return (BUS_PROBE_DEFAULT);
@@ -754,15 +821,6 @@ cxgbe_attach(device_t dev)
ifp->if_softc = pi;
callout_init(&pi->tick, CALLOUT_MPSAFE);
- pi->tq = taskqueue_create("cxgbe_taskq", M_NOWAIT,
- taskqueue_thread_enqueue, &pi->tq);
- if (pi->tq == NULL) {
- device_printf(dev, "failed to allocate port task queue\n");
- if_free(pi->ifp);
- return (ENOMEM);
- }
- taskqueue_start_threads(&pi->tq, 1, PI_NET, "%s taskq",
- device_get_nameunit(dev));
if_initname(ifp, device_get_name(dev), device_get_unit(dev));
ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
@@ -778,6 +836,10 @@ cxgbe_attach(device_t dev)
IFQ_SET_READY(&ifp->if_snd);
ifp->if_capabilities = T4_CAP;
+#ifndef TCP_OFFLOAD_DISABLE
+ if (is_offload(pi->adapter))
+ ifp->if_capabilities |= IFCAP_TOE4;
+#endif
ifp->if_capenable = T4_CAP_ENABLE;
ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO;
@@ -788,9 +850,14 @@ cxgbe_attach(device_t dev)
ether_ifattach(ifp, pi->hw_addr);
-#ifdef INVARIANTS
- device_printf(dev, "%p, %d txq, %d rxq\n", pi, pi->ntxq, pi->nrxq);
+#ifndef TCP_OFFLOAD_DISABLE
+ if (is_offload(pi->adapter)) {
+ device_printf(dev,
+ "%d txq, %d rxq (NIC); %d txq, %d rxq (TOE)\n",
+ pi->ntxq, pi->nrxq, pi->nofldtxq, pi->nofldrxq);
+ } else
#endif
+ device_printf(dev, "%d txq, %d rxq\n", pi->ntxq, pi->nrxq);
cxgbe_sysctls(pi);
@@ -802,7 +869,7 @@ cxgbe_detach(device_t dev)
{
struct port_info *pi = device_get_softc(dev);
struct adapter *sc = pi->adapter;
- int rc;
+ struct ifnet *ifp = pi->ifp;
/* Tell if_ioctl and if_init that the port is going away */
ADAPTER_LOCK(sc);
@@ -813,11 +880,15 @@ cxgbe_detach(device_t dev)
SET_BUSY(sc);
ADAPTER_UNLOCK(sc);
- rc = cxgbe_uninit_synchronized(pi);
- if (rc != 0)
- device_printf(dev, "port uninit failed: %d.\n", rc);
+ PORT_LOCK(pi);
+ ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+ callout_stop(&pi->tick);
+ PORT_UNLOCK(pi);
+ callout_drain(&pi->tick);
- taskqueue_free(pi->tq);
+ /* Let detach proceed even if these fail. */
+ cxgbe_uninit_synchronized(pi);
+ port_full_uninit(pi);
ifmedia_removeall(&pi->media);
ether_ifdetach(pi->ifp);
@@ -956,6 +1027,7 @@ fail:
if_printf(ifp,
"enable txcsum first.\n");
rc = EAGAIN;
+ goto fail;
}
} else
ifp->if_hwassist &= ~CSUM_TSO;
@@ -968,15 +1040,21 @@ fail:
ifp->if_capenable ^= IFCAP_LRO;
for_each_rxq(pi, i, rxq) {
if (ifp->if_capenable & IFCAP_LRO)
- rxq->flags |= RXQ_LRO_ENABLED;
+ rxq->iq.flags |= IQ_LRO_ENABLED;
else
- rxq->flags &= ~RXQ_LRO_ENABLED;
+ rxq->iq.flags &= ~IQ_LRO_ENABLED;
}
#endif
}
#ifndef TCP_OFFLOAD_DISABLE
- if (mask & IFCAP_TOE4) {
- rc = EOPNOTSUPP;
+ if (mask & IFCAP_TOE) {
+ int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE;
+
+ rc = toe_capability(pi, enable);
+ if (rc != 0)
+ goto fail;
+
+ ifp->if_capenable ^= mask;
}
#endif
if (mask & IFCAP_VLAN_HWTAGGING) {
@@ -1041,9 +1119,9 @@ cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
M_ASSERTPKTHDR(m);
- if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
+ if (__predict_false(pi->link_cfg.link_ok == 0)) {
m_freem(m);
- return (0);
+ return (ENETDOWN);
}
if (m->m_flags & M_FLOWID)
@@ -1051,13 +1129,20 @@ cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
br = txq->br;
if (TXQ_TRYLOCK(txq) == 0) {
+ struct sge_eq *eq = &txq->eq;
+
/*
- * XXX: make sure that this packet really is sent out. There is
- * a small race where t4_eth_tx may stop draining the drbr and
- * goes away, just before we enqueued this mbuf.
+ * It is possible that t4_eth_tx finishes up and releases the
+ * lock between the TRYLOCK above and the drbr_enqueue here. We
+ * need to make sure that this mbuf doesn't just sit there in
+ * the drbr.
*/
- return (drbr_enqueue(ifp, br, m));
+ rc = drbr_enqueue(ifp, br, m);
+ if (rc == 0 && callout_pending(&eq->tx_callout) == 0 &&
+ !(eq->flags & EQ_DOOMED))
+ callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
+ return (rc);
}
/*
@@ -1098,11 +1183,12 @@ cxgbe_qflush(struct ifnet *ifp)
int i;
struct mbuf *m;
- /* queues do not exist if !IFF_DRV_RUNNING. */
- if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+ /* queues do not exist if !PORT_INIT_DONE. */
+ if (pi->flags & PORT_INIT_DONE) {
for_each_txq(pi, i, txq) {
TXQ_LOCK(txq);
m_freem(txq->m);
+ txq->m = NULL;
while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
m_freem(m);
TXQ_UNLOCK(txq);
@@ -1216,14 +1302,25 @@ static int
cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
struct intrs_and_queues *iaq)
{
- int rc, itype, navail, nc, nrxq10g, nrxq1g;
+ int rc, itype, navail, nrxq10g, nrxq1g, n;
+ int nofldrxq10g = 0, nofldrxq1g = 0;
bzero(iaq, sizeof(*iaq));
- nc = mp_ncpus; /* our snapshot of the number of CPUs */
+
+ iaq->ntxq10g = t4_ntxq10g;
+ iaq->ntxq1g = t4_ntxq1g;
+ iaq->nrxq10g = nrxq10g = t4_nrxq10g;
+ iaq->nrxq1g = nrxq1g = t4_nrxq1g;
+#ifndef TCP_OFFLOAD_DISABLE
+ iaq->nofldtxq10g = t4_nofldtxq10g;
+ iaq->nofldtxq1g = t4_nofldtxq1g;
+ iaq->nofldrxq10g = nofldrxq10g = t4_nofldrxq10g;
+ iaq->nofldrxq1g = nofldrxq1g = t4_nofldrxq1g;
+#endif
for (itype = INTR_MSIX; itype; itype >>= 1) {
- if ((itype & intr_types) == 0)
+ if ((itype & t4_intr_types) == 0)
continue; /* not allowed */
if (itype == INTR_MSIX)
@@ -1232,60 +1329,93 @@ cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g,
navail = pci_msi_count(sc->dev);
else
navail = 1;
-
+restart:
if (navail == 0)
continue;
iaq->intr_type = itype;
+ iaq->intr_flags = 0;
- iaq->ntxq10g = min(nc, max_ntxq_10g);
- iaq->ntxq1g = min(nc, max_ntxq_1g);
-
- nrxq10g = min(nc, max_nrxq_10g);
- nrxq1g = min(nc, max_nrxq_1g);
-
- iaq->nirq = n10g * nrxq10g + n1g * nrxq1g + T4_EXTRA_INTR;
- if (iaq->nirq <= navail && intr_shared == 0) {
-
- if (itype == INTR_MSI && !powerof2(iaq->nirq))
- goto share;
-
- /* One for err, one for fwq, and one for each rxq */
-
- iaq->intr_shared = 0;
- iaq->nrxq10g = nrxq10g;
- iaq->nrxq1g = nrxq1g;
+ /*
+ * Best option: an interrupt vector for errors, one for the
+ * firmware event queue, and one each for each rxq (NIC as well
+ * as offload).
+ */
+ iaq->nirq = T4_EXTRA_INTR;
+ iaq->nirq += n10g * (nrxq10g + nofldrxq10g);
+ iaq->nirq += n1g * (nrxq1g + nofldrxq1g);
+ if (iaq->nirq <= navail &&
+ (itype != INTR_MSI || powerof2(iaq->nirq))) {
+ iaq->intr_flags |= INTR_DIRECT;
+ goto allocate;
+ }
- } else {
-share:
- iaq->intr_shared = 1;
+ /*
+ * Second best option: an interrupt vector for errors, one for
+ * the firmware event queue, and one each for either NIC or
+ * offload rxq's.
+ */
+ iaq->nirq = T4_EXTRA_INTR;
+ iaq->nirq += n10g * max(nrxq10g, nofldrxq10g);
+ iaq->nirq += n1g * max(nrxq1g, nofldrxq1g);
+ if (iaq->nirq <= navail &&
+ (itype != INTR_MSI || powerof2(iaq->nirq)))
+ goto allocate;
- if (navail >= nc + T4_EXTRA_INTR) {
- if (itype == INTR_MSIX)
- navail = nc + T4_EXTRA_INTR;
+ /*
+ * Next best option: an interrupt vector for errors, one for the
+ * firmware event queue, and at least one per port. At this
+ * point we know we'll have to downsize nrxq or nofldrxq to fit
+ * what's available to us.
+ */
+ iaq->nirq = T4_EXTRA_INTR;
+ iaq->nirq += n10g + n1g;
+ if (iaq->nirq <= navail) {
+ int leftover = navail - iaq->nirq;
+
+ if (n10g > 0) {
+ int target = max(nrxq10g, nofldrxq10g);
+
+ n = 1;
+ while (n < target && leftover >= n10g) {
+ leftover -= n10g;
+ iaq->nirq += n10g;
+ n++;
+ }
+ iaq->nrxq10g = min(n, nrxq10g);
+#ifndef TCP_OFFLOAD_DISABLE
+ iaq->nofldrxq10g = min(n, nofldrxq10g);
+#endif
+ }
- /* navail is and must remain a pow2 for MSI */
- if (itype == INTR_MSI) {
- KASSERT(powerof2(navail),
- ("%d not power of 2", navail));
+ if (n1g > 0) {
+ int target = max(nrxq1g, nofldrxq1g);
- while (navail / 2 >= nc + T4_EXTRA_INTR)
- navail /= 2;
+ n = 1;
+ while (n < target && leftover >= n1g) {
+ leftover -= n1g;
+ iaq->nirq += n1g;
+ n++;
}
+ iaq->nrxq1g = min(n, nrxq1g);
+#ifndef TCP_OFFLOAD_DISABLE
+ iaq->nofldrxq1g = min(n, nofldrxq1g);
+#endif
}
- iaq->nirq = navail; /* total # of interrupts */
- /*
- * If we have multiple vectors available reserve one
- * exclusively for errors. The rest will be shared by
- * the fwq and data.
- */
- if (navail > 1)
- navail--;
- iaq->nrxq10g = min(nrxq10g, navail);
- iaq->nrxq1g = min(nrxq1g, navail);
+ if (itype != INTR_MSI || powerof2(iaq->nirq))
+ goto allocate;
}
+ /*
+ * Least desirable option: one interrupt vector for everything.
+ */
+ iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1;
+#ifndef TCP_OFFLOAD_DISABLE
+ iaq->nofldrxq10g = iaq->nofldrxq1g = 1;
+#endif
+
+allocate:
navail = iaq->nirq;
rc = 0;
if (itype == INTR_MSIX)
@@ -1301,8 +1431,11 @@ share:
* Didn't get the number requested. Use whatever number
* the kernel is willing to allocate (it's in navail).
*/
+ device_printf(sc->dev, "fewer vectors than requested, "
+ "type=%d, req=%d, rcvd=%d; will downshift req.\n",
+ itype, iaq->nirq, navail);
pci_release_msi(sc->dev);
- goto share;
+ goto restart;
}
device_printf(sc->dev,
@@ -1312,26 +1445,30 @@ share:
device_printf(sc->dev,
"failed to find a usable interrupt type. "
- "allowed=%d, msi-x=%d, msi=%d, intx=1", intr_types,
+ "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
pci_msix_count(sc->dev), pci_msi_count(sc->dev));
return (ENXIO);
}
/*
- * Install a compatible firmware (if required), establish contact with it,
- * become the master, and reset the device.
+ * Install a compatible firmware (if required), establish contact with it (by
+ * saying hello), and reset the device. If we end up as the master driver,
+ * partition adapter resources by providing a configuration file to the
+ * firmware.
*/
static int
prep_firmware(struct adapter *sc)
{
- const struct firmware *fw;
+ const struct firmware *fw = NULL, *cfg = NULL, *default_cfg;
int rc;
enum dev_state state;
+ default_cfg = firmware_get(T4_CFGNAME);
+
/* Check firmware version and install a different one if necessary */
rc = t4_check_fw_version(sc);
- if (rc != 0 || force_firmware_install) {
+ if (rc != 0) {
uint32_t v = 0;
fw = firmware_get(T4_FWNAME);
@@ -1343,7 +1480,7 @@ prep_firmware(struct adapter *sc)
/*
* The firmware module will not be used if it isn't the
* same major version as what the driver was compiled
- * with. This check trumps force_firmware_install.
+ * with.
*/
if (G_FW_HDR_FW_VER_MAJOR(v) != FW_VERSION_MAJOR) {
device_printf(sc->dev,
@@ -1356,17 +1493,16 @@ prep_firmware(struct adapter *sc)
}
}
- if (fw == NULL && (rc < 0 || force_firmware_install)) {
+ if (fw == NULL && rc < 0) {
device_printf(sc->dev, "No usable firmware. "
- "card has %d.%d.%d, driver compiled with %d.%d.%d, "
- "force_firmware_install%s set",
+ "card has %d.%d.%d, driver compiled with %d.%d.%d",
G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
FW_VERSION_MAJOR, FW_VERSION_MINOR,
- FW_VERSION_MICRO,
- force_firmware_install ? "" : " not");
- return (EAGAIN);
+ FW_VERSION_MICRO);
+ rc = EAGAIN;
+ goto done;
}
/*
@@ -1374,8 +1510,7 @@ prep_firmware(struct adapter *sc)
* Downgrade only for a major version mismatch or if
* force_firmware_install was specified.
*/
- if (fw != NULL && (rc < 0 || force_firmware_install ||
- v > sc->params.fw_vers)) {
+ if (fw != NULL && (rc < 0 || v > sc->params.fw_vers)) {
device_printf(sc->dev,
"installing firmware %d.%d.%d.%d on card.\n",
G_FW_HDR_FW_VER_MAJOR(v), G_FW_HDR_FW_VER_MINOR(v),
@@ -1385,26 +1520,24 @@ prep_firmware(struct adapter *sc)
if (rc != 0) {
device_printf(sc->dev,
"failed to install firmware: %d\n", rc);
- firmware_put(fw, FIRMWARE_UNLOAD);
- return (rc);
+ goto done;
} else {
/* refresh */
(void) t4_check_fw_version(sc);
}
}
-
- if (fw != NULL)
- firmware_put(fw, FIRMWARE_UNLOAD);
}
- /* Contact firmware, request master */
- rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MUST, &state);
+ /* Contact firmware. */
+ rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MAY, &state);
if (rc < 0) {
rc = -rc;
device_printf(sc->dev,
"failed to connect to the firmware: %d.\n", rc);
- return (rc);
+ goto done;
}
+ if (rc == sc->mbox)
+ sc->flags |= MASTER_PF;
/* Reset device */
rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST);
@@ -1412,7 +1545,26 @@ prep_firmware(struct adapter *sc)
device_printf(sc->dev, "firmware reset failed: %d.\n", rc);
if (rc != ETIMEDOUT && rc != EIO)
t4_fw_bye(sc, sc->mbox);
- return (rc);
+ goto done;
+ }
+
+ /* Partition adapter resources as specified in the config file. */
+ if (sc->flags & MASTER_PF) {
+ if (strncmp(t4_cfg_file, "default", sizeof(t4_cfg_file))) {
+ char s[32];
+
+ snprintf(s, sizeof(s), "t4fw_cfg_%s", t4_cfg_file);
+ cfg = firmware_get(s);
+ if (cfg == NULL) {
+ device_printf(sc->dev,
+ "unable to locate %s module, "
+ "will use default config file.\n", s);
+ }
+ }
+
+ rc = partition_resources(sc, cfg ? cfg : default_cfg);
+ if (rc != 0)
+ goto done; /* error message displayed already */
}
snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
@@ -1422,84 +1574,207 @@ prep_firmware(struct adapter *sc)
G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
sc->flags |= FW_OK;
- return (0);
+done:
+ if (fw != NULL)
+ firmware_put(fw, FIRMWARE_UNLOAD);
+ if (cfg != NULL)
+ firmware_put(cfg, FIRMWARE_UNLOAD);
+ if (default_cfg != NULL)
+ firmware_put(default_cfg, FIRMWARE_UNLOAD);
+
+ return (rc);
}
+#define FW_PARAM_DEV(param) \
+ (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
+ V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
+#define FW_PARAM_PFVF(param) \
+ (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
+ V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
+
+/*
+ * Upload configuration file to card's memory.
+ */
static int
-get_devlog_params(struct adapter *sc, struct devlog_params *dlog)
+upload_config_file(struct adapter *sc, const struct firmware *fw, uint32_t *mt,
+ uint32_t *ma)
{
- struct fw_devlog_cmd devlog_cmd;
- uint32_t meminfo;
- int rc;
+ int rc, i;
+ uint32_t param, val, mtype, maddr, bar, off, win, remaining;
+ const uint32_t *b;
- bzero(&devlog_cmd, sizeof(devlog_cmd));
- devlog_cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
- F_FW_CMD_REQUEST | F_FW_CMD_READ);
- devlog_cmd.retval_len16 = htobe32(FW_LEN16(devlog_cmd));
- rc = -t4_wr_mbox(sc, sc->mbox, &devlog_cmd, sizeof(devlog_cmd),
- &devlog_cmd);
+ /* Figure out where the firmware wants us to upload it. */
+ param = FW_PARAM_DEV(CF);
+ rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, &param, &val);
if (rc != 0) {
+ /* Firmwares without config file support will fail this way */
device_printf(sc->dev,
- "failed to get devlog parameters: %d.\n", rc);
- bzero(dlog, sizeof (*dlog));
+ "failed to query config file location: %d.\n", rc);
return (rc);
}
+ *mt = mtype = G_FW_PARAMS_PARAM_Y(val);
+ *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16;
+
+ if (maddr & 3) {
+ device_printf(sc->dev,
+ "cannot upload config file (type %u, addr %x).\n",
+ mtype, maddr);
+ return (EFAULT);
+ }
- meminfo = be32toh(devlog_cmd.memtype_devlog_memaddr16_devlog);
- dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(meminfo);
- dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(meminfo) << 4;
- dlog->size = be32toh(devlog_cmd.memsize_devlog);
+ /* Translate mtype/maddr to an address suitable for the PCIe window */
+ val = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
+ val &= F_EDRAM0_ENABLE | F_EDRAM1_ENABLE | F_EXT_MEM_ENABLE;
+ switch (mtype) {
+ case FW_MEMTYPE_CF_EDC0:
+ if (!(val & F_EDRAM0_ENABLE))
+ goto err;
+ bar = t4_read_reg(sc, A_MA_EDRAM0_BAR);
+ maddr += G_EDRAM0_BASE(bar) << 20;
+ break;
- return (0);
+ case FW_MEMTYPE_CF_EDC1:
+ if (!(val & F_EDRAM1_ENABLE))
+ goto err;
+ bar = t4_read_reg(sc, A_MA_EDRAM1_BAR);
+ maddr += G_EDRAM1_BASE(bar) << 20;
+ break;
+
+ case FW_MEMTYPE_CF_EXTMEM:
+ if (!(val & F_EXT_MEM_ENABLE))
+ goto err;
+ bar = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
+ maddr += G_EXT_MEM_BASE(bar) << 20;
+ break;
+
+ default:
+err:
+ device_printf(sc->dev,
+ "cannot upload config file (type %u, enabled %u).\n",
+ mtype, val);
+ return (EFAULT);
+ }
+
+ /*
+ * Position the PCIe window (we use memwin2) to the 16B aligned area
+ * just at/before the upload location.
+ */
+ win = maddr & ~0xf;
+ off = maddr - win; /* offset from the start of the window. */
+ t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
+ t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
+
+ remaining = fw->datasize;
+ if (remaining > FLASH_CFG_MAX_SIZE ||
+ remaining > MEMWIN2_APERTURE - off) {
+ device_printf(sc->dev, "cannot upload config file all at once "
+ "(size %u, max %u, room %u).\n",
+ remaining, FLASH_CFG_MAX_SIZE, MEMWIN2_APERTURE - off);
+ return (EFBIG);
+ }
+
+ /*
+ * XXX: sheer laziness. We deliberately added 4 bytes of useless
+ * stuffing/comments at the end of the config file so it's ok to simply
+ * throw away the last remaining bytes when the config file is not an
+ * exact multiple of 4.
+ */
+ b = fw->data;
+ for (i = 0; remaining >= 4; i += 4, remaining -= 4)
+ t4_write_reg(sc, MEMWIN2_BASE + off + i, *b++);
+
+ return (rc);
}
+/*
+ * Partition chip resources for use between various PFs, VFs, etc. This is done
+ * by uploading the firmware configuration file to the adapter and instructing
+ * the firmware to process it.
+ */
static int
-get_capabilities(struct adapter *sc, struct fw_caps_config_cmd *caps)
+partition_resources(struct adapter *sc, const struct firmware *cfg)
{
int rc;
+ struct fw_caps_config_cmd caps;
+ uint32_t mtype, maddr, finicsum, cfcsum;
- bzero(caps, sizeof(*caps));
- caps->op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
- F_FW_CMD_REQUEST | F_FW_CMD_READ);
- caps->retval_len16 = htobe32(FW_LEN16(*caps));
+ rc = cfg ? upload_config_file(sc, cfg, &mtype, &maddr) : ENOENT;
+ if (rc != 0) {
+ mtype = FW_MEMTYPE_CF_FLASH;
+ maddr = t4_flash_cfg_addr(sc);
+ }
- rc = -t4_wr_mbox(sc, sc->mbox, caps, sizeof(*caps), caps);
- if (rc != 0)
+ bzero(&caps, sizeof(caps));
+ caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
+ F_FW_CMD_REQUEST | F_FW_CMD_READ);
+ caps.cfvalid_to_len16 = htobe32(F_FW_CAPS_CONFIG_CMD_CFVALID |
+ V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) |
+ V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps));
+ rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to pre-process config file: %d.\n", rc);
return (rc);
+ }
- if (caps->niccaps & htobe16(FW_CAPS_CONFIG_NIC_VM))
- caps->niccaps ^= htobe16(FW_CAPS_CONFIG_NIC_VM);
+ finicsum = be32toh(caps.finicsum);
+ cfcsum = be32toh(caps.cfcsum);
+ if (finicsum != cfcsum) {
+ device_printf(sc->dev,
+ "WARNING: config file checksum mismatch: %08x %08x\n",
+ finicsum, cfcsum);
+ }
+ sc->cfcsum = cfcsum;
+
+#define LIMIT_CAPS(x) do { \
+ caps.x &= htobe16(t4_##x##_allowed); \
+ sc->x = htobe16(caps.x); \
+} while (0)
- caps->op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
+ /*
+ * Let the firmware know what features will (not) be used so it can tune
+ * things accordingly.
+ */
+ LIMIT_CAPS(linkcaps);
+ LIMIT_CAPS(niccaps);
+ LIMIT_CAPS(toecaps);
+ LIMIT_CAPS(rdmacaps);
+ LIMIT_CAPS(iscsicaps);
+ LIMIT_CAPS(fcoecaps);
+#undef LIMIT_CAPS
+
+ caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
F_FW_CMD_REQUEST | F_FW_CMD_WRITE);
- rc = -t4_wr_mbox(sc, sc->mbox, caps, sizeof(*caps), NULL);
+ caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
+ rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), NULL);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to process config file: %d.\n", rc);
+ return (rc);
+ }
- return (rc);
+ return (0);
}
+/*
+ * Retrieve parameters that are needed (or nice to have) prior to calling
+ * t4_sge_init and t4_fw_initialize.
+ */
static int
-get_params(struct adapter *sc, struct fw_caps_config_cmd *caps)
+get_params__pre_init(struct adapter *sc)
{
int rc;
- uint32_t params[7], val[7];
+ uint32_t param[2], val[2];
+ struct fw_devlog_cmd cmd;
+ struct devlog_params *dlog = &sc->params.devlog;
-#define FW_PARAM_DEV(param) \
- (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
- V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
-#define FW_PARAM_PFVF(param) \
- (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
- V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
-
- params[0] = FW_PARAM_DEV(PORTVEC);
- params[1] = FW_PARAM_PFVF(IQFLINT_START);
- params[2] = FW_PARAM_PFVF(EQ_START);
- params[3] = FW_PARAM_PFVF(FILTER_START);
- params[4] = FW_PARAM_PFVF(FILTER_END);
- rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 5, params, val);
+ param[0] = FW_PARAM_DEV(PORTVEC);
+ param[1] = FW_PARAM_DEV(CCLK);
+ rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
if (rc != 0) {
device_printf(sc->dev,
- "failed to query parameters: %d.\n", rc);
- goto done;
+ "failed to query parameters (pre_init): %d.\n", rc);
+ return (rc);
}
sc->params.portvec = val[0];
@@ -1509,24 +1784,81 @@ get_params(struct adapter *sc, struct fw_caps_config_cmd *caps)
val[0] &= val[0] - 1;
}
- sc->sge.iq_start = val[1];
- sc->sge.eq_start = val[2];
- sc->tids.ftid_base = val[3];
- sc->tids.nftids = val[4] - val[3] + 1;
+ sc->params.vpd.cclk = val[1];
+
+ /* Read device log parameters. */
+ bzero(&cmd, sizeof(cmd));
+ cmd.op_to_write = htobe32(V_FW_CMD_OP(FW_DEVLOG_CMD) |
+ F_FW_CMD_REQUEST | F_FW_CMD_READ);
+ cmd.retval_len16 = htobe32(FW_LEN16(cmd));
+ rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof(cmd), &cmd);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to get devlog parameters: %d.\n", rc);
+ bzero(dlog, sizeof (*dlog));
+ rc = 0; /* devlog isn't critical for device operation */
+ } else {
+ val[0] = be32toh(cmd.memtype_devlog_memaddr16_devlog);
+ dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]);
+ dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4;
+ dlog->size = be32toh(cmd.memsize_devlog);
+ }
- if (caps->toecaps) {
+ return (rc);
+}
+
+/*
+ * Retrieve various parameters that are of interest to the driver. The device
+ * has been initialized by the firmware at this point.
+ */
+static int
+get_params__post_init(struct adapter *sc)
+{
+ int rc;
+ uint32_t param[7], val[7];
+ struct fw_caps_config_cmd caps;
+
+ param[0] = FW_PARAM_PFVF(IQFLINT_START);
+ param[1] = FW_PARAM_PFVF(EQ_START);
+ param[2] = FW_PARAM_PFVF(FILTER_START);
+ param[3] = FW_PARAM_PFVF(FILTER_END);
+ rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 4, param, val);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to query parameters (post_init): %d.\n", rc);
+ return (rc);
+ }
+
+ sc->sge.iq_start = val[0];
+ sc->sge.eq_start = val[1];
+ sc->tids.ftid_base = val[2];
+ sc->tids.nftids = val[3] - val[2] + 1;
+
+ /* get capabilites */
+ bzero(&caps, sizeof(caps));
+ caps.op_to_write = htobe32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
+ F_FW_CMD_REQUEST | F_FW_CMD_READ);
+ caps.cfvalid_to_len16 = htobe32(FW_LEN16(caps));
+ rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof(caps), &caps);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to get card capabilities: %d.\n", rc);
+ return (rc);
+ }
+
+ if (caps.toecaps) {
/* query offload-related parameters */
- params[0] = FW_PARAM_DEV(NTID);
- params[1] = FW_PARAM_PFVF(SERVER_START);
- params[2] = FW_PARAM_PFVF(SERVER_END);
- params[3] = FW_PARAM_PFVF(TDDP_START);
- params[4] = FW_PARAM_PFVF(TDDP_END);
- params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
- rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, params, val);
+ param[0] = FW_PARAM_DEV(NTID);
+ param[1] = FW_PARAM_PFVF(SERVER_START);
+ param[2] = FW_PARAM_PFVF(SERVER_END);
+ param[3] = FW_PARAM_PFVF(TDDP_START);
+ param[4] = FW_PARAM_PFVF(TDDP_END);
+ param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
+ rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query TOE parameters: %d.\n", rc);
- goto done;
+ return (rc);
}
sc->tids.ntids = val[0];
sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
@@ -1537,18 +1869,18 @@ get_params(struct adapter *sc, struct fw_caps_config_cmd *caps)
sc->params.ofldq_wr_cred = val[5];
sc->params.offload = 1;
}
- if (caps->rdmacaps) {
- params[0] = FW_PARAM_PFVF(STAG_START);
- params[1] = FW_PARAM_PFVF(STAG_END);
- params[2] = FW_PARAM_PFVF(RQ_START);
- params[3] = FW_PARAM_PFVF(RQ_END);
- params[4] = FW_PARAM_PFVF(PBL_START);
- params[5] = FW_PARAM_PFVF(PBL_END);
- rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, params, val);
+ if (caps.rdmacaps) {
+ param[0] = FW_PARAM_PFVF(STAG_START);
+ param[1] = FW_PARAM_PFVF(STAG_END);
+ param[2] = FW_PARAM_PFVF(RQ_START);
+ param[3] = FW_PARAM_PFVF(RQ_END);
+ param[4] = FW_PARAM_PFVF(PBL_START);
+ param[5] = FW_PARAM_PFVF(PBL_END);
+ rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val);
if (rc != 0) {
device_printf(sc->dev,
- "failed to query RDMA parameters: %d.\n", rc);
- goto done;
+ "failed to query RDMA parameters(1): %d.\n", rc);
+ return (rc);
}
sc->vres.stag.start = val[0];
sc->vres.stag.size = val[1] - val[0] + 1;
@@ -1556,37 +1888,59 @@ get_params(struct adapter *sc, struct fw_caps_config_cmd *caps)
sc->vres.rq.size = val[3] - val[2] + 1;
sc->vres.pbl.start = val[4];
sc->vres.pbl.size = val[5] - val[4] + 1;
+
+ param[0] = FW_PARAM_PFVF(SQRQ_START);
+ param[1] = FW_PARAM_PFVF(SQRQ_END);
+ param[2] = FW_PARAM_PFVF(CQ_START);
+ param[3] = FW_PARAM_PFVF(CQ_END);
+ param[4] = FW_PARAM_PFVF(OCQ_START);
+ param[5] = FW_PARAM_PFVF(OCQ_END);
+ rc = -t4_query_params(sc, 0, 0, 0, 6, param, val);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to query RDMA parameters(2): %d.\n", rc);
+ return (rc);
+ }
+ sc->vres.qp.start = val[0];
+ sc->vres.qp.size = val[1] - val[0] + 1;
+ sc->vres.cq.start = val[2];
+ sc->vres.cq.size = val[3] - val[2] + 1;
+ sc->vres.ocq.start = val[4];
+ sc->vres.ocq.size = val[5] - val[4] + 1;
}
- if (caps->iscsicaps) {
- params[0] = FW_PARAM_PFVF(ISCSI_START);
- params[1] = FW_PARAM_PFVF(ISCSI_END);
- rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, params, val);
+ if (caps.iscsicaps) {
+ param[0] = FW_PARAM_PFVF(ISCSI_START);
+ param[1] = FW_PARAM_PFVF(ISCSI_END);
+ rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
if (rc != 0) {
device_printf(sc->dev,
"failed to query iSCSI parameters: %d.\n", rc);
- goto done;
+ return (rc);
}
sc->vres.iscsi.start = val[0];
sc->vres.iscsi.size = val[1] - val[0] + 1;
}
-#undef FW_PARAM_PFVF
-#undef FW_PARAM_DEV
-done:
+ /* These are finalized by FW initialization, load their values now */
+ val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION);
+ sc->params.tp.tre = G_TIMERRESOLUTION(val[0]);
+ sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]);
+ t4_read_mtu_tbl(sc, sc->params.mtus, NULL);
+
return (rc);
}
+#undef FW_PARAM_PFVF
+#undef FW_PARAM_DEV
+
static void
t4_set_desc(struct adapter *sc)
{
char buf[128];
struct adapter_params *p = &sc->params;
- snprintf(buf, sizeof(buf),
- "Chelsio %s (rev %d) %d port %sNIC PCIe-x%d %d %s, S/N:%s, E/C:%s",
- p->vpd.id, p->rev, p->nports, is_offload(sc) ? "R" : "",
- p->pci.width, sc->intr_count, sc->intr_type == INTR_MSIX ? "MSI-X" :
- (sc->intr_type == INTR_MSI ? "MSI" : "INTx"), p->vpd.sn, p->vpd.ec);
+ snprintf(buf, sizeof(buf), "Chelsio %s %sNIC (rev %d), S/N:%s, E/C:%s",
+ p->vpd.id, is_offload(sc) ? "R" : "", p->rev, p->vpd.sn, p->vpd.ec);
device_set_desc_copy(sc->dev, buf);
}
@@ -1803,9 +2157,7 @@ cxgbe_init_synchronized(struct port_info *pi)
{
struct adapter *sc = pi->adapter;
struct ifnet *ifp = pi->ifp;
- int rc = 0, i;
- uint16_t *rss;
- struct sge_rxq *rxq;
+ int rc = 0;
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
@@ -1815,30 +2167,13 @@ cxgbe_init_synchronized(struct port_info *pi)
return (0); /* already running */
}
- if (sc->open_device_map == 0 && ((rc = first_port_up(sc)) != 0))
+ if (!(sc->flags & FULL_INIT_DONE) &&
+ ((rc = adapter_full_init(sc)) != 0))
return (rc); /* error message displayed already */
- /*
- * Allocate tx/rx/fl queues for this port.
- */
- rc = t4_setup_eth_queues(pi);
- if (rc != 0)
- goto done; /* error message displayed already */
-
- /*
- * Setup RSS for this port.
- */
- rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
- for_each_rxq(pi, i, rxq) {
- rss[i] = rxq->iq.abs_id;
- }
- rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, pi->rss_size, rss,
- pi->nrxq);
- free(rss, M_CXGBE);
- if (rc != 0) {
- if_printf(ifp, "rss_config failed: %d\n", rc);
- goto done;
- }
+ if (!(pi->flags & PORT_INIT_DONE) &&
+ ((rc = port_full_init(pi)) != 0))
+ return (rc); /* error message displayed already */
PORT_LOCK(pi);
rc = update_mac_settings(pi, XGMAC_ALL);
@@ -1857,12 +2192,10 @@ cxgbe_init_synchronized(struct port_info *pi)
if_printf(ifp, "enable_vi failed: %d\n", rc);
goto done;
}
- pi->flags |= VI_ENABLED;
/* all ok */
setbit(&sc->open_device_map, pi->port_id);
ifp->if_drv_flags |= IFF_DRV_RUNNING;
- ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
callout_reset(&pi->tick, hz, cxgbe_tick, pi);
done:
@@ -1915,51 +2248,28 @@ cxgbe_uninit_synchronized(struct port_info *pi)
struct ifnet *ifp = pi->ifp;
int rc;
- /*
- * taskqueue_drain may cause a deadlock if the adapter lock is held.
- */
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
/*
- * Clear this port's bit from the open device map, and then drain
- * tasks and callouts.
+ * Disable the VI so that all its data in either direction is discarded
+ * by the MPS. Leave everything else (the queues, interrupts, and 1Hz
+ * tick) intact as the TP can deliver negative advice or data that it's
+ * holding in its RAM (for an offloaded connection) even after the VI is
+ * disabled.
*/
- clrbit(&sc->open_device_map, pi->port_id);
-
- PORT_LOCK(pi);
- ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
- callout_stop(&pi->tick);
- PORT_UNLOCK(pi);
- callout_drain(&pi->tick);
-
- /*
- * Stop and then free the queues' resources, including the queues
- * themselves.
- *
- * XXX: we could just stop the queues here (on ifconfig down) and free
- * them later (on port detach), but having up/down go through the entire
- * allocate/activate/deactivate/free sequence is a good way to find
- * leaks and bugs.
- */
- rc = t4_teardown_eth_queues(pi);
- if (rc != 0)
- if_printf(ifp, "teardown failed: %d\n", rc);
-
- if (pi->flags & VI_ENABLED) {
- rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
- if (rc)
- if_printf(ifp, "disable_vi failed: %d\n", rc);
- else
- pi->flags &= ~VI_ENABLED;
+ rc = -t4_enable_vi(sc, sc->mbox, pi->viid, false, false);
+ if (rc) {
+ if_printf(ifp, "disable_vi failed: %d\n", rc);
+ return (rc);
}
+ clrbit(&sc->open_device_map, pi->port_id);
+ ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
+
pi->link_cfg.link_ok = 0;
pi->link_cfg.speed = 0;
t4_os_link_changed(sc, pi->port_id, 0);
- if (sc->open_device_map == 0)
- last_port_down(sc);
-
return (0);
}
@@ -1968,15 +2278,22 @@ cxgbe_uninit_synchronized(struct port_info *pi)
if (rc != 0) \
goto done; \
} while (0)
+
static int
-first_port_up(struct adapter *sc)
+adapter_full_init(struct adapter *sc)
{
int rc, i, rid, p, q;
char s[8];
struct irq *irq;
- struct sge_iq *intrq;
+ struct port_info *pi;
+ struct sge_rxq *rxq;
+#ifndef TCP_OFFLOAD_DISABLE
+ struct sge_ofld_rxq *ofld_rxq;
+#endif
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
+ KASSERT((sc->flags & FULL_INIT_DONE) == 0,
+ ("%s: FULL_INIT_DONE already", __func__));
/*
* queues that belong to the adapter (not any particular port).
@@ -1985,95 +2302,265 @@ first_port_up(struct adapter *sc)
if (rc != 0)
goto done;
+ for (i = 0; i < ARRAY_SIZE(sc->tq); i++) {
+ sc->tq[i] = taskqueue_create("t4 taskq", M_NOWAIT,
+ taskqueue_thread_enqueue, &sc->tq[i]);
+ if (sc->tq[i] == NULL) {
+ device_printf(sc->dev,
+ "failed to allocate task queue %d\n", i);
+ rc = ENOMEM;
+ goto done;
+ }
+ taskqueue_start_threads(&sc->tq[i], 1, PI_NET, "%s tq%d",
+ device_get_nameunit(sc->dev), i);
+ }
+
/*
* Setup interrupts.
*/
irq = &sc->irq[0];
rid = sc->intr_type == INTR_INTX ? 0 : 1;
if (sc->intr_count == 1) {
- KASSERT(sc->flags & INTR_SHARED,
- ("%s: single interrupt but not shared?", __func__));
+ KASSERT(!(sc->flags & INTR_DIRECT),
+ ("%s: single interrupt && INTR_DIRECT?", __func__));
T4_ALLOC_IRQ(sc, irq, rid, t4_intr_all, sc, "all");
} else {
- /* Multiple interrupts. The first one is always error intr */
+ /* Multiple interrupts. */
+ KASSERT(sc->intr_count >= T4_EXTRA_INTR + sc->params.nports,
+ ("%s: too few intr.", __func__));
+
+ /* The first one is always error intr */
T4_ALLOC_IRQ(sc, irq, rid, t4_intr_err, sc, "err");
irq++;
rid++;
- /* Firmware event queue normally has an interrupt of its own */
- if (sc->intr_count > T4_EXTRA_INTR) {
- T4_ALLOC_IRQ(sc, irq, rid, t4_intr_evt, &sc->sge.fwq,
- "evt");
- irq++;
- rid++;
- }
-
- intrq = &sc->sge.intrq[0];
- if (sc->flags & INTR_SHARED) {
+ /* The second one is always the firmware event queue */
+ T4_ALLOC_IRQ(sc, irq, rid, t4_intr_evt, &sc->sge.fwq, "evt");
+ irq++;
+ rid++;
- /* All ports share these interrupt queues */
+ /*
+ * Note that if INTR_DIRECT is not set then either the NIC rx
+ * queues or (exclusive or) the TOE rx queueus will be taking
+ * direct interrupts.
+ *
+ * There is no need to check for is_offload(sc) as nofldrxq
+ * will be 0 if offload is disabled.
+ */
+ for_each_port(sc, p) {
+ pi = sc->port[p];
- for (i = 0; i < NINTRQ(sc); i++) {
- snprintf(s, sizeof(s), "*.%d", i);
- T4_ALLOC_IRQ(sc, irq, rid, t4_intr, intrq, s);
+#ifndef TCP_OFFLOAD_DISABLE
+ /*
+ * Skip over the NIC queues if they aren't taking direct
+ * interrupts.
+ */
+ if (!(sc->flags & INTR_DIRECT) &&
+ pi->nofldrxq > pi->nrxq)
+ goto ofld_queues;
+#endif
+ rxq = &sc->sge.rxq[pi->first_rxq];
+ for (q = 0; q < pi->nrxq; q++, rxq++) {
+ snprintf(s, sizeof(s), "%d.%d", p, q);
+ T4_ALLOC_IRQ(sc, irq, rid, t4_intr, rxq, s);
irq++;
rid++;
- intrq++;
}
- } else {
-
- /* Each port has its own set of interrupt queues */
- for (p = 0; p < sc->params.nports; p++) {
- for (q = 0; q < sc->port[p]->nrxq; q++) {
- snprintf(s, sizeof(s), "%d.%d", p, q);
- T4_ALLOC_IRQ(sc, irq, rid, t4_intr,
- intrq, s);
- irq++;
- rid++;
- intrq++;
- }
+#ifndef TCP_OFFLOAD_DISABLE
+ /*
+ * Skip over the offload queues if they aren't taking
+ * direct interrupts.
+ */
+ if (!(sc->flags & INTR_DIRECT))
+ continue;
+ofld_queues:
+ ofld_rxq = &sc->sge.ofld_rxq[pi->first_ofld_rxq];
+ for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) {
+ snprintf(s, sizeof(s), "%d,%d", p, q);
+ T4_ALLOC_IRQ(sc, irq, rid, t4_intr, ofld_rxq, s);
+ irq++;
+ rid++;
}
+#endif
}
}
t4_intr_enable(sc);
sc->flags |= FULL_INIT_DONE;
-
done:
if (rc != 0)
- last_port_down(sc);
+ adapter_full_uninit(sc);
return (rc);
}
#undef T4_ALLOC_IRQ
-/*
- * Idempotent.
- */
static int
-last_port_down(struct adapter *sc)
+adapter_full_uninit(struct adapter *sc)
{
int i;
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
- t4_intr_disable(sc);
-
t4_teardown_adapter_queues(sc);
for (i = 0; i < sc->intr_count; i++)
t4_free_irq(sc, &sc->irq[i]);
+ for (i = 0; i < ARRAY_SIZE(sc->tq) && sc->tq[i]; i++) {
+ taskqueue_free(sc->tq[i]);
+ sc->tq[i] = NULL;
+ }
+
sc->flags &= ~FULL_INIT_DONE;
return (0);
}
static int
+port_full_init(struct port_info *pi)
+{
+ struct adapter *sc = pi->adapter;
+ struct ifnet *ifp = pi->ifp;
+ uint16_t *rss;
+ struct sge_rxq *rxq;
+ int rc, i;
+
+ ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
+ KASSERT((pi->flags & PORT_INIT_DONE) == 0,
+ ("%s: PORT_INIT_DONE already", __func__));
+
+ sysctl_ctx_init(&pi->ctx);
+ pi->flags |= PORT_SYSCTL_CTX;
+
+ /*
+ * Allocate tx/rx/fl queues for this port.
+ */
+ rc = t4_setup_port_queues(pi);
+ if (rc != 0)
+ goto done; /* error message displayed already */
+
+ /*
+ * Setup RSS for this port.
+ */
+ rss = malloc(pi->nrxq * sizeof (*rss), M_CXGBE,
+ M_ZERO | M_WAITOK);
+ for_each_rxq(pi, i, rxq) {
+ rss[i] = rxq->iq.abs_id;
+ }
+ rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0,
+ pi->rss_size, rss, pi->nrxq);
+ free(rss, M_CXGBE);
+ if (rc != 0) {
+ if_printf(ifp, "rss_config failed: %d\n", rc);
+ goto done;
+ }
+
+ pi->flags |= PORT_INIT_DONE;
+done:
+ if (rc != 0)
+ port_full_uninit(pi);
+
+ return (rc);
+}
+
+/*
+ * Idempotent.
+ */
+static int
+port_full_uninit(struct port_info *pi)
+{
+ struct adapter *sc = pi->adapter;
+ int i;
+ struct sge_rxq *rxq;
+ struct sge_txq *txq;
+#ifndef TCP_OFFLOAD_DISABLE
+ struct sge_ofld_rxq *ofld_rxq;
+ struct sge_wrq *ofld_txq;
+#endif
+
+ if (pi->flags & PORT_INIT_DONE) {
+
+ /* Need to quiesce queues. XXX: ctrl queues? */
+
+ for_each_txq(pi, i, txq) {
+ quiesce_eq(sc, &txq->eq);
+ }
+
+#ifndef TCP_OFFLOAD_DISABLE
+ for_each_ofld_txq(pi, i, ofld_txq) {
+ quiesce_eq(sc, &ofld_txq->eq);
+ }
+#endif
+
+ for_each_rxq(pi, i, rxq) {
+ quiesce_iq(sc, &rxq->iq);
+ quiesce_fl(sc, &rxq->fl);
+ }
+
+#ifndef TCP_OFFLOAD_DISABLE
+ for_each_ofld_rxq(pi, i, ofld_rxq) {
+ quiesce_iq(sc, &ofld_rxq->iq);
+ quiesce_fl(sc, &ofld_rxq->fl);
+ }
+#endif
+ }
+
+ t4_teardown_port_queues(pi);
+ pi->flags &= ~PORT_INIT_DONE;
+
+ return (0);
+}
+
+static void
+quiesce_eq(struct adapter *sc, struct sge_eq *eq)
+{
+ EQ_LOCK(eq);
+ eq->flags |= EQ_DOOMED;
+
+ /*
+ * Wait for the response to a credit flush if one's
+ * pending.
+ */
+ while (eq->flags & EQ_CRFLUSHED)
+ mtx_sleep(eq, &eq->eq_lock, 0, "crflush", 0);
+ EQ_UNLOCK(eq);
+
+ callout_drain(&eq->tx_callout); /* XXX: iffy */
+ pause("callout", 10); /* Still iffy */
+
+ taskqueue_drain(sc->tq[eq->tx_chan], &eq->tx_task);
+}
+
+static void
+quiesce_iq(struct adapter *sc, struct sge_iq *iq)
+{
+ (void) sc; /* unused */
+
+ /* Synchronize with the interrupt handler */
+ while (!atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_DISABLED))
+ pause("iqfree", 1);
+}
+
+static void
+quiesce_fl(struct adapter *sc, struct sge_fl *fl)
+{
+ mtx_lock(&sc->sfl_lock);
+ FL_LOCK(fl);
+ fl->flags |= FL_DOOMED;
+ FL_UNLOCK(fl);
+ mtx_unlock(&sc->sfl_lock);
+
+ callout_drain(&sc->sfl_callout);
+ KASSERT((fl->flags & FL_STARVING) == 0,
+ ("%s: still starving", __func__));
+}
+
+static int
t4_alloc_irq(struct adapter *sc, struct irq *irq, int rid,
- iq_intr_handler_t *handler, void *arg, char *name)
+ driver_intr_t *handler, void *arg, char *name)
{
int rc;
@@ -2365,12 +2852,12 @@ cxgbe_tick(void *arg)
t4_get_port_stats(pi->adapter, pi->tx_chan, s);
- ifp->if_opackets = s->tx_frames;
- ifp->if_ipackets = s->rx_frames;
- ifp->if_obytes = s->tx_octets;
- ifp->if_ibytes = s->rx_octets;
- ifp->if_omcasts = s->tx_mcast_frames;
- ifp->if_imcasts = s->rx_mcast_frames;
+ ifp->if_opackets = s->tx_frames - s->tx_pause;
+ ifp->if_ipackets = s->rx_frames - s->rx_pause;
+ ifp->if_obytes = s->tx_octets - s->tx_pause * 64;
+ ifp->if_ibytes = s->rx_octets - s->rx_pause * 64;
+ ifp->if_omcasts = s->tx_mcast_frames - s->tx_pause;
+ ifp->if_imcasts = s->rx_mcast_frames - s->rx_pause;
ifp->if_iqdrops = s->rx_ovflow0 + s->rx_ovflow1 + s->rx_ovflow2 +
s->rx_ovflow3;
@@ -2388,15 +2875,58 @@ cxgbe_tick(void *arg)
}
static int
+cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
+{
+#ifdef INVARIANTS
+ panic("%s: opcode %02x on iq %p with payload %p",
+ __func__, rss->opcode, iq, m);
+#else
+ log(LOG_ERR, "%s: opcode %02x on iq %p with payload %p",
+ __func__, rss->opcode, iq, m);
+ m_freem(m);
+#endif
+ return (EDOOFUS);
+}
+
+int
+t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
+{
+ uintptr_t *loc, new;
+
+ if (opcode >= ARRAY_SIZE(sc->cpl_handler))
+ return (EINVAL);
+
+ new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
+ loc = (uintptr_t *) &sc->cpl_handler[opcode];
+ atomic_store_rel_ptr(loc, new);
+
+ return (0);
+}
+
+static int
t4_sysctls(struct adapter *sc)
{
struct sysctl_ctx_list *ctx;
struct sysctl_oid *oid;
- struct sysctl_oid_list *children;
+ struct sysctl_oid_list *children, *c0;
+ static char *caps[] = {
+ "\20\1PPP\2QFC\3DCBX", /* caps[0] linkcaps */
+ "\20\1NIC\2VM\3IDS\4UM\5UM_ISGL", /* caps[1] niccaps */
+ "\20\1TOE", /* caps[2] toecaps */
+ "\20\1RDDP\2RDMAC", /* caps[3] rdmacaps */
+ "\20\1INITIATOR_PDU\2TARGET_PDU" /* caps[4] iscsicaps */
+ "\3INITIATOR_CNXOFLD\4TARGET_CNXOFLD"
+ "\5INITIATOR_SSNOFLD\6TARGET_SSNOFLD",
+ "\20\1INITIATOR\2TARGET\3CTRL_OFLD" /* caps[5] fcoecaps */
+ };
ctx = device_get_sysctl_ctx(sc->dev);
+
+ /*
+ * dev.t4nex.X.
+ */
oid = device_get_sysctl_tree(sc->dev);
- children = SYSCTL_CHILDREN(oid);
+ c0 = children = SYSCTL_CHILDREN(oid);
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nports", CTLFLAG_RD,
&sc->params.nports, 0, "# of ports");
@@ -2407,23 +2937,146 @@ t4_sysctls(struct adapter *sc)
SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "firmware_version",
CTLFLAG_RD, &sc->fw_version, 0, "firmware version");
- SYSCTL_ADD_INT(ctx, children, OID_AUTO, "TOE", CTLFLAG_RD,
- &sc->params.offload, 0, "hardware is capable of TCP offload");
+ SYSCTL_ADD_STRING(ctx, children, OID_AUTO, "cf",
+ CTLFLAG_RD, &t4_cfg_file, 0, "configuration file");
+
+ SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cfcsum", CTLFLAG_RD,
+ &sc->cfcsum, 0, "config file checksum");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "linkcaps",
+ CTLTYPE_STRING | CTLFLAG_RD, caps[0], sc->linkcaps,
+ sysctl_bitfield, "A", "available link capabilities");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "niccaps",
+ CTLTYPE_STRING | CTLFLAG_RD, caps[1], sc->niccaps,
+ sysctl_bitfield, "A", "available NIC capabilities");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "toecaps",
+ CTLTYPE_STRING | CTLFLAG_RD, caps[2], sc->toecaps,
+ sysctl_bitfield, "A", "available TCP offload capabilities");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdmacaps",
+ CTLTYPE_STRING | CTLFLAG_RD, caps[3], sc->rdmacaps,
+ sysctl_bitfield, "A", "available RDMA capabilities");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "iscsicaps",
+ CTLTYPE_STRING | CTLFLAG_RD, caps[4], sc->iscsicaps,
+ sysctl_bitfield, "A", "available iSCSI capabilities");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoecaps",
+ CTLTYPE_STRING | CTLFLAG_RD, caps[5], sc->fcoecaps,
+ sysctl_bitfield, "A", "available FCoE capabilities");
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "core_clock", CTLFLAG_RD,
&sc->params.vpd.cclk, 0, "core clock frequency (in KHz)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_timers",
- CTLTYPE_STRING | CTLFLAG_RD, &intr_timer, sizeof(intr_timer),
- sysctl_int_array, "A", "interrupt holdoff timer values (us)");
+ CTLTYPE_STRING | CTLFLAG_RD, sc->sge.timer_val,
+ sizeof(sc->sge.timer_val), sysctl_int_array, "A",
+ "interrupt holdoff timer values (us)");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_pkt_counts",
- CTLTYPE_STRING | CTLFLAG_RD, &intr_pktcount, sizeof(intr_pktcount),
- sysctl_int_array, "A", "interrupt holdoff packet counter values");
+ CTLTYPE_STRING | CTLFLAG_RD, sc->sge.counter_val,
+ sizeof(sc->sge.counter_val), sysctl_int_array, "A",
+ "interrupt holdoff packet counter values");
+
+ /*
+ * dev.t4nex.X.misc. Marked CTLFLAG_SKIP to avoid information overload.
+ */
+ oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "misc",
+ CTLFLAG_RD | CTLFLAG_SKIP, NULL,
+ "logs and miscellaneous information");
+ children = SYSCTL_CHILDREN(oid);
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cctrl",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_cctrl, "A", "congestion control");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cpl_stats",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_cpl_stats, "A", "CPL statistics");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "ddp_stats",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_ddp_stats, "A", "DDP statistics");
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "devlog",
CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
- sysctl_devlog, "A", "device log");
+ sysctl_devlog, "A", "firmware's device log");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "fcoe_stats",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_fcoe_stats, "A", "FCoE statistics");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "hw_sched",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_hw_sched, "A", "hardware scheduler ");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "l2t",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_l2t, "A", "hardware L2 table");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "lb_stats",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_lb_stats, "A", "loopback statistics");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "meminfo",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_meminfo, "A", "memory regions");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "path_mtus",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_path_mtus, "A", "path MTUs");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pm_stats",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_pm_stats, "A", "PM statistics");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "rdma_stats",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_rdma_stats, "A", "RDMA statistics");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tcp_stats",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_tcp_stats, "A", "TCP statistics");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tids",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_tids, "A", "TID information");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tp_err_stats",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_tp_err_stats, "A", "TP error statistics");
+
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "tx_rate",
+ CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
+ sysctl_tx_rate, "A", "Tx rate");
+
+#ifndef TCP_OFFLOAD_DISABLE
+ if (is_offload(sc)) {
+ /*
+ * dev.t4nex.X.toe.
+ */
+ oid = SYSCTL_ADD_NODE(ctx, c0, OID_AUTO, "toe", CTLFLAG_RD,
+ NULL, "TOE parameters");
+ children = SYSCTL_CHILDREN(oid);
+
+ sc->tt.sndbuf = 256 * 1024;
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "sndbuf", CTLFLAG_RW,
+ &sc->tt.sndbuf, 0, "max hardware send buffer size");
+
+ sc->tt.ddp = 0;
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp", CTLFLAG_RW,
+ &sc->tt.ddp, 0, "DDP allowed");
+ sc->tt.indsz = M_INDICATESIZE;
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "indsz", CTLFLAG_RW,
+ &sc->tt.indsz, 0, "DDP max indicate size allowed");
+ sc->tt.ddp_thres = 3*4096;
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "ddp_thres", CTLFLAG_RW,
+ &sc->tt.ddp_thres, 0, "DDP threshold");
+ }
+#endif
+
return (0);
}
@@ -2452,6 +3105,23 @@ cxgbe_sysctls(struct port_info *pi)
SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_txq", CTLFLAG_RD,
&pi->first_txq, 0, "index of first tx queue");
+#ifndef TCP_OFFLOAD_DISABLE
+ if (is_offload(pi->adapter)) {
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldrxq", CTLFLAG_RD,
+ &pi->nofldrxq, 0,
+ "# of rx queues for offloaded TCP connections");
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "nofldtxq", CTLFLAG_RD,
+ &pi->nofldtxq, 0,
+ "# of tx queues for offloaded TCP connections");
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_rxq",
+ CTLFLAG_RD, &pi->first_ofld_rxq, 0,
+ "index of first TOE rx queue");
+ SYSCTL_ADD_INT(ctx, children, OID_AUTO, "first_ofld_txq",
+ CTLFLAG_RD, &pi->first_ofld_txq, 0,
+ "index of first TOE tx queue");
+ }
+#endif
+
SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "holdoff_tmr_idx",
CTLTYPE_INT | CTLFLAG_RW, pi, 0, sysctl_holdoff_tmr_idx, "I",
"holdoff timer index");
@@ -2642,11 +3312,31 @@ sysctl_int_array(SYSCTL_HANDLER_ARGS)
}
static int
+sysctl_bitfield(SYSCTL_HANDLER_ARGS)
+{
+ int rc;
+ struct sbuf *sb;
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return(rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ sbuf_printf(sb, "%b", (int)arg2, (char *)arg1);
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
{
struct port_info *pi = arg1;
struct adapter *sc = pi->adapter;
- struct sge_rxq *rxq;
int idx, rc, i;
idx = pi->tmr_idx;
@@ -2661,9 +3351,16 @@ sysctl_holdoff_tmr_idx(SYSCTL_HANDLER_ARGS)
ADAPTER_LOCK(sc);
rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
if (rc == 0) {
+ struct sge_rxq *rxq;
+ uint8_t v;
+
+ v = V_QINTR_TIMER_IDX(idx) | V_QINTR_CNT_EN(pi->pktc_idx != -1);
for_each_rxq(pi, i, rxq) {
- rxq->iq.intr_params = V_QINTR_TIMER_IDX(idx) |
- V_QINTR_CNT_EN(pi->pktc_idx != -1);
+#ifdef atomic_store_rel_8
+ atomic_store_rel_8(&rxq->iq.intr_params, v);
+#else
+ rxq->iq.intr_params = v;
+#endif
}
pi->tmr_idx = idx;
}
@@ -2690,8 +3387,8 @@ sysctl_holdoff_pktc_idx(SYSCTL_HANDLER_ARGS)
ADAPTER_LOCK(sc);
rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
- if (rc == 0 && pi->ifp->if_drv_flags & IFF_DRV_RUNNING)
- rc = EBUSY; /* can be changed only when port is down */
+ if (rc == 0 && pi->flags & PORT_INIT_DONE)
+ rc = EBUSY; /* cannot be changed once the queues are created */
if (rc == 0)
pi->pktc_idx = idx;
@@ -2718,8 +3415,8 @@ sysctl_qsize_rxq(SYSCTL_HANDLER_ARGS)
ADAPTER_LOCK(sc);
rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
- if (rc == 0 && pi->ifp->if_drv_flags & IFF_DRV_RUNNING)
- rc = EBUSY; /* can be changed only when port is down */
+ if (rc == 0 && pi->flags & PORT_INIT_DONE)
+ rc = EBUSY; /* cannot be changed once the queues are created */
if (rc == 0)
pi->qsize_rxq = qsize;
@@ -2746,8 +3443,8 @@ sysctl_qsize_txq(SYSCTL_HANDLER_ARGS)
ADAPTER_LOCK(sc);
rc = IS_DOOMED(pi) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
- if (rc == 0 && pi->ifp->if_drv_flags & IFF_DRV_RUNNING)
- rc = EBUSY; /* can be changed only when port is down */
+ if (rc == 0 && pi->flags & PORT_INIT_DONE)
+ rc = EBUSY; /* cannot be changed once the queues are created */
if (rc == 0)
pi->qsize_txq = qsize;
@@ -2768,6 +3465,103 @@ sysctl_handle_t4_reg64(SYSCTL_HANDLER_ARGS)
return (sysctl_handle_64(oidp, &val, 0, req));
}
+static int
+sysctl_cctrl(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc, i;
+ uint16_t incr[NMTUS][NCCTRL_WIN];
+ static const char *dec_fac[] = {
+ "0.5", "0.5625", "0.625", "0.6875", "0.75", "0.8125", "0.875",
+ "0.9375"
+ };
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_read_cong_tbl(sc, incr);
+
+ for (i = 0; i < NCCTRL_WIN; ++i) {
+ sbuf_printf(sb, "%2d: %4u %4u %4u %4u %4u %4u %4u %4u\n", i,
+ incr[0][i], incr[1][i], incr[2][i], incr[3][i], incr[4][i],
+ incr[5][i], incr[6][i], incr[7][i]);
+ sbuf_printf(sb, "%8u %4u %4u %4u %4u %4u %4u %4u %5u %s\n",
+ incr[8][i], incr[9][i], incr[10][i], incr[11][i],
+ incr[12][i], incr[13][i], incr[14][i], incr[15][i],
+ sc->params.a_wnd[i], dec_fac[sc->params.b_wnd[i]]);
+ }
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_cpl_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc;
+ struct tp_cpl_stats stats;
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_tp_get_cpl_stats(sc, &stats);
+
+ sbuf_printf(sb, " channel 0 channel 1 channel 2 "
+ "channel 3\n");
+ sbuf_printf(sb, "CPL requests: %10u %10u %10u %10u\n",
+ stats.req[0], stats.req[1], stats.req[2], stats.req[3]);
+ sbuf_printf(sb, "CPL responses: %10u %10u %10u %10u",
+ stats.rsp[0], stats.rsp[1], stats.rsp[2], stats.rsp[3]);
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_ddp_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc;
+ struct tp_usm_stats stats;
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return(rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_get_usm_stats(sc, &stats);
+
+ sbuf_printf(sb, "Frames: %u\n", stats.frames);
+ sbuf_printf(sb, "Octets: %ju\n", stats.octets);
+ sbuf_printf(sb, "Drops: %u", stats.drops);
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
const char *devlog_level_strings[] = {
[FW_DEVLOG_LEVEL_EMERG] = "EMERG",
[FW_DEVLOG_LEVEL_CRIT] = "CRIT",
@@ -2852,7 +3646,11 @@ sysctl_devlog(SYSCTL_HANDLER_ARGS)
goto done;
sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
- sbuf_printf(sb, "\n%10s %15s %8s %8s %s\n",
+ if (sb == NULL) {
+ rc = ENOMEM;
+ goto done;
+ }
+ sbuf_printf(sb, "%10s %15s %8s %8s %s\n",
"Seq#", "Tstamp", "Level", "Facility", "Message");
i = first;
@@ -2882,6 +3680,624 @@ done:
return (rc);
}
+static int
+sysctl_fcoe_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc;
+ struct tp_fcoe_stats stats[4];
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_get_fcoe_stats(sc, 0, &stats[0]);
+ t4_get_fcoe_stats(sc, 1, &stats[1]);
+ t4_get_fcoe_stats(sc, 2, &stats[2]);
+ t4_get_fcoe_stats(sc, 3, &stats[3]);
+
+ sbuf_printf(sb, " channel 0 channel 1 "
+ "channel 2 channel 3\n");
+ sbuf_printf(sb, "octetsDDP: %16ju %16ju %16ju %16ju\n",
+ stats[0].octetsDDP, stats[1].octetsDDP, stats[2].octetsDDP,
+ stats[3].octetsDDP);
+ sbuf_printf(sb, "framesDDP: %16u %16u %16u %16u\n", stats[0].framesDDP,
+ stats[1].framesDDP, stats[2].framesDDP, stats[3].framesDDP);
+ sbuf_printf(sb, "framesDrop: %16u %16u %16u %16u",
+ stats[0].framesDrop, stats[1].framesDrop, stats[2].framesDrop,
+ stats[3].framesDrop);
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_hw_sched(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc, i;
+ unsigned int map, kbps, ipg, mode;
+ unsigned int pace_tab[NTX_SCHED];
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ map = t4_read_reg(sc, A_TP_TX_MOD_QUEUE_REQ_MAP);
+ mode = G_TIMERMODE(t4_read_reg(sc, A_TP_MOD_CONFIG));
+ t4_read_pace_tbl(sc, pace_tab);
+
+ sbuf_printf(sb, "Scheduler Mode Channel Rate (Kbps) "
+ "Class IPG (0.1 ns) Flow IPG (us)");
+
+ for (i = 0; i < NTX_SCHED; ++i, map >>= 2) {
+ t4_get_tx_sched(sc, i, &kbps, &ipg);
+ sbuf_printf(sb, "\n %u %-5s %u ", i,
+ (mode & (1 << i)) ? "flow" : "class", map & 3);
+ if (kbps)
+ sbuf_printf(sb, "%9u ", kbps);
+ else
+ sbuf_printf(sb, " disabled ");
+
+ if (ipg)
+ sbuf_printf(sb, "%13u ", ipg);
+ else
+ sbuf_printf(sb, " disabled ");
+
+ if (pace_tab[i])
+ sbuf_printf(sb, "%10u", pace_tab[i]);
+ else
+ sbuf_printf(sb, " disabled");
+ }
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_lb_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc, i, j;
+ uint64_t *p0, *p1;
+ struct lb_port_stats s[2];
+ static const char *stat_name[] = {
+ "OctetsOK:", "FramesOK:", "BcastFrames:", "McastFrames:",
+ "UcastFrames:", "ErrorFrames:", "Frames64:", "Frames65To127:",
+ "Frames128To255:", "Frames256To511:", "Frames512To1023:",
+ "Frames1024To1518:", "Frames1519ToMax:", "FramesDropped:",
+ "BG0FramesDropped:", "BG1FramesDropped:", "BG2FramesDropped:",
+ "BG3FramesDropped:", "BG0FramesTrunc:", "BG1FramesTrunc:",
+ "BG2FramesTrunc:", "BG3FramesTrunc:"
+ };
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ memset(s, 0, sizeof(s));
+
+ for (i = 0; i < 4; i += 2) {
+ t4_get_lb_stats(sc, i, &s[0]);
+ t4_get_lb_stats(sc, i + 1, &s[1]);
+
+ p0 = &s[0].octets;
+ p1 = &s[1].octets;
+ sbuf_printf(sb, "%s Loopback %u"
+ " Loopback %u", i == 0 ? "" : "\n", i, i + 1);
+
+ for (j = 0; j < ARRAY_SIZE(stat_name); j++)
+ sbuf_printf(sb, "\n%-17s %20ju %20ju", stat_name[j],
+ *p0++, *p1++);
+ }
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+struct mem_desc {
+ unsigned int base;
+ unsigned int limit;
+ unsigned int idx;
+};
+
+static int
+mem_desc_cmp(const void *a, const void *b)
+{
+ return ((const struct mem_desc *)a)->base -
+ ((const struct mem_desc *)b)->base;
+}
+
+static void
+mem_region_show(struct sbuf *sb, const char *name, unsigned int from,
+ unsigned int to)
+{
+ unsigned int size;
+
+ size = to - from + 1;
+ if (size == 0)
+ return;
+
+ /* XXX: need humanize_number(3) in libkern for a more readable 'size' */
+ sbuf_printf(sb, "%-15s %#x-%#x [%u]\n", name, from, to, size);
+}
+
+static int
+sysctl_meminfo(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc, i, n;
+ uint32_t lo, hi;
+ static const char *memory[] = { "EDC0:", "EDC1:", "MC:" };
+ static const char *region[] = {
+ "DBQ contexts:", "IMSG contexts:", "FLM cache:", "TCBs:",
+ "Pstructs:", "Timers:", "Rx FL:", "Tx FL:", "Pstruct FL:",
+ "Tx payload:", "Rx payload:", "LE hash:", "iSCSI region:",
+ "TDDP region:", "TPT region:", "STAG region:", "RQ region:",
+ "RQUDP region:", "PBL region:", "TXPBL region:", "ULPRX state:",
+ "ULPTX state:", "On-chip queues:"
+ };
+ struct mem_desc avail[3];
+ struct mem_desc mem[ARRAY_SIZE(region) + 3]; /* up to 3 holes */
+ struct mem_desc *md = mem;
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 4096, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ for (i = 0; i < ARRAY_SIZE(mem); i++) {
+ mem[i].limit = 0;
+ mem[i].idx = i;
+ }
+
+ /* Find and sort the populated memory ranges */
+ i = 0;
+ lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
+ if (lo & F_EDRAM0_ENABLE) {
+ hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
+ avail[i].base = G_EDRAM0_BASE(hi) << 20;
+ avail[i].limit = avail[i].base + (G_EDRAM0_SIZE(hi) << 20);
+ avail[i].idx = 0;
+ i++;
+ }
+ if (lo & F_EDRAM1_ENABLE) {
+ hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
+ avail[i].base = G_EDRAM1_BASE(hi) << 20;
+ avail[i].limit = avail[i].base + (G_EDRAM1_SIZE(hi) << 20);
+ avail[i].idx = 1;
+ i++;
+ }
+ if (lo & F_EXT_MEM_ENABLE) {
+ hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
+ avail[i].base = G_EXT_MEM_BASE(hi) << 20;
+ avail[i].limit = avail[i].base + (G_EXT_MEM_SIZE(hi) << 20);
+ avail[i].idx = 2;
+ i++;
+ }
+ if (!i) /* no memory available */
+ return 0;
+ qsort(avail, i, sizeof(struct mem_desc), mem_desc_cmp);
+
+ (md++)->base = t4_read_reg(sc, A_SGE_DBQ_CTXT_BADDR);
+ (md++)->base = t4_read_reg(sc, A_SGE_IMSG_CTXT_BADDR);
+ (md++)->base = t4_read_reg(sc, A_SGE_FLM_CACHE_BADDR);
+ (md++)->base = t4_read_reg(sc, A_TP_CMM_TCB_BASE);
+ (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_BASE);
+ (md++)->base = t4_read_reg(sc, A_TP_CMM_TIMER_BASE);
+ (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_RX_FLST_BASE);
+ (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_TX_FLST_BASE);
+ (md++)->base = t4_read_reg(sc, A_TP_CMM_MM_PS_FLST_BASE);
+
+ /* the next few have explicit upper bounds */
+ md->base = t4_read_reg(sc, A_TP_PMM_TX_BASE);
+ md->limit = md->base - 1 +
+ t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE) *
+ G_PMTXMAXPAGE(t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE));
+ md++;
+
+ md->base = t4_read_reg(sc, A_TP_PMM_RX_BASE);
+ md->limit = md->base - 1 +
+ t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) *
+ G_PMRXMAXPAGE(t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE));
+ md++;
+
+ if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
+ hi = t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4;
+ md->base = t4_read_reg(sc, A_LE_DB_HASH_TID_BASE);
+ md->limit = (sc->tids.ntids - hi) * 16 + md->base - 1;
+ } else {
+ md->base = 0;
+ md->idx = ARRAY_SIZE(region); /* hide it */
+ }
+ md++;
+
+#define ulp_region(reg) \
+ md->base = t4_read_reg(sc, A_ULP_ ## reg ## _LLIMIT);\
+ (md++)->limit = t4_read_reg(sc, A_ULP_ ## reg ## _ULIMIT)
+
+ ulp_region(RX_ISCSI);
+ ulp_region(RX_TDDP);
+ ulp_region(TX_TPT);
+ ulp_region(RX_STAG);
+ ulp_region(RX_RQ);
+ ulp_region(RX_RQUDP);
+ ulp_region(RX_PBL);
+ ulp_region(TX_PBL);
+#undef ulp_region
+
+ md->base = t4_read_reg(sc, A_ULP_RX_CTX_BASE);
+ md->limit = md->base + sc->tids.ntids - 1;
+ md++;
+ md->base = t4_read_reg(sc, A_ULP_TX_ERR_TABLE_BASE);
+ md->limit = md->base + sc->tids.ntids - 1;
+ md++;
+
+ md->base = sc->vres.ocq.start;
+ if (sc->vres.ocq.size)
+ md->limit = md->base + sc->vres.ocq.size - 1;
+ else
+ md->idx = ARRAY_SIZE(region); /* hide it */
+ md++;
+
+ /* add any address-space holes, there can be up to 3 */
+ for (n = 0; n < i - 1; n++)
+ if (avail[n].limit < avail[n + 1].base)
+ (md++)->base = avail[n].limit;
+ if (avail[n].limit)
+ (md++)->base = avail[n].limit;
+
+ n = md - mem;
+ qsort(mem, n, sizeof(struct mem_desc), mem_desc_cmp);
+
+ for (lo = 0; lo < i; lo++)
+ mem_region_show(sb, memory[avail[lo].idx], avail[lo].base,
+ avail[lo].limit - 1);
+
+ sbuf_printf(sb, "\n");
+ for (i = 0; i < n; i++) {
+ if (mem[i].idx >= ARRAY_SIZE(region))
+ continue; /* skip holes */
+ if (!mem[i].limit)
+ mem[i].limit = i < n - 1 ? mem[i + 1].base - 1 : ~0;
+ mem_region_show(sb, region[mem[i].idx], mem[i].base,
+ mem[i].limit);
+ }
+
+ sbuf_printf(sb, "\n");
+ lo = t4_read_reg(sc, A_CIM_SDRAM_BASE_ADDR);
+ hi = t4_read_reg(sc, A_CIM_SDRAM_ADDR_SIZE) + lo - 1;
+ mem_region_show(sb, "uP RAM:", lo, hi);
+
+ lo = t4_read_reg(sc, A_CIM_EXTMEM2_BASE_ADDR);
+ hi = t4_read_reg(sc, A_CIM_EXTMEM2_ADDR_SIZE) + lo - 1;
+ mem_region_show(sb, "uP Extmem2:", lo, hi);
+
+ lo = t4_read_reg(sc, A_TP_PMM_RX_MAX_PAGE);
+ sbuf_printf(sb, "\n%u Rx pages of size %uKiB for %u channels\n",
+ G_PMRXMAXPAGE(lo),
+ t4_read_reg(sc, A_TP_PMM_RX_PAGE_SIZE) >> 10,
+ (lo & F_PMRXNUMCHN) ? 2 : 1);
+
+ lo = t4_read_reg(sc, A_TP_PMM_TX_MAX_PAGE);
+ hi = t4_read_reg(sc, A_TP_PMM_TX_PAGE_SIZE);
+ sbuf_printf(sb, "%u Tx pages of size %u%ciB for %u channels\n",
+ G_PMTXMAXPAGE(lo),
+ hi >= (1 << 20) ? (hi >> 20) : (hi >> 10),
+ hi >= (1 << 20) ? 'M' : 'K', 1 << G_PMTXNUMCHN(lo));
+ sbuf_printf(sb, "%u p-structs\n",
+ t4_read_reg(sc, A_TP_CMM_MM_MAX_PSTRUCT));
+
+ for (i = 0; i < 4; i++) {
+ lo = t4_read_reg(sc, A_MPS_RX_PG_RSV0 + i * 4);
+ sbuf_printf(sb, "\nPort %d using %u pages out of %u allocated",
+ i, G_USED(lo), G_ALLOC(lo));
+ }
+ for (i = 0; i < 4; i++) {
+ lo = t4_read_reg(sc, A_MPS_RX_PG_RSV4 + i * 4);
+ sbuf_printf(sb,
+ "\nLoopback %d using %u pages out of %u allocated",
+ i, G_USED(lo), G_ALLOC(lo));
+ }
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_path_mtus(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc;
+ uint16_t mtus[NMTUS];
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_read_mtu_tbl(sc, mtus, NULL);
+
+ sbuf_printf(sb, "%u %u %u %u %u %u %u %u %u %u %u %u %u %u %u %u",
+ mtus[0], mtus[1], mtus[2], mtus[3], mtus[4], mtus[5], mtus[6],
+ mtus[7], mtus[8], mtus[9], mtus[10], mtus[11], mtus[12], mtus[13],
+ mtus[14], mtus[15]);
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_pm_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc, i;
+ uint32_t tx_cnt[PM_NSTATS], rx_cnt[PM_NSTATS];
+ uint64_t tx_cyc[PM_NSTATS], rx_cyc[PM_NSTATS];
+ static const char *pm_stats[] = {
+ "Read:", "Write bypass:", "Write mem:", "Flush:", "FIFO wait:"
+ };
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_pmtx_get_stats(sc, tx_cnt, tx_cyc);
+ t4_pmrx_get_stats(sc, rx_cnt, rx_cyc);
+
+ sbuf_printf(sb, " Tx count Tx cycles "
+ "Rx count Rx cycles");
+ for (i = 0; i < PM_NSTATS; i++)
+ sbuf_printf(sb, "\n%-13s %10u %20ju %10u %20ju",
+ pm_stats[i], tx_cnt[i], tx_cyc[i], rx_cnt[i], rx_cyc[i]);
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_rdma_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc;
+ struct tp_rdma_stats stats;
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_tp_get_rdma_stats(sc, &stats);
+ sbuf_printf(sb, "NoRQEModDefferals: %u\n", stats.rqe_dfr_mod);
+ sbuf_printf(sb, "NoRQEPktDefferals: %u", stats.rqe_dfr_pkt);
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_tcp_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc;
+ struct tp_tcp_stats v4, v6;
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_tp_get_tcp_stats(sc, &v4, &v6);
+ sbuf_printf(sb,
+ " IP IPv6\n");
+ sbuf_printf(sb, "OutRsts: %20u %20u\n",
+ v4.tcpOutRsts, v6.tcpOutRsts);
+ sbuf_printf(sb, "InSegs: %20ju %20ju\n",
+ v4.tcpInSegs, v6.tcpInSegs);
+ sbuf_printf(sb, "OutSegs: %20ju %20ju\n",
+ v4.tcpOutSegs, v6.tcpOutSegs);
+ sbuf_printf(sb, "RetransSegs: %20ju %20ju",
+ v4.tcpRetransSegs, v6.tcpRetransSegs);
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_tids(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc;
+ struct tid_info *t = &sc->tids;
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ if (t->natids) {
+ sbuf_printf(sb, "ATID range: 0-%u, in use: %u\n", t->natids - 1,
+ t->atids_in_use);
+ }
+
+ if (t->ntids) {
+ if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
+ uint32_t b = t4_read_reg(sc, A_LE_DB_SERVER_INDEX) / 4;
+
+ if (b) {
+ sbuf_printf(sb, "TID range: 0-%u, %u-%u", b - 1,
+ t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
+ t->ntids - 1);
+ } else {
+ sbuf_printf(sb, "TID range: %u-%u",
+ t4_read_reg(sc, A_LE_DB_TID_HASHBASE) / 4,
+ t->ntids - 1);
+ }
+ } else
+ sbuf_printf(sb, "TID range: 0-%u", t->ntids - 1);
+ sbuf_printf(sb, ", in use: %u\n",
+ atomic_load_acq_int(&t->tids_in_use));
+ }
+
+ if (t->nstids) {
+ sbuf_printf(sb, "STID range: %u-%u, in use: %u\n", t->stid_base,
+ t->stid_base + t->nstids - 1, t->stids_in_use);
+ }
+
+ if (t->nftids) {
+ sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
+ t->ftid_base + t->nftids - 1);
+ }
+
+ sbuf_printf(sb, "HW TID usage: %u IP users, %u IPv6 users",
+ t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV4),
+ t4_read_reg(sc, A_LE_DB_ACT_CNT_IPV6));
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_tp_err_stats(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc;
+ struct tp_err_stats stats;
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_tp_get_err_stats(sc, &stats);
+
+ sbuf_printf(sb, " channel 0 channel 1 channel 2 "
+ "channel 3\n");
+ sbuf_printf(sb, "macInErrs: %10u %10u %10u %10u\n",
+ stats.macInErrs[0], stats.macInErrs[1], stats.macInErrs[2],
+ stats.macInErrs[3]);
+ sbuf_printf(sb, "hdrInErrs: %10u %10u %10u %10u\n",
+ stats.hdrInErrs[0], stats.hdrInErrs[1], stats.hdrInErrs[2],
+ stats.hdrInErrs[3]);
+ sbuf_printf(sb, "tcpInErrs: %10u %10u %10u %10u\n",
+ stats.tcpInErrs[0], stats.tcpInErrs[1], stats.tcpInErrs[2],
+ stats.tcpInErrs[3]);
+ sbuf_printf(sb, "tcp6InErrs: %10u %10u %10u %10u\n",
+ stats.tcp6InErrs[0], stats.tcp6InErrs[1], stats.tcp6InErrs[2],
+ stats.tcp6InErrs[3]);
+ sbuf_printf(sb, "tnlCongDrops: %10u %10u %10u %10u\n",
+ stats.tnlCongDrops[0], stats.tnlCongDrops[1], stats.tnlCongDrops[2],
+ stats.tnlCongDrops[3]);
+ sbuf_printf(sb, "tnlTxDrops: %10u %10u %10u %10u\n",
+ stats.tnlTxDrops[0], stats.tnlTxDrops[1], stats.tnlTxDrops[2],
+ stats.tnlTxDrops[3]);
+ sbuf_printf(sb, "ofldVlanDrops: %10u %10u %10u %10u\n",
+ stats.ofldVlanDrops[0], stats.ofldVlanDrops[1],
+ stats.ofldVlanDrops[2], stats.ofldVlanDrops[3]);
+ sbuf_printf(sb, "ofldChanDrops: %10u %10u %10u %10u\n\n",
+ stats.ofldChanDrops[0], stats.ofldChanDrops[1],
+ stats.ofldChanDrops[2], stats.ofldChanDrops[3]);
+ sbuf_printf(sb, "ofldNoNeigh: %u\nofldCongDefer: %u",
+ stats.ofldNoNeigh, stats.ofldCongDefer);
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
+static int
+sysctl_tx_rate(SYSCTL_HANDLER_ARGS)
+{
+ struct adapter *sc = arg1;
+ struct sbuf *sb;
+ int rc;
+ u64 nrate[NCHAN], orate[NCHAN];
+
+ rc = sysctl_wire_old_buffer(req, 0);
+ if (rc != 0)
+ return (rc);
+
+ sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
+ if (sb == NULL)
+ return (ENOMEM);
+
+ t4_get_chan_txrate(sc, nrate, orate);
+ sbuf_printf(sb, " channel 0 channel 1 channel 2 "
+ "channel 3\n");
+ sbuf_printf(sb, "NIC B/s: %10ju %10ju %10ju %10ju\n",
+ nrate[0], nrate[1], nrate[2], nrate[3]);
+ sbuf_printf(sb, "Offload B/s: %10ju %10ju %10ju %10ju",
+ orate[0], orate[1], orate[2], orate[3]);
+
+ rc = sbuf_finish(sb);
+ sbuf_delete(sb);
+
+ return (rc);
+}
+
static inline void
txq_start(struct ifnet *ifp, struct sge_txq *txq)
{
@@ -2897,17 +4313,57 @@ txq_start(struct ifnet *ifp, struct sge_txq *txq)
}
void
-cxgbe_txq_start(void *arg, int count)
+t4_tx_callout(void *arg)
{
- struct sge_txq *txq = arg;
+ struct sge_eq *eq = arg;
+ struct adapter *sc;
+
+ if (EQ_TRYLOCK(eq) == 0)
+ goto reschedule;
+
+ if (eq->flags & EQ_STALLED && !can_resume_tx(eq)) {
+ EQ_UNLOCK(eq);
+reschedule:
+ if (__predict_true(!(eq->flags && EQ_DOOMED)))
+ callout_schedule(&eq->tx_callout, 1);
+ return;
+ }
+
+ EQ_LOCK_ASSERT_OWNED(eq);
+
+ if (__predict_true((eq->flags & EQ_DOOMED) == 0)) {
+
+ if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
+ struct sge_txq *txq = arg;
+ struct port_info *pi = txq->ifp->if_softc;
+
+ sc = pi->adapter;
+ } else {
+ struct sge_wrq *wrq = arg;
+
+ sc = wrq->adapter;
+ }
+
+ taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
+ }
- TXQ_LOCK(txq);
- if (txq->eq.flags & EQ_CRFLUSHED) {
- txq->eq.flags &= ~EQ_CRFLUSHED;
+ EQ_UNLOCK(eq);
+}
+
+void
+t4_tx_task(void *arg, int count)
+{
+ struct sge_eq *eq = arg;
+
+ EQ_LOCK(eq);
+ if ((eq->flags & EQ_TYPEMASK) == EQ_ETH) {
+ struct sge_txq *txq = arg;
txq_start(txq->ifp, txq);
- } else
- wakeup_one(txq); /* txq is going away, wakeup free_txq */
- TXQ_UNLOCK(txq);
+ } else {
+ struct sge_wrq *wrq = arg;
+ t4_wrq_tx_locked(wrq->adapter, wrq, NULL);
+ }
+ EQ_UNLOCK(eq);
}
static uint32_t
@@ -2937,10 +4393,10 @@ fconf_to_mode(uint32_t fconf)
mode |= T4_FILTER_IP_TOS;
if (fconf & F_VLAN)
- mode |= T4_FILTER_IVLAN;
+ mode |= T4_FILTER_VLAN;
if (fconf & F_VNIC_ID)
- mode |= T4_FILTER_OVLAN;
+ mode |= T4_FILTER_VNIC;
if (fconf & F_PORT)
mode |= T4_FILTER_PORT;
@@ -2974,10 +4430,10 @@ mode_to_fconf(uint32_t mode)
if (mode & T4_FILTER_IP_TOS)
fconf |= F_TOS;
- if (mode & T4_FILTER_IVLAN)
+ if (mode & T4_FILTER_VLAN)
fconf |= F_VLAN;
- if (mode & T4_FILTER_OVLAN)
+ if (mode & T4_FILTER_VNIC)
fconf |= F_VNIC_ID;
if (mode & T4_FILTER_PORT)
@@ -3012,10 +4468,10 @@ fspec_to_fconf(struct t4_filter_specification *fs)
if (fs->val.tos || fs->mask.tos)
fconf |= F_TOS;
- if (fs->val.ivlan_vld || fs->mask.ivlan_vld)
+ if (fs->val.vlan_vld || fs->mask.vlan_vld)
fconf |= F_VLAN;
- if (fs->val.ovlan_vld || fs->mask.ovlan_vld)
+ if (fs->val.vnic_vld || fs->mask.vnic_vld)
fconf |= F_VNIC_ID;
if (fs->val.iport || fs->mask.iport)
@@ -3035,7 +4491,13 @@ get_filter_mode(struct adapter *sc, uint32_t *mode)
t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
A_TP_VLAN_PRI_MAP);
- *mode = fconf_to_mode(fconf);
+ if (sc->filter_mode != fconf) {
+ log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
+ device_get_nameunit(sc->dev), sc->filter_mode, fconf);
+ sc->filter_mode = fconf;
+ }
+
+ *mode = fconf_to_mode(sc->filter_mode);
return (0);
}
@@ -3059,7 +4521,21 @@ set_filter_mode(struct adapter *sc, uint32_t mode)
goto done;
}
+#ifndef TCP_OFFLOAD_DISABLE
+ if (sc->offload_map) {
+ rc = EBUSY;
+ goto done;
+ }
+#endif
+
+#ifdef notyet
rc = -t4_set_filter_mode(sc, fconf);
+ if (rc == 0)
+ sc->filter_mode = fconf;
+#else
+ rc = ENOTSUP;
+#endif
+
done:
ADAPTER_UNLOCK(sc);
return (rc);
@@ -3119,7 +4595,6 @@ get_filter(struct adapter *sc, struct t4_filter *t)
static int
set_filter(struct adapter *sc, struct t4_filter *t)
{
- uint32_t fconf;
unsigned int nfilters, nports;
struct filter_entry *f;
int i;
@@ -3139,9 +4614,7 @@ set_filter(struct adapter *sc, struct t4_filter *t)
return (EINVAL);
/* Validate against the global filter mode */
- t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
- A_TP_VLAN_PRI_MAP);
- if ((fconf | fspec_to_fconf(&t->fs)) != fconf)
+ if ((sc->filter_mode | fspec_to_fconf(&t->fs)) != sc->filter_mode)
return (E2BIG);
if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports)
@@ -3238,7 +4711,6 @@ clear_filter(struct filter_entry *f)
static int
set_filter_wr(struct adapter *sc, int fidx)
{
- int rc;
struct filter_entry *f = &sc->tids.ftid_tab[fidx];
struct mbuf *m;
struct fw_filter_wr *fwr;
@@ -3298,13 +4770,13 @@ set_filter_wr(struct adapter *sc, int fidx)
fwr->frag_to_ovlan_vldm =
(V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
- V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.ivlan_vld) |
- V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.ovlan_vld) |
- V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.ivlan_vld) |
- V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.ovlan_vld));
+ V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
+ V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
+ V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
+ V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
fwr->smac_sel = 0;
fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
- V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.intrq[0].abs_id));
+ V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
fwr->maci_to_matchtypem =
htobe32(V_FW_FILTER_WR_MACI(f->fs.val.macidx) |
V_FW_FILTER_WR_MACIM(f->fs.mask.macidx) |
@@ -3318,10 +4790,10 @@ set_filter_wr(struct adapter *sc, int fidx)
fwr->ptclm = f->fs.mask.proto;
fwr->ttyp = f->fs.val.tos;
fwr->ttypm = f->fs.mask.tos;
- fwr->ivlan = htobe16(f->fs.val.ivlan);
- fwr->ivlanm = htobe16(f->fs.mask.ivlan);
- fwr->ovlan = htobe16(f->fs.val.ovlan);
- fwr->ovlanm = htobe16(f->fs.mask.ovlan);
+ fwr->ivlan = htobe16(f->fs.val.vlan);
+ fwr->ivlanm = htobe16(f->fs.mask.vlan);
+ fwr->ovlan = htobe16(f->fs.val.vnic);
+ fwr->ovlanm = htobe16(f->fs.mask.vnic);
bcopy(f->fs.val.dip, fwr->lip, sizeof (fwr->lip));
bcopy(f->fs.mask.dip, fwr->lipm, sizeof (fwr->lipm));
bcopy(f->fs.val.sip, fwr->fip, sizeof (fwr->fip));
@@ -3335,13 +4807,9 @@ set_filter_wr(struct adapter *sc, int fidx)
f->pending = 1;
sc->tids.ftids_in_use++;
- rc = t4_mgmt_tx(sc, m);
- if (rc != 0) {
- sc->tids.ftids_in_use--;
- m_freem(m);
- clear_filter(f);
- }
- return (rc);
+
+ t4_mgmt_tx(sc, m);
+ return (0);
}
static int
@@ -3350,7 +4818,7 @@ del_filter_wr(struct adapter *sc, int fidx)
struct filter_entry *f = &sc->tids.ftid_tab[fidx];
struct mbuf *m;
struct fw_filter_wr *fwr;
- unsigned int rc, ftid;
+ unsigned int ftid;
ADAPTER_LOCK_ASSERT_OWNED(sc);
@@ -3364,55 +4832,48 @@ del_filter_wr(struct adapter *sc, int fidx)
m->m_len = m->m_pkthdr.len = sizeof(*fwr);
bzero(fwr, sizeof (*fwr));
- t4_mk_filtdelwr(ftid, fwr, sc->sge.intrq[0].abs_id);
+ t4_mk_filtdelwr(ftid, fwr, sc->sge.fwq.abs_id);
f->pending = 1;
- rc = t4_mgmt_tx(sc, m);
- if (rc != 0) {
- f->pending = 0;
- m_freem(m);
- }
- return (rc);
+ t4_mgmt_tx(sc, m);
+ return (0);
}
-/* XXX move intr handlers to main.c and make this static */
-void
-filter_rpl(struct adapter *sc, const struct cpl_set_tcb_rpl *rpl)
+static int
+filter_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
{
+ struct adapter *sc = iq->adapter;
+ const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
unsigned int idx = GET_TID(rpl);
+ KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
+ rss->opcode));
+
if (idx >= sc->tids.ftid_base &&
(idx -= sc->tids.ftid_base) < sc->tids.nftids) {
unsigned int rc = G_COOKIE(rpl->cookie);
struct filter_entry *f = &sc->tids.ftid_tab[idx];
- if (rc == FW_FILTER_WR_FLT_DELETED) {
- /*
- * Clear the filter when we get confirmation from the
- * hardware that the filter has been deleted.
- */
- clear_filter(f);
- sc->tids.ftids_in_use--;
- } else if (rc == FW_FILTER_WR_SMT_TBL_FULL) {
- device_printf(sc->dev,
- "filter %u setup failed due to full SMT\n", idx);
- clear_filter(f);
- sc->tids.ftids_in_use--;
- } else if (rc == FW_FILTER_WR_FLT_ADDED) {
+ if (rc == FW_FILTER_WR_FLT_ADDED) {
f->smtidx = (be64toh(rpl->oldval) >> 24) & 0xff;
f->pending = 0; /* asynchronous setup completed */
f->valid = 1;
- } else {
- /*
- * Something went wrong. Issue a warning about the
- * problem and clear everything out.
- */
+ return (0);
+ }
+
+ if (rc != FW_FILTER_WR_FLT_DELETED) {
+ /* Add or delete failed, need to display an error */
device_printf(sc->dev,
"filter %u setup failed with error %u\n", idx, rc);
- clear_filter(f);
- sc->tids.ftids_in_use--;
}
+
+ clear_filter(f);
+ ADAPTER_LOCK(sc);
+ sc->tids.ftids_in_use--;
+ ADAPTER_UNLOCK(sc);
}
+
+ return (0);
}
static int
@@ -3444,6 +4905,84 @@ get_sge_context(struct adapter *sc, struct t4_sge_context *cntxt)
return (rc);
}
+static int
+read_card_mem(struct adapter *sc, struct t4_mem_range *mr)
+{
+ uint32_t base, size, lo, hi, win, off, remaining, i, n;
+ uint32_t *buf, *b;
+ int rc;
+
+ /* reads are in multiples of 32 bits */
+ if (mr->addr & 3 || mr->len & 3 || mr->len == 0)
+ return (EINVAL);
+
+ /*
+ * We don't want to deal with potential holes so we mandate that the
+ * requested region must lie entirely within one of the 3 memories.
+ */
+ lo = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE);
+ if (lo & F_EDRAM0_ENABLE) {
+ hi = t4_read_reg(sc, A_MA_EDRAM0_BAR);
+ base = G_EDRAM0_BASE(hi) << 20;
+ size = G_EDRAM0_SIZE(hi) << 20;
+ if (size > 0 &&
+ mr->addr >= base && mr->addr < base + size &&
+ mr->addr + mr->len <= base + size)
+ goto proceed;
+ }
+ if (lo & F_EDRAM1_ENABLE) {
+ hi = t4_read_reg(sc, A_MA_EDRAM1_BAR);
+ base = G_EDRAM1_BASE(hi) << 20;
+ size = G_EDRAM1_SIZE(hi) << 20;
+ if (size > 0 &&
+ mr->addr >= base && mr->addr < base + size &&
+ mr->addr + mr->len <= base + size)
+ goto proceed;
+ }
+ if (lo & F_EXT_MEM_ENABLE) {
+ hi = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR);
+ base = G_EXT_MEM_BASE(hi) << 20;
+ size = G_EXT_MEM_SIZE(hi) << 20;
+ if (size > 0 &&
+ mr->addr >= base && mr->addr < base + size &&
+ mr->addr + mr->len <= base + size)
+ goto proceed;
+ }
+ return (ENXIO);
+
+proceed:
+ buf = b = malloc(mr->len, M_CXGBE, M_WAITOK);
+
+ /*
+ * Position the PCIe window (we use memwin2) to the 16B aligned area
+ * just at/before the requested region.
+ */
+ win = mr->addr & ~0xf;
+ off = mr->addr - win; /* offset of the requested region in the win */
+ remaining = mr->len;
+
+ while (remaining) {
+ t4_write_reg(sc,
+ PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2), win);
+ t4_read_reg(sc,
+ PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, 2));
+
+ /* number of bytes that we'll copy in the inner loop */
+ n = min(remaining, MEMWIN2_APERTURE - off);
+
+ for (i = 0; i < n; i += 4, remaining -= 4)
+ *b++ = t4_read_reg(sc, MEMWIN2_BASE + off + i);
+
+ win += MEMWIN2_APERTURE;
+ off = 0;
+ }
+
+ rc = copyout(buf, mr->data, mr->len);
+ free(buf, M_CXGBE);
+
+ return (rc);
+}
+
int
t4_os_find_pci_capability(struct adapter *sc, int cap)
{
@@ -3514,6 +5053,22 @@ t4_os_link_changed(struct adapter *sc, int idx, int link_stat)
if_link_state_change(ifp, LINK_STATE_DOWN);
}
+void
+t4_iterate(void (*func)(struct adapter *, void *), void *arg)
+{
+ struct adapter *sc;
+
+ mtx_lock(&t4_list_lock);
+ SLIST_FOREACH(sc, &t4_list, link) {
+ /*
+ * func should not make any assumptions about what state sc is
+ * in - the only guarantee is that sc->sc_lock is a valid lock.
+ */
+ func(sc, arg);
+ }
+ mtx_unlock(&t4_list_lock);
+}
+
static int
t4_open(struct cdev *dev, int flags, int type, struct thread *td)
{
@@ -3610,6 +5165,27 @@ t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
case CHELSIO_T4_GET_SGE_CONTEXT:
rc = get_sge_context(sc, (struct t4_sge_context *)data);
break;
+ case CHELSIO_T4_LOAD_FW: {
+ struct t4_data *fw = (struct t4_data *)data;
+ uint8_t *fw_data;
+
+ if (sc->flags & FULL_INIT_DONE)
+ return (EBUSY);
+
+ fw_data = malloc(fw->len, M_CXGBE, M_NOWAIT);
+ if (fw_data == NULL)
+ return (ENOMEM);
+
+ rc = copyin(fw->data, fw_data, fw->len);
+ if (rc == 0)
+ rc = -t4_load_fw(sc, fw_data, fw->len);
+
+ free(fw_data, M_CXGBE);
+ break;
+ }
+ case CHELSIO_T4_GET_MEM:
+ rc = read_card_mem(sc, (struct t4_mem_range *)data);
+ break;
default:
rc = EINVAL;
}
@@ -3617,14 +5193,247 @@ t4_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
return (rc);
}
+#ifndef TCP_OFFLOAD_DISABLE
+static int
+toe_capability(struct port_info *pi, int enable)
+{
+ int rc;
+ struct adapter *sc = pi->adapter;
+
+ ADAPTER_LOCK_ASSERT_OWNED(sc);
+
+ if (!is_offload(sc))
+ return (ENODEV);
+
+ if (enable) {
+ if (isset(&sc->offload_map, pi->port_id))
+ return (0);
+
+ if (sc->offload_map == 0) {
+ rc = activate_uld(sc, ULD_TOM, &sc->tom);
+ if (rc != 0)
+ return (rc);
+ }
+
+ setbit(&sc->offload_map, pi->port_id);
+ } else {
+ if (!isset(&sc->offload_map, pi->port_id))
+ return (0);
+
+ clrbit(&sc->offload_map, pi->port_id);
+
+ if (sc->offload_map == 0) {
+ rc = deactivate_uld(&sc->tom);
+ if (rc != 0) {
+ setbit(&sc->offload_map, pi->port_id);
+ return (rc);
+ }
+ }
+ }
+
+ return (0);
+}
+
+/*
+ * Add an upper layer driver to the global list.
+ */
+int
+t4_register_uld(struct uld_info *ui)
+{
+ int rc = 0;
+ struct uld_info *u;
+
+ mtx_lock(&t4_uld_list_lock);
+ SLIST_FOREACH(u, &t4_uld_list, link) {
+ if (u->uld_id == ui->uld_id) {
+ rc = EEXIST;
+ goto done;
+ }
+ }
+
+ SLIST_INSERT_HEAD(&t4_uld_list, ui, link);
+ ui->refcount = 0;
+done:
+ mtx_unlock(&t4_uld_list_lock);
+ return (rc);
+}
+
+int
+t4_unregister_uld(struct uld_info *ui)
+{
+ int rc = EINVAL;
+ struct uld_info *u;
+
+ mtx_lock(&t4_uld_list_lock);
+
+ SLIST_FOREACH(u, &t4_uld_list, link) {
+ if (u == ui) {
+ if (ui->refcount > 0) {
+ rc = EBUSY;
+ goto done;
+ }
+
+ SLIST_REMOVE(&t4_uld_list, ui, uld_info, link);
+ rc = 0;
+ goto done;
+ }
+ }
+done:
+ mtx_unlock(&t4_uld_list_lock);
+ return (rc);
+}
+
+static int
+activate_uld(struct adapter *sc, int id, struct uld_softc *usc)
+{
+ int rc = EAGAIN;
+ struct uld_info *ui;
+
+ mtx_lock(&t4_uld_list_lock);
+
+ SLIST_FOREACH(ui, &t4_uld_list, link) {
+ if (ui->uld_id == id) {
+ rc = ui->attach(sc, &usc->softc);
+ if (rc == 0) {
+ KASSERT(usc->softc != NULL,
+ ("%s: ULD %d has no state", __func__, id));
+ ui->refcount++;
+ usc->uld = ui;
+ }
+ goto done;
+ }
+ }
+done:
+ mtx_unlock(&t4_uld_list_lock);
+
+ return (rc);
+}
+
+static int
+deactivate_uld(struct uld_softc *usc)
+{
+ int rc;
+
+ mtx_lock(&t4_uld_list_lock);
+
+ if (usc->uld == NULL || usc->softc == NULL) {
+ rc = EINVAL;
+ goto done;
+ }
+
+ rc = usc->uld->detach(usc->softc);
+ if (rc == 0) {
+ KASSERT(usc->uld->refcount > 0,
+ ("%s: ULD has bad refcount", __func__));
+ usc->uld->refcount--;
+ usc->uld = NULL;
+ usc->softc = NULL;
+ }
+done:
+ mtx_unlock(&t4_uld_list_lock);
+
+ return (rc);
+}
+#endif
+
+/*
+ * Come up with reasonable defaults for some of the tunables, provided they're
+ * not set by the user (in which case we'll use the values as is).
+ */
+static void
+tweak_tunables(void)
+{
+ int nc = mp_ncpus; /* our snapshot of the number of CPUs */
+
+ if (t4_ntxq10g < 1)
+ t4_ntxq10g = min(nc, NTXQ_10G);
+
+ if (t4_ntxq1g < 1)
+ t4_ntxq1g = min(nc, NTXQ_1G);
+
+ if (t4_nrxq10g < 1)
+ t4_nrxq10g = min(nc, NRXQ_10G);
+
+ if (t4_nrxq1g < 1)
+ t4_nrxq1g = min(nc, NRXQ_1G);
+
+#ifndef TCP_OFFLOAD_DISABLE
+ if (t4_nofldtxq10g < 1)
+ t4_nofldtxq10g = min(nc, NOFLDTXQ_10G);
+
+ if (t4_nofldtxq1g < 1)
+ t4_nofldtxq1g = min(nc, NOFLDTXQ_1G);
+
+ if (t4_nofldrxq10g < 1)
+ t4_nofldrxq10g = min(nc, NOFLDRXQ_10G);
+
+ if (t4_nofldrxq1g < 1)
+ t4_nofldrxq1g = min(nc, NOFLDRXQ_1G);
+#endif
+
+ if (t4_tmr_idx_10g < 0 || t4_tmr_idx_10g >= SGE_NTIMERS)
+ t4_tmr_idx_10g = TMR_IDX_10G;
+
+ if (t4_pktc_idx_10g < -1 || t4_pktc_idx_10g >= SGE_NCOUNTERS)
+ t4_pktc_idx_10g = PKTC_IDX_10G;
+
+ if (t4_tmr_idx_1g < 0 || t4_tmr_idx_1g >= SGE_NTIMERS)
+ t4_tmr_idx_1g = TMR_IDX_1G;
+
+ if (t4_pktc_idx_1g < -1 || t4_pktc_idx_1g >= SGE_NCOUNTERS)
+ t4_pktc_idx_1g = PKTC_IDX_1G;
+
+ if (t4_qsize_txq < 128)
+ t4_qsize_txq = 128;
+
+ if (t4_qsize_rxq < 128)
+ t4_qsize_rxq = 128;
+ while (t4_qsize_rxq & 7)
+ t4_qsize_rxq++;
+
+ t4_intr_types &= INTR_MSIX | INTR_MSI | INTR_INTX;
+}
+
static int
t4_mod_event(module_t mod, int cmd, void *arg)
{
+ int rc = 0;
- if (cmd == MOD_LOAD)
+ switch (cmd) {
+ case MOD_LOAD:
t4_sge_modload();
+ mtx_init(&t4_list_lock, "T4 adapters", 0, MTX_DEF);
+ SLIST_INIT(&t4_list);
+#ifndef TCP_OFFLOAD_DISABLE
+ mtx_init(&t4_uld_list_lock, "T4 ULDs", 0, MTX_DEF);
+ SLIST_INIT(&t4_uld_list);
+#endif
+ tweak_tunables();
+ break;
- return (0);
+ case MOD_UNLOAD:
+#ifndef TCP_OFFLOAD_DISABLE
+ mtx_lock(&t4_uld_list_lock);
+ if (!SLIST_EMPTY(&t4_uld_list)) {
+ rc = EBUSY;
+ mtx_unlock(&t4_uld_list_lock);
+ break;
+ }
+ mtx_unlock(&t4_uld_list_lock);
+ mtx_destroy(&t4_uld_list_lock);
+#endif
+ mtx_lock(&t4_list_lock);
+ if (!SLIST_EMPTY(&t4_list)) {
+ rc = EBUSY;
+ mtx_unlock(&t4_list_lock);
+ break;
+ }
+ mtx_unlock(&t4_list_lock);
+ mtx_destroy(&t4_list_lock);
+ break;
+ }
+
+ return (rc);
}
static devclass_t t4_devclass;
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index 09e3c19..41630e4 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
#include <sys/queue.h>
#include <sys/taskqueue.h>
#include <sys/sysctl.h>
+#include <sys/smp.h>
#include <net/bpf.h>
#include <net/ethernet.h>
#include <net/if.h>
@@ -50,7 +51,7 @@ __FBSDID("$FreeBSD$");
#include "common/t4_regs.h"
#include "common/t4_regs_values.h"
#include "common/t4_msg.h"
-#include "common/t4fw_interface.h"
+#include "t4_l2t.h"
struct fl_buf_info {
int size;
@@ -91,12 +92,15 @@ struct sgl {
bus_dma_segment_t seg[TX_SGL_SEGS];
};
-static void t4_evt_rx(void *);
-static void t4_eth_rx(void *);
+static int service_iq(struct sge_iq *, int);
+static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, uint32_t,
+ int *);
+static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf *);
static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int,
- int, iq_intr_handler_t *, char *);
-static inline void init_fl(struct sge_fl *, int, char *);
-static inline void init_eq(struct sge_eq *, int, char *);
+ int, char *);
+static inline void init_fl(struct sge_fl *, int, int, char *);
+static inline void init_eq(struct sge_eq *, int, int, uint8_t, uint16_t,
+ char *);
static int alloc_ring(struct adapter *, size_t, bus_dma_tag_t *, bus_dmamap_t *,
bus_addr_t *, void **);
static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
@@ -104,26 +108,41 @@ static int free_ring(struct adapter *, bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
static int alloc_iq_fl(struct port_info *, struct sge_iq *, struct sge_fl *,
int, int);
static int free_iq_fl(struct port_info *, struct sge_iq *, struct sge_fl *);
-static int alloc_intrq(struct adapter *, int, int, int);
-static int free_intrq(struct sge_iq *);
-static int alloc_fwq(struct adapter *, int);
-static int free_fwq(struct sge_iq *);
-static int alloc_rxq(struct port_info *, struct sge_rxq *, int, int);
+static int alloc_fwq(struct adapter *);
+static int free_fwq(struct adapter *);
+static int alloc_mgmtq(struct adapter *);
+static int free_mgmtq(struct adapter *);
+static int alloc_rxq(struct port_info *, struct sge_rxq *, int, int,
+ struct sysctl_oid *);
static int free_rxq(struct port_info *, struct sge_rxq *);
-static int alloc_ctrlq(struct adapter *, struct sge_ctrlq *, int);
-static int free_ctrlq(struct adapter *, struct sge_ctrlq *);
-static int alloc_txq(struct port_info *, struct sge_txq *, int);
+#ifndef TCP_OFFLOAD_DISABLE
+static int alloc_ofld_rxq(struct port_info *, struct sge_ofld_rxq *, int, int,
+ struct sysctl_oid *);
+static int free_ofld_rxq(struct port_info *, struct sge_ofld_rxq *);
+#endif
+static int ctrl_eq_alloc(struct adapter *, struct sge_eq *);
+static int eth_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
+#ifndef TCP_OFFLOAD_DISABLE
+static int ofld_eq_alloc(struct adapter *, struct port_info *, struct sge_eq *);
+#endif
+static int alloc_eq(struct adapter *, struct port_info *, struct sge_eq *);
+static int free_eq(struct adapter *, struct sge_eq *);
+static int alloc_wrq(struct adapter *, struct port_info *, struct sge_wrq *,
+ struct sysctl_oid *);
+static int free_wrq(struct adapter *, struct sge_wrq *);
+static int alloc_txq(struct port_info *, struct sge_txq *, int,
+ struct sysctl_oid *);
static int free_txq(struct port_info *, struct sge_txq *);
static void oneseg_dma_callback(void *, bus_dma_segment_t *, int, int);
static inline bool is_new_response(const struct sge_iq *, struct rsp_ctrl **);
static inline void iq_next(struct sge_iq *);
static inline void ring_fl_db(struct adapter *, struct sge_fl *);
-static void refill_fl(struct adapter *, struct sge_fl *, int, int);
+static int refill_fl(struct adapter *, struct sge_fl *, int);
+static void refill_sfl(void *);
static int alloc_fl_sdesc(struct sge_fl *);
static void free_fl_sdesc(struct sge_fl *);
-static int alloc_tx_maps(struct sge_txq *);
-static void free_tx_maps(struct sge_txq *);
static void set_fl_tag_idx(struct sge_fl *, int);
+static void add_fl_to_sfl(struct adapter *, struct sge_fl *);
static int get_pkt_sgl(struct sge_txq *, struct mbuf **, struct sgl *, int);
static int free_pkt_sgl(struct sge_txq *, struct sgl *);
@@ -141,15 +160,13 @@ static inline int reclaimable(struct sge_eq *);
static int reclaim_tx_descs(struct sge_txq *, int, int);
static void write_eqflush_wr(struct sge_eq *);
static __be64 get_flit(bus_dma_segment_t *, int, int);
-static int handle_sge_egr_update(struct adapter *,
- const struct cpl_sge_egr_update *);
-static void handle_cpl(struct adapter *, struct sge_iq *);
+static int handle_sge_egr_update(struct sge_iq *, const struct rss_header *,
+ struct mbuf *);
+static int handle_fw_rpl(struct sge_iq *, const struct rss_header *,
+ struct mbuf *);
-static int ctrl_tx(struct adapter *, struct sge_ctrlq *, struct mbuf *);
static int sysctl_uint16(SYSCTL_HANDLER_ARGS);
-extern void filter_rpl(struct adapter *, const struct cpl_set_tcb_rpl *);
-
/*
* Called on MOD_LOAD and fills up fl_buf_info[].
*/
@@ -181,46 +198,103 @@ t4_sge_modload(void)
* We do not initialize any of the queues here, instead the driver
* top-level must request them individually.
*/
-void
+int
t4_sge_init(struct adapter *sc)
{
struct sge *s = &sc->sge;
- int i;
+ int i, rc = 0;
+ uint32_t ctrl_mask, ctrl_val, hpsize, v;
+
+ ctrl_mask = V_PKTSHIFT(M_PKTSHIFT) | F_RXPKTCPLMODE |
+ V_INGPADBOUNDARY(M_INGPADBOUNDARY) |
+ F_EGRSTATUSPAGESIZE;
+ ctrl_val = V_PKTSHIFT(FL_PKTSHIFT) | F_RXPKTCPLMODE |
+ V_INGPADBOUNDARY(ilog2(FL_ALIGN) - 5) |
+ V_EGRSTATUSPAGESIZE(SPG_LEN == 128);
+
+ hpsize = V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10) |
+ V_HOSTPAGESIZEPF1(PAGE_SHIFT - 10) |
+ V_HOSTPAGESIZEPF2(PAGE_SHIFT - 10) |
+ V_HOSTPAGESIZEPF3(PAGE_SHIFT - 10) |
+ V_HOSTPAGESIZEPF4(PAGE_SHIFT - 10) |
+ V_HOSTPAGESIZEPF5(PAGE_SHIFT - 10) |
+ V_HOSTPAGESIZEPF6(PAGE_SHIFT - 10) |
+ V_HOSTPAGESIZEPF7(PAGE_SHIFT - 10);
+
+ if (sc->flags & MASTER_PF) {
+ int intr_timer[SGE_NTIMERS] = {1, 5, 10, 50, 100, 200};
+ int intr_pktcount[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */
+
+ t4_set_reg_field(sc, A_SGE_CONTROL, ctrl_mask, ctrl_val);
+ t4_write_reg(sc, A_SGE_HOST_PAGE_SIZE, hpsize);
+ for (i = 0; i < FL_BUF_SIZES; i++) {
+ t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i),
+ FL_BUF_SIZE(i));
+ }
+
+ t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
+ V_THRESHOLD_0(intr_pktcount[0]) |
+ V_THRESHOLD_1(intr_pktcount[1]) |
+ V_THRESHOLD_2(intr_pktcount[2]) |
+ V_THRESHOLD_3(intr_pktcount[3]));
+
+ t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
+ V_TIMERVALUE0(us_to_core_ticks(sc, intr_timer[0])) |
+ V_TIMERVALUE1(us_to_core_ticks(sc, intr_timer[1])));
+ t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
+ V_TIMERVALUE2(us_to_core_ticks(sc, intr_timer[2])) |
+ V_TIMERVALUE3(us_to_core_ticks(sc, intr_timer[3])));
+ t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
+ V_TIMERVALUE4(us_to_core_ticks(sc, intr_timer[4])) |
+ V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5])));
+ }
+
+ v = t4_read_reg(sc, A_SGE_CONTROL);
+ if ((v & ctrl_mask) != ctrl_val) {
+ device_printf(sc->dev, "invalid SGE_CONTROL(0x%x)\n", v);
+ rc = EINVAL;
+ }
- t4_set_reg_field(sc, A_SGE_CONTROL, V_PKTSHIFT(M_PKTSHIFT) |
- V_INGPADBOUNDARY(M_INGPADBOUNDARY) |
- F_EGRSTATUSPAGESIZE,
- V_INGPADBOUNDARY(ilog2(FL_ALIGN) - 5) |
- V_PKTSHIFT(FL_PKTSHIFT) |
- F_RXPKTCPLMODE |
- V_EGRSTATUSPAGESIZE(SPG_LEN == 128));
- t4_set_reg_field(sc, A_SGE_HOST_PAGE_SIZE,
- V_HOSTPAGESIZEPF0(M_HOSTPAGESIZEPF0),
- V_HOSTPAGESIZEPF0(PAGE_SHIFT - 10));
+ v = t4_read_reg(sc, A_SGE_HOST_PAGE_SIZE);
+ if (v != hpsize) {
+ device_printf(sc->dev, "invalid SGE_HOST_PAGE_SIZE(0x%x)\n", v);
+ rc = EINVAL;
+ }
for (i = 0; i < FL_BUF_SIZES; i++) {
- t4_write_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i),
- FL_BUF_SIZE(i));
+ v = t4_read_reg(sc, A_SGE_FL_BUFFER_SIZE0 + (4 * i));
+ if (v != FL_BUF_SIZE(i)) {
+ device_printf(sc->dev,
+ "invalid SGE_FL_BUFFER_SIZE[%d](0x%x)\n", i, v);
+ rc = EINVAL;
+ }
}
- i = t4_read_reg(sc, A_SGE_CONM_CTRL);
- s->fl_starve_threshold = G_EGRTHRESHOLD(i) * 2 + 1;
+ v = t4_read_reg(sc, A_SGE_CONM_CTRL);
+ s->fl_starve_threshold = G_EGRTHRESHOLD(v) * 2 + 1;
+
+ v = t4_read_reg(sc, A_SGE_INGRESS_RX_THRESHOLD);
+ sc->sge.counter_val[0] = G_THRESHOLD_0(v);
+ sc->sge.counter_val[1] = G_THRESHOLD_1(v);
+ sc->sge.counter_val[2] = G_THRESHOLD_2(v);
+ sc->sge.counter_val[3] = G_THRESHOLD_3(v);
+
+ v = t4_read_reg(sc, A_SGE_TIMER_VALUE_0_AND_1);
+ sc->sge.timer_val[0] = G_TIMERVALUE0(v) / core_ticks_per_usec(sc);
+ sc->sge.timer_val[1] = G_TIMERVALUE1(v) / core_ticks_per_usec(sc);
+ v = t4_read_reg(sc, A_SGE_TIMER_VALUE_2_AND_3);
+ sc->sge.timer_val[2] = G_TIMERVALUE2(v) / core_ticks_per_usec(sc);
+ sc->sge.timer_val[3] = G_TIMERVALUE3(v) / core_ticks_per_usec(sc);
+ v = t4_read_reg(sc, A_SGE_TIMER_VALUE_4_AND_5);
+ sc->sge.timer_val[4] = G_TIMERVALUE4(v) / core_ticks_per_usec(sc);
+ sc->sge.timer_val[5] = G_TIMERVALUE5(v) / core_ticks_per_usec(sc);
- t4_write_reg(sc, A_SGE_INGRESS_RX_THRESHOLD,
- V_THRESHOLD_0(s->counter_val[0]) |
- V_THRESHOLD_1(s->counter_val[1]) |
- V_THRESHOLD_2(s->counter_val[2]) |
- V_THRESHOLD_3(s->counter_val[3]));
+ t4_register_cpl_handler(sc, CPL_FW4_MSG, handle_fw_rpl);
+ t4_register_cpl_handler(sc, CPL_FW6_MSG, handle_fw_rpl);
+ t4_register_cpl_handler(sc, CPL_SGE_EGR_UPDATE, handle_sge_egr_update);
+ t4_register_cpl_handler(sc, CPL_RX_PKT, t4_eth_rx);
- t4_write_reg(sc, A_SGE_TIMER_VALUE_0_AND_1,
- V_TIMERVALUE0(us_to_core_ticks(sc, s->timer_val[0])) |
- V_TIMERVALUE1(us_to_core_ticks(sc, s->timer_val[1])));
- t4_write_reg(sc, A_SGE_TIMER_VALUE_2_AND_3,
- V_TIMERVALUE2(us_to_core_ticks(sc, s->timer_val[2])) |
- V_TIMERVALUE3(us_to_core_ticks(sc, s->timer_val[3])));
- t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5,
- V_TIMERVALUE4(us_to_core_ticks(sc, s->timer_val[4])) |
- V_TIMERVALUE5(us_to_core_ticks(sc, s->timer_val[5])));
+ return (rc);
}
int
@@ -250,8 +324,7 @@ t4_destroy_dma_tag(struct adapter *sc)
}
/*
- * Allocate and initialize the firmware event queue, control queues, and the
- * interrupt queues. The adapter owns all of these queues.
+ * Allocate and initialize the firmware event queue and the management queue.
*
* Returns errno on failure. Resources allocated up to that point may still be
* allocated. Caller is responsible for cleanup in case this function fails.
@@ -259,110 +332,32 @@ t4_destroy_dma_tag(struct adapter *sc)
int
t4_setup_adapter_queues(struct adapter *sc)
{
- int i, j, rc, intr_idx, qsize;
- struct sge_iq *iq;
- struct sge_ctrlq *ctrlq;
- iq_intr_handler_t *handler;
- char name[16];
+ int rc;
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
- if (sysctl_ctx_init(&sc->ctx) == 0) {
- struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
- struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
-
- sc->oid_fwq = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO,
- "fwq", CTLFLAG_RD, NULL, "firmware event queue");
- sc->oid_ctrlq = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO,
- "ctrlq", CTLFLAG_RD, NULL, "ctrl queues");
- sc->oid_intrq = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO,
- "intrq", CTLFLAG_RD, NULL, "interrupt queues");
- }
-
- /*
- * Interrupt queues
- */
- intr_idx = sc->intr_count - NINTRQ(sc);
- if (sc->flags & INTR_SHARED) {
- qsize = max((sc->sge.nrxq + 1) * 2, INTR_IQ_QSIZE);
- for (i = 0; i < NINTRQ(sc); i++, intr_idx++) {
- snprintf(name, sizeof(name), "%s intrq%d",
- device_get_nameunit(sc->dev), i);
-
- iq = &sc->sge.intrq[i];
- init_iq(iq, sc, 0, 0, qsize, INTR_IQ_ESIZE, NULL, name);
- rc = alloc_intrq(sc, i % sc->params.nports, i,
- intr_idx);
-
- if (rc != 0) {
- device_printf(sc->dev,
- "failed to create %s: %d\n", name, rc);
- return (rc);
- }
- }
- } else {
- int qidx = 0;
- struct port_info *pi;
-
- for (i = 0; i < sc->params.nports; i++) {
- pi = sc->port[i];
- qsize = max((pi->nrxq + 1) * 2, INTR_IQ_QSIZE);
- for (j = 0; j < pi->nrxq; j++, qidx++, intr_idx++) {
- snprintf(name, sizeof(name), "%s intrq%d",
- device_get_nameunit(pi->dev), j);
-
- iq = &sc->sge.intrq[qidx];
- init_iq(iq, sc, 0, 0, qsize, INTR_IQ_ESIZE,
- NULL, name);
- rc = alloc_intrq(sc, i, qidx, intr_idx);
-
- if (rc != 0) {
- device_printf(sc->dev,
- "failed to create %s: %d\n",
- name, rc);
- return (rc);
- }
- }
- }
- }
+ sysctl_ctx_init(&sc->ctx);
+ sc->flags |= ADAP_SYSCTL_CTX;
/*
* Firmware event queue
*/
- snprintf(name, sizeof(name), "%s fwq", device_get_nameunit(sc->dev));
- if (sc->intr_count > T4_EXTRA_INTR) {
- handler = NULL;
- intr_idx = 1;
- } else {
- handler = t4_evt_rx;
- intr_idx = 0;
- }
-
- iq = &sc->sge.fwq;
- init_iq(iq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE, handler, name);
- rc = alloc_fwq(sc, intr_idx);
+ rc = alloc_fwq(sc);
if (rc != 0) {
device_printf(sc->dev,
"failed to create firmware event queue: %d\n", rc);
-
return (rc);
}
/*
- * Control queues - one per port.
+ * Management queue. This is just a control queue that uses the fwq as
+ * its associated iq.
*/
- ctrlq = &sc->sge.ctrlq[0];
- for (i = 0; i < sc->params.nports; i++, ctrlq++) {
- snprintf(name, sizeof(name), "%s ctrlq%d",
- device_get_nameunit(sc->dev), i);
- init_eq(&ctrlq->eq, CTRL_EQ_QSIZE, name);
-
- rc = alloc_ctrlq(sc, ctrlq, i);
- if (rc != 0) {
- device_printf(sc->dev,
- "failed to create control queue %d: %d\n", i, rc);
- return (rc);
- }
+ rc = alloc_mgmtq(sc);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to create management queue: %d\n", rc);
+ return (rc);
}
return (rc);
@@ -374,86 +369,278 @@ t4_setup_adapter_queues(struct adapter *sc)
int
t4_teardown_adapter_queues(struct adapter *sc)
{
- int i;
- struct sge_iq *iq;
ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
- /* Do this before freeing the queues */
- if (sc->oid_fwq || sc->oid_ctrlq || sc->oid_intrq) {
+ /* Do this before freeing the queue */
+ if (sc->flags & ADAP_SYSCTL_CTX) {
sysctl_ctx_free(&sc->ctx);
- sc->oid_fwq = NULL;
- sc->oid_ctrlq = NULL;
- sc->oid_intrq = NULL;
+ sc->flags &= ~ADAP_SYSCTL_CTX;
}
- for (i = 0; i < sc->params.nports; i++)
- free_ctrlq(sc, &sc->sge.ctrlq[i]);
+ free_mgmtq(sc);
+ free_fwq(sc);
+
+ return (0);
+}
+
+static inline int
+first_vector(struct port_info *pi)
+{
+ struct adapter *sc = pi->adapter;
+ int rc = T4_EXTRA_INTR, i;
+
+ if (sc->intr_count == 1)
+ return (0);
- iq = &sc->sge.fwq;
- free_fwq(iq);
+ for_each_port(sc, i) {
+ if (i == pi->port_id)
+ break;
+
+#ifndef TCP_OFFLOAD_DISABLE
+ if (sc->flags & INTR_DIRECT)
+ rc += pi->nrxq + pi->nofldrxq;
+ else
+ rc += max(pi->nrxq, pi->nofldrxq);
+#else
+ /*
+ * Not compiled with offload support and intr_count > 1. Only
+ * NIC queues exist and they'd better be taking direct
+ * interrupts.
+ */
+ KASSERT(sc->flags & INTR_DIRECT,
+ ("%s: intr_count %d, !INTR_DIRECT", __func__,
+ sc->intr_count));
- for (i = 0; i < NINTRQ(sc); i++) {
- iq = &sc->sge.intrq[i];
- free_intrq(iq);
+ rc += pi->nrxq;
+#endif
}
- return (0);
+ return (rc);
+}
+
+/*
+ * Given an arbitrary "index," come up with an iq that can be used by other
+ * queues (of this port) for interrupt forwarding, SGE egress updates, etc.
+ * The iq returned is guaranteed to be something that takes direct interrupts.
+ */
+static struct sge_iq *
+port_intr_iq(struct port_info *pi, int idx)
+{
+ struct adapter *sc = pi->adapter;
+ struct sge *s = &sc->sge;
+ struct sge_iq *iq = NULL;
+
+ if (sc->intr_count == 1)
+ return (&sc->sge.fwq);
+
+#ifndef TCP_OFFLOAD_DISABLE
+ if (sc->flags & INTR_DIRECT) {
+ idx %= pi->nrxq + pi->nofldrxq;
+
+ if (idx >= pi->nrxq) {
+ idx -= pi->nrxq;
+ iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
+ } else
+ iq = &s->rxq[pi->first_rxq + idx].iq;
+
+ } else {
+ idx %= max(pi->nrxq, pi->nofldrxq);
+
+ if (pi->nrxq >= pi->nofldrxq)
+ iq = &s->rxq[pi->first_rxq + idx].iq;
+ else
+ iq = &s->ofld_rxq[pi->first_ofld_rxq + idx].iq;
+ }
+#else
+ /*
+ * Not compiled with offload support and intr_count > 1. Only NIC
+ * queues exist and they'd better be taking direct interrupts.
+ */
+ KASSERT(sc->flags & INTR_DIRECT,
+ ("%s: intr_count %d, !INTR_DIRECT", __func__, sc->intr_count));
+
+ idx %= pi->nrxq;
+ iq = &s->rxq[pi->first_rxq + idx].iq;
+#endif
+
+ KASSERT(iq->flags & IQ_INTR, ("%s: EDOOFUS", __func__));
+ return (iq);
}
int
-t4_setup_eth_queues(struct port_info *pi)
+t4_setup_port_queues(struct port_info *pi)
{
- int rc = 0, i, intr_idx;
+ int rc = 0, i, j, intr_idx, iqid;
struct sge_rxq *rxq;
struct sge_txq *txq;
+ struct sge_wrq *ctrlq;
+#ifndef TCP_OFFLOAD_DISABLE
+ struct sge_ofld_rxq *ofld_rxq;
+ struct sge_wrq *ofld_txq;
+#endif
char name[16];
struct adapter *sc = pi->adapter;
+ struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev), *oid2 = NULL;
+ struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
- if (sysctl_ctx_init(&pi->ctx) == 0) {
- struct sysctl_oid *oid = device_get_sysctl_tree(pi->dev);
- struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
+ oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "rxq", CTLFLAG_RD,
+ NULL, "rx queues");
- pi->oid_rxq = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO,
- "rxq", CTLFLAG_RD, NULL, "rx queues");
- pi->oid_txq = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO,
- "txq", CTLFLAG_RD, NULL, "tx queues");
+#ifndef TCP_OFFLOAD_DISABLE
+ if (is_offload(sc)) {
+ oid2 = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_rxq",
+ CTLFLAG_RD, NULL,
+ "rx queues for offloaded TCP connections");
}
+#endif
+
+ /* Interrupt vector to start from (when using multiple vectors) */
+ intr_idx = first_vector(pi);
+ /*
+ * First pass over all rx queues (NIC and TOE):
+ * a) initialize iq and fl
+ * b) allocate queue iff it will take direct interrupts.
+ */
for_each_rxq(pi, i, rxq) {
snprintf(name, sizeof(name), "%s rxq%d-iq",
device_get_nameunit(pi->dev), i);
- init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
- pi->qsize_rxq, RX_IQ_ESIZE, t4_eth_rx, name);
+ init_iq(&rxq->iq, sc, pi->tmr_idx, pi->pktc_idx, pi->qsize_rxq,
+ RX_IQ_ESIZE, name);
snprintf(name, sizeof(name), "%s rxq%d-fl",
device_get_nameunit(pi->dev), i);
- init_fl(&rxq->fl, pi->qsize_rxq / 8, name);
+ init_fl(&rxq->fl, pi->qsize_rxq / 8, pi->ifp->if_mtu, name);
+
+ if (sc->flags & INTR_DIRECT
+#ifndef TCP_OFFLOAD_DISABLE
+ || (sc->intr_count > 1 && pi->nrxq >= pi->nofldrxq)
+#endif
+ ) {
+ rxq->iq.flags |= IQ_INTR;
+ rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
+ if (rc != 0)
+ goto done;
+ intr_idx++;
+ }
+ }
- intr_idx = pi->first_rxq + i;
- if (sc->flags & INTR_SHARED)
- intr_idx %= NINTRQ(sc);
+#ifndef TCP_OFFLOAD_DISABLE
+ for_each_ofld_rxq(pi, i, ofld_rxq) {
- rc = alloc_rxq(pi, rxq, intr_idx, i);
+ snprintf(name, sizeof(name), "%s ofld_rxq%d-iq",
+ device_get_nameunit(pi->dev), i);
+ init_iq(&ofld_rxq->iq, sc, pi->tmr_idx, pi->pktc_idx,
+ pi->qsize_rxq, RX_IQ_ESIZE, name);
+
+ snprintf(name, sizeof(name), "%s ofld_rxq%d-fl",
+ device_get_nameunit(pi->dev), i);
+ init_fl(&ofld_rxq->fl, pi->qsize_rxq / 8, MJUM16BYTES, name);
+
+ if (sc->flags & INTR_DIRECT ||
+ (sc->intr_count > 1 && pi->nofldrxq > pi->nrxq)) {
+ ofld_rxq->iq.flags |= IQ_INTR;
+ rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
+ if (rc != 0)
+ goto done;
+ intr_idx++;
+ }
+ }
+#endif
+
+ /*
+ * Second pass over all rx queues (NIC and TOE). The queues forwarding
+ * their interrupts are allocated now.
+ */
+ j = 0;
+ for_each_rxq(pi, i, rxq) {
+ if (rxq->iq.flags & IQ_INTR)
+ continue;
+
+ intr_idx = port_intr_iq(pi, j)->abs_id;
+
+ rc = alloc_rxq(pi, rxq, intr_idx, i, oid);
+ if (rc != 0)
+ goto done;
+ j++;
+ }
+
+#ifndef TCP_OFFLOAD_DISABLE
+ for_each_ofld_rxq(pi, i, ofld_rxq) {
+ if (ofld_rxq->iq.flags & IQ_INTR)
+ continue;
+
+ intr_idx = port_intr_iq(pi, j)->abs_id;
+
+ rc = alloc_ofld_rxq(pi, ofld_rxq, intr_idx, i, oid2);
if (rc != 0)
goto done;
+ j++;
}
+#endif
+ /*
+ * Now the tx queues. Only one pass needed.
+ */
+ oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "txq", CTLFLAG_RD,
+ NULL, "tx queues");
+ j = 0;
for_each_txq(pi, i, txq) {
+ uint16_t iqid;
+
+ iqid = port_intr_iq(pi, j)->cntxt_id;
snprintf(name, sizeof(name), "%s txq%d",
device_get_nameunit(pi->dev), i);
- init_eq(&txq->eq, pi->qsize_txq, name);
+ init_eq(&txq->eq, EQ_ETH, pi->qsize_txq, pi->tx_chan, iqid,
+ name);
- rc = alloc_txq(pi, txq, i);
+ rc = alloc_txq(pi, txq, i, oid);
if (rc != 0)
goto done;
+ j++;
}
+#ifndef TCP_OFFLOAD_DISABLE
+ oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ofld_txq",
+ CTLFLAG_RD, NULL, "tx queues for offloaded TCP connections");
+ for_each_ofld_txq(pi, i, ofld_txq) {
+ uint16_t iqid;
+
+ iqid = port_intr_iq(pi, j)->cntxt_id;
+
+ snprintf(name, sizeof(name), "%s ofld_txq%d",
+ device_get_nameunit(pi->dev), i);
+ init_eq(&ofld_txq->eq, EQ_OFLD, pi->qsize_txq, pi->tx_chan,
+ iqid, name);
+
+ snprintf(name, sizeof(name), "%d", i);
+ oid2 = SYSCTL_ADD_NODE(&pi->ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
+ name, CTLFLAG_RD, NULL, "offload tx queue");
+
+ rc = alloc_wrq(sc, pi, ofld_txq, oid2);
+ if (rc != 0)
+ goto done;
+ j++;
+ }
+#endif
+
+ /*
+ * Finally, the control queue.
+ */
+ oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "ctrlq", CTLFLAG_RD,
+ NULL, "ctrl queue");
+ ctrlq = &sc->sge.ctrlq[pi->port_id];
+ iqid = port_intr_iq(pi, 0)->cntxt_id;
+ snprintf(name, sizeof(name), "%s ctrlq", device_get_nameunit(pi->dev));
+ init_eq(&ctrlq->eq, EQ_CTRL, CTRL_EQ_QSIZE, pi->tx_chan, iqid, name);
+ rc = alloc_wrq(sc, pi, ctrlq, oid);
+
done:
if (rc)
- t4_teardown_eth_queues(pi);
+ t4_teardown_port_queues(pi);
return (rc);
}
@@ -462,90 +649,91 @@ done:
* Idempotent
*/
int
-t4_teardown_eth_queues(struct port_info *pi)
+t4_teardown_port_queues(struct port_info *pi)
{
int i;
+ struct adapter *sc = pi->adapter;
struct sge_rxq *rxq;
struct sge_txq *txq;
+#ifndef TCP_OFFLOAD_DISABLE
+ struct sge_ofld_rxq *ofld_rxq;
+ struct sge_wrq *ofld_txq;
+#endif
/* Do this before freeing the queues */
- if (pi->oid_txq || pi->oid_rxq) {
+ if (pi->flags & PORT_SYSCTL_CTX) {
sysctl_ctx_free(&pi->ctx);
- pi->oid_txq = pi->oid_rxq = NULL;
+ pi->flags &= ~PORT_SYSCTL_CTX;
}
+ /*
+ * Take down all the tx queues first, as they reference the rx queues
+ * (for egress updates, etc.).
+ */
+
+ free_wrq(sc, &sc->sge.ctrlq[pi->port_id]);
+
for_each_txq(pi, i, txq) {
free_txq(pi, txq);
}
+#ifndef TCP_OFFLOAD_DISABLE
+ for_each_ofld_txq(pi, i, ofld_txq) {
+ free_wrq(sc, ofld_txq);
+ }
+#endif
+
+ /*
+ * Then take down the rx queues that forward their interrupts, as they
+ * reference other rx queues.
+ */
+
+ for_each_rxq(pi, i, rxq) {
+ if ((rxq->iq.flags & IQ_INTR) == 0)
+ free_rxq(pi, rxq);
+ }
+
+#ifndef TCP_OFFLOAD_DISABLE
+ for_each_ofld_rxq(pi, i, ofld_rxq) {
+ if ((ofld_rxq->iq.flags & IQ_INTR) == 0)
+ free_ofld_rxq(pi, ofld_rxq);
+ }
+#endif
+
+ /*
+ * Then take down the rx queues that take direct interrupts.
+ */
+
for_each_rxq(pi, i, rxq) {
- free_rxq(pi, rxq);
+ if (rxq->iq.flags & IQ_INTR)
+ free_rxq(pi, rxq);
}
+#ifndef TCP_OFFLOAD_DISABLE
+ for_each_ofld_rxq(pi, i, ofld_rxq) {
+ if (ofld_rxq->iq.flags & IQ_INTR)
+ free_ofld_rxq(pi, ofld_rxq);
+ }
+#endif
+
return (0);
}
-/* Deals with errors and the first (and only) interrupt queue */
+/*
+ * Deals with errors and the firmware event queue. All data rx queues forward
+ * their interrupt to the firmware event queue.
+ */
void
t4_intr_all(void *arg)
{
struct adapter *sc = arg;
+ struct sge_iq *fwq = &sc->sge.fwq;
t4_intr_err(arg);
- t4_intr(&sc->sge.intrq[0]);
-}
-
-/* Deals with interrupts, and a few CPLs, on the given interrupt queue */
-void
-t4_intr(void *arg)
-{
- struct sge_iq *iq = arg, *q;
- struct adapter *sc = iq->adapter;
- struct rsp_ctrl *ctrl;
- const struct rss_header *rss;
- int ndesc_pending = 0, ndesc_total = 0;
- int qid, rsp_type;
-
- if (!atomic_cmpset_32(&iq->state, IQS_IDLE, IQS_BUSY))
- return;
-
- while (is_new_response(iq, &ctrl)) {
-
- rmb();
-
- rss = (const void *)iq->cdesc;
- rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
-
- if (__predict_false(rsp_type == X_RSPD_TYPE_CPL)) {
- handle_cpl(sc, iq);
- goto nextdesc;
- }
-
- qid = ntohl(ctrl->pldbuflen_qid) - sc->sge.iq_start;
- q = sc->sge.iqmap[qid];
-
- if (atomic_cmpset_32(&q->state, IQS_IDLE, IQS_BUSY)) {
- q->handler(q);
- atomic_cmpset_32(&q->state, IQS_BUSY, IQS_IDLE);
- }
-
-nextdesc: ndesc_total++;
- if (++ndesc_pending >= iq->qsize / 4) {
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
- V_CIDXINC(ndesc_pending) |
- V_INGRESSQID(iq->cntxt_id) |
- V_SEINTARM(
- V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
- ndesc_pending = 0;
- }
-
- iq_next(iq);
+ if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) {
+ service_iq(fwq, 0);
+ atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE);
}
-
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndesc_pending) |
- V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
-
- atomic_cmpset_32(&iq->state, IQS_BUSY, IQS_IDLE);
}
/* Deals with error interrupts */
@@ -558,259 +746,452 @@ t4_intr_err(void *arg)
t4_slow_intr_handler(sc);
}
-/* Deals with the firmware event queue */
void
t4_intr_evt(void *arg)
{
struct sge_iq *iq = arg;
- if (atomic_cmpset_32(&iq->state, IQS_IDLE, IQS_BUSY)) {
- t4_evt_rx(arg);
- atomic_cmpset_32(&iq->state, IQS_BUSY, IQS_IDLE);
+ if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
+ service_iq(iq, 0);
+ atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
}
}
-static void
-t4_evt_rx(void *arg)
+void
+t4_intr(void *arg)
{
struct sge_iq *iq = arg;
+
+ if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
+ service_iq(iq, 0);
+ atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
+ }
+}
+
+/*
+ * Deals with anything and everything on the given ingress queue.
+ */
+static int
+service_iq(struct sge_iq *iq, int budget)
+{
+ struct sge_iq *q;
+ struct sge_rxq *rxq = (void *)iq; /* Use iff iq is part of rxq */
+ struct sge_fl *fl = &rxq->fl; /* Use iff IQ_HAS_FL */
struct adapter *sc = iq->adapter;
struct rsp_ctrl *ctrl;
- int ndesc_pending = 0, ndesc_total = 0;
+ const struct rss_header *rss;
+ int ndescs = 0, limit, fl_bufs_used = 0;
+ int rsp_type;
+ uint32_t lq;
+ struct mbuf *m0;
+ STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
+
+ limit = budget ? budget : iq->qsize / 8;
+
+ KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
+
+ /*
+ * We always come back and check the descriptor ring for new indirect
+ * interrupts and other responses after running a single handler.
+ */
+ for (;;) {
+ while (is_new_response(iq, &ctrl)) {
+
+ rmb();
+
+ m0 = NULL;
+ rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
+ lq = be32toh(ctrl->pldbuflen_qid);
+ rss = (const void *)iq->cdesc;
+
+ switch (rsp_type) {
+ case X_RSPD_TYPE_FLBUF:
+
+ KASSERT(iq->flags & IQ_HAS_FL,
+ ("%s: data for an iq (%p) with no freelist",
+ __func__, iq));
+
+ m0 = get_fl_payload(sc, fl, lq, &fl_bufs_used);
+#ifdef T4_PKT_TIMESTAMP
+ /*
+ * 60 bit timestamp for the payload is
+ * *(uint64_t *)m0->m_pktdat. Note that it is
+ * in the leading free-space in the mbuf. The
+ * kernel can clobber it during a pullup,
+ * m_copymdata, etc. You need to make sure that
+ * the mbuf reaches you unmolested if you care
+ * about the timestamp.
+ */
+ *(uint64_t *)m0->m_pktdat =
+ be64toh(ctrl->u.last_flit) &
+ 0xfffffffffffffff;
+#endif
+
+ /* fall through */
+
+ case X_RSPD_TYPE_CPL:
+ KASSERT(rss->opcode < NUM_CPL_CMDS,
+ ("%s: bad opcode %02x.", __func__,
+ rss->opcode));
+ sc->cpl_handler[rss->opcode](iq, rss, m0);
+ break;
+
+ case X_RSPD_TYPE_INTR:
+
+ /*
+ * Interrupts should be forwarded only to queues
+ * that are not forwarding their interrupts.
+ * This means service_iq can recurse but only 1
+ * level deep.
+ */
+ KASSERT(budget == 0,
+ ("%s: budget %u, rsp_type %u", __func__,
+ budget, rsp_type));
+
+ q = sc->sge.iqmap[lq - sc->sge.iq_start];
+ if (atomic_cmpset_int(&q->state, IQS_IDLE,
+ IQS_BUSY)) {
+ if (service_iq(q, q->qsize / 8) == 0) {
+ atomic_cmpset_int(&q->state,
+ IQS_BUSY, IQS_IDLE);
+ } else {
+ STAILQ_INSERT_TAIL(&iql, q,
+ link);
+ }
+ }
+ break;
- KASSERT(iq == &sc->sge.fwq, ("%s: unexpected ingress queue", __func__));
+ default:
+ panic("%s: rsp_type %u", __func__, rsp_type);
+ }
- while (is_new_response(iq, &ctrl)) {
- int rsp_type;
+ iq_next(iq);
+ if (++ndescs == limit) {
+ t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
+ V_CIDXINC(ndescs) |
+ V_INGRESSQID(iq->cntxt_id) |
+ V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
+ ndescs = 0;
+
+ if (fl_bufs_used > 0) {
+ FL_LOCK(fl);
+ fl->needed += fl_bufs_used;
+ refill_fl(sc, fl, fl->cap / 8);
+ FL_UNLOCK(fl);
+ fl_bufs_used = 0;
+ }
- rmb();
+ if (budget)
+ return (EINPROGRESS);
+ }
+ }
- rsp_type = G_RSPD_TYPE(ctrl->u.type_gen);
- if (__predict_false(rsp_type != X_RSPD_TYPE_CPL))
- panic("%s: unexpected rsp_type %d", __func__, rsp_type);
+ if (STAILQ_EMPTY(&iql))
+ break;
- handle_cpl(sc, iq);
+ /*
+ * Process the head only, and send it to the back of the list if
+ * it's still not done.
+ */
+ q = STAILQ_FIRST(&iql);
+ STAILQ_REMOVE_HEAD(&iql, link);
+ if (service_iq(q, q->qsize / 8) == 0)
+ atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
+ else
+ STAILQ_INSERT_TAIL(&iql, q, link);
+ }
- ndesc_total++;
- if (++ndesc_pending >= iq->qsize / 4) {
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
- V_CIDXINC(ndesc_pending) |
- V_INGRESSQID(iq->cntxt_id) |
- V_SEINTARM(
- V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
- ndesc_pending = 0;
+#ifdef INET
+ if (iq->flags & IQ_LRO_ENABLED) {
+ struct lro_ctrl *lro = &rxq->lro;
+ struct lro_entry *l;
+
+ while (!SLIST_EMPTY(&lro->lro_active)) {
+ l = SLIST_FIRST(&lro->lro_active);
+ SLIST_REMOVE_HEAD(&lro->lro_active, next);
+ tcp_lro_flush(lro, l);
}
+ }
+#endif
+
+ t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
+ V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
+
+ if (iq->flags & IQ_HAS_FL) {
+ int starved;
- iq_next(iq);
+ FL_LOCK(fl);
+ fl->needed += fl_bufs_used;
+ starved = refill_fl(sc, fl, fl->cap / 4);
+ FL_UNLOCK(fl);
+ if (__predict_false(starved != 0))
+ add_fl_to_sfl(sc, fl);
}
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndesc_pending) |
- V_INGRESSQID(iq->cntxt_id) | V_SEINTARM(iq->intr_params));
+ return (0);
}
+
#ifdef T4_PKT_TIMESTAMP
#define RX_COPY_THRESHOLD (MINCLSIZE - 8)
#else
#define RX_COPY_THRESHOLD MINCLSIZE
#endif
-static void
-t4_eth_rx(void *arg)
+static struct mbuf *
+get_fl_payload(struct adapter *sc, struct sge_fl *fl, uint32_t len_newbuf,
+ int *fl_bufs_used)
{
- struct sge_rxq *rxq = arg;
- struct sge_iq *iq = arg;
- struct adapter *sc = iq->adapter;
- struct rsp_ctrl *ctrl;
- struct ifnet *ifp = rxq->ifp;
- struct sge_fl *fl = &rxq->fl;
- struct fl_sdesc *sd = &fl->sdesc[fl->cidx], *sd_next;
- const struct rss_header *rss;
- const struct cpl_rx_pkt *cpl;
- uint32_t len;
- int ndescs = 0, i;
struct mbuf *m0, *m;
-#ifdef INET
- struct lro_ctrl *lro = &rxq->lro;
- struct lro_entry *l;
-#endif
+ struct fl_sdesc *sd = &fl->sdesc[fl->cidx];
+ unsigned int nbuf, len;
- prefetch(sd->m);
- prefetch(sd->cl);
+ /*
+ * No assertion for the fl lock because we don't need it. This routine
+ * is called only from the rx interrupt handler and it only updates
+ * fl->cidx. (Contrast that with fl->pidx/fl->needed which could be
+ * updated in the rx interrupt handler or the starvation helper routine.
+ * That's why code that manipulates fl->pidx/fl->needed needs the fl
+ * lock but this routine does not).
+ */
- iq->intr_next = iq->intr_params;
- while (is_new_response(iq, &ctrl)) {
+ if (__predict_false((len_newbuf & F_RSPD_NEWBUF) == 0))
+ panic("%s: cannot handle packed frames", __func__);
+ len = G_RSPD_LEN(len_newbuf);
- rmb();
+ m0 = sd->m;
+ sd->m = NULL; /* consumed */
- rss = (const void *)iq->cdesc;
- i = G_RSPD_TYPE(ctrl->u.type_gen);
+ bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map, BUS_DMASYNC_POSTREAD);
+ m_init(m0, NULL, 0, M_NOWAIT, MT_DATA, M_PKTHDR);
+#ifdef T4_PKT_TIMESTAMP
+ /* Leave room for a timestamp */
+ m0->m_data += 8;
+#endif
- KASSERT(i == X_RSPD_TYPE_FLBUF && rss->opcode == CPL_RX_PKT,
- ("%s: unexpected type %d CPL opcode 0x%x",
- __func__, i, rss->opcode));
+ if (len < RX_COPY_THRESHOLD) {
+ /* copy data to mbuf, buffer will be recycled */
+ bcopy(sd->cl, mtod(m0, caddr_t), len);
+ m0->m_len = len;
+ } else {
+ bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map);
+ m_cljset(m0, sd->cl, FL_BUF_TYPE(sd->tag_idx));
+ sd->cl = NULL; /* consumed */
+ m0->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
+ }
+ m0->m_pkthdr.len = len;
- sd_next = sd + 1;
- if (__predict_false(fl->cidx + 1 == fl->cap))
- sd_next = fl->sdesc;
- prefetch(sd_next->m);
- prefetch(sd_next->cl);
+ sd++;
+ if (__predict_false(++fl->cidx == fl->cap)) {
+ sd = fl->sdesc;
+ fl->cidx = 0;
+ }
- cpl = (const void *)(rss + 1);
+ m = m0;
+ len -= m->m_len;
+ nbuf = 1; /* # of fl buffers used */
- m0 = sd->m;
+ while (len > 0) {
+ m->m_next = sd->m;
sd->m = NULL; /* consumed */
-
- len = be32toh(ctrl->pldbuflen_qid);
- if (__predict_false((len & F_RSPD_NEWBUF) == 0))
- panic("%s: cannot handle packed frames", __func__);
- len = G_RSPD_LEN(len);
+ m = m->m_next;
bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map,
BUS_DMASYNC_POSTREAD);
- m_init(m0, NULL, 0, M_NOWAIT, MT_DATA, M_PKTHDR);
-
-#ifdef T4_PKT_TIMESTAMP
- *mtod(m0, uint64_t *) =
- be64toh(ctrl->u.last_flit & 0xfffffffffffffff);
- m0->m_data += 8;
-
- /*
- * 60 bit timestamp value is *(uint64_t *)m0->m_pktdat. Note
- * that it is in the leading free-space (see M_LEADINGSPACE) in
- * the mbuf. The kernel can clobber it during a pullup,
- * m_copymdata, etc. You need to make sure that the mbuf
- * reaches you unmolested if you care about the timestamp.
- */
-#endif
-
- if (len < RX_COPY_THRESHOLD) {
- /* copy data to mbuf, buffer will be recycled */
- bcopy(sd->cl, mtod(m0, caddr_t), len);
- m0->m_len = len;
+ m_init(m, NULL, 0, M_NOWAIT, MT_DATA, 0);
+ if (len <= MLEN) {
+ bcopy(sd->cl, mtod(m, caddr_t), len);
+ m->m_len = len;
} else {
- bus_dmamap_unload(fl->tag[sd->tag_idx], sd->map);
- m_cljset(m0, sd->cl, FL_BUF_TYPE(sd->tag_idx));
+ bus_dmamap_unload(fl->tag[sd->tag_idx],
+ sd->map);
+ m_cljset(m, sd->cl, FL_BUF_TYPE(sd->tag_idx));
sd->cl = NULL; /* consumed */
- m0->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
+ m->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
}
- len -= FL_PKTSHIFT;
- m0->m_len -= FL_PKTSHIFT;
- m0->m_data += FL_PKTSHIFT;
-
- m0->m_pkthdr.len = len;
- m0->m_pkthdr.rcvif = ifp;
- m0->m_flags |= M_FLOWID;
- m0->m_pkthdr.flowid = rss->hash_val;
-
- if (cpl->csum_calc && !cpl->err_vec &&
- ifp->if_capenable & IFCAP_RXCSUM) {
- m0->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED |
- CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
- if (cpl->ip_frag)
- m0->m_pkthdr.csum_data = be16toh(cpl->csum);
- else
- m0->m_pkthdr.csum_data = 0xffff;
- rxq->rxcsum++;
- }
-
- if (cpl->vlan_ex) {
- m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
- m0->m_flags |= M_VLANTAG;
- rxq->vlan_extraction++;
- }
-
- i = 1; /* # of fl sdesc used */
- sd = sd_next;
- if (__predict_false(++fl->cidx == fl->cap))
+ sd++;
+ if (__predict_false(++fl->cidx == fl->cap)) {
+ sd = fl->sdesc;
fl->cidx = 0;
+ }
- len -= m0->m_len;
- m = m0;
- while (len) {
- i++;
-
- sd_next = sd + 1;
- if (__predict_false(fl->cidx + 1 == fl->cap))
- sd_next = fl->sdesc;
- prefetch(sd_next->m);
- prefetch(sd_next->cl);
-
- m->m_next = sd->m;
- sd->m = NULL; /* consumed */
- m = m->m_next;
-
- bus_dmamap_sync(fl->tag[sd->tag_idx], sd->map,
- BUS_DMASYNC_POSTREAD);
-
- m_init(m, NULL, 0, M_NOWAIT, MT_DATA, 0);
- if (len <= MLEN) {
- bcopy(sd->cl, mtod(m, caddr_t), len);
- m->m_len = len;
- } else {
- bus_dmamap_unload(fl->tag[sd->tag_idx],
- sd->map);
- m_cljset(m, sd->cl, FL_BUF_TYPE(sd->tag_idx));
- sd->cl = NULL; /* consumed */
- m->m_len = min(len, FL_BUF_SIZE(sd->tag_idx));
- }
+ len -= m->m_len;
+ nbuf++;
+ }
- i++;
- sd = sd_next;
- if (__predict_false(++fl->cidx == fl->cap))
- fl->cidx = 0;
+ (*fl_bufs_used) += nbuf;
- len -= m->m_len;
- }
+ return (m0);
+}
+static int
+t4_eth_rx(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m0)
+{
+ struct sge_rxq *rxq = (void *)iq;
+ struct ifnet *ifp = rxq->ifp;
+ const struct cpl_rx_pkt *cpl = (const void *)(rss + 1);
#ifdef INET
- if (cpl->l2info & htobe32(F_RXF_LRO) &&
- rxq->flags & RXQ_LRO_ENABLED &&
- tcp_lro_rx(lro, m0, 0) == 0) {
- /* queued for LRO */
- } else
+ struct lro_ctrl *lro = &rxq->lro;
#endif
- ifp->if_input(ifp, m0);
- FL_LOCK(fl);
- fl->needed += i;
- if (fl->needed >= 32)
- refill_fl(sc, fl, 64, 32);
- FL_UNLOCK(fl);
+ KASSERT(m0 != NULL, ("%s: no payload with opcode %02x", __func__,
+ rss->opcode));
- if (++ndescs > 32) {
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS),
- V_CIDXINC(ndescs) |
- V_INGRESSQID((u32)iq->cntxt_id) |
- V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
- ndescs = 0;
- }
+ m0->m_pkthdr.len -= FL_PKTSHIFT;
+ m0->m_len -= FL_PKTSHIFT;
+ m0->m_data += FL_PKTSHIFT;
+
+ m0->m_pkthdr.rcvif = ifp;
+ m0->m_flags |= M_FLOWID;
+ m0->m_pkthdr.flowid = rss->hash_val;
- iq_next(iq);
+ if (cpl->csum_calc && !cpl->err_vec &&
+ ifp->if_capenable & IFCAP_RXCSUM) {
+ m0->m_pkthdr.csum_flags |= (CSUM_IP_CHECKED |
+ CSUM_IP_VALID | CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
+ if (cpl->ip_frag)
+ m0->m_pkthdr.csum_data = be16toh(cpl->csum);
+ else
+ m0->m_pkthdr.csum_data = 0xffff;
+ rxq->rxcsum++;
}
-#ifdef INET
- while (!SLIST_EMPTY(&lro->lro_active)) {
- l = SLIST_FIRST(&lro->lro_active);
- SLIST_REMOVE_HEAD(&lro->lro_active, next);
- tcp_lro_flush(lro, l);
+ if (cpl->vlan_ex) {
+ m0->m_pkthdr.ether_vtag = be16toh(cpl->vlan);
+ m0->m_flags |= M_VLANTAG;
+ rxq->vlan_extraction++;
}
-#endif
- t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_CIDXINC(ndescs) |
- V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_next));
+#ifdef INET
+ if (cpl->l2info & htobe32(F_RXF_LRO) &&
+ iq->flags & IQ_LRO_ENABLED &&
+ tcp_lro_rx(lro, m0, 0) == 0) {
+ /* queued for LRO */
+ } else
+#endif
+ ifp->if_input(ifp, m0);
- FL_LOCK(fl);
- if (fl->needed >= 32)
- refill_fl(sc, fl, 128, 8);
- FL_UNLOCK(fl);
+ return (0);
}
int
t4_mgmt_tx(struct adapter *sc, struct mbuf *m)
{
- return ctrl_tx(sc, &sc->sge.ctrlq[0], m);
+ return t4_wrq_tx(sc, &sc->sge.mgmtq, m);
+}
+
+/*
+ * Doesn't fail. Holds on to work requests it can't send right away.
+ */
+int
+t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, struct mbuf *m0)
+{
+ struct sge_eq *eq = &wrq->eq;
+ int can_reclaim;
+ caddr_t dst;
+ struct mbuf *wr, *next;
+
+ TXQ_LOCK_ASSERT_OWNED(wrq);
+ KASSERT((eq->flags & EQ_TYPEMASK) == EQ_OFLD ||
+ (eq->flags & EQ_TYPEMASK) == EQ_CTRL,
+ ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
+
+ if (__predict_true(m0 != NULL)) {
+ if (wrq->head)
+ wrq->tail->m_nextpkt = m0;
+ else
+ wrq->head = m0;
+ while (m0->m_nextpkt)
+ m0 = m0->m_nextpkt;
+ wrq->tail = m0;
+ }
+
+ can_reclaim = reclaimable(eq);
+ if (__predict_false(eq->flags & EQ_STALLED)) {
+ if (can_reclaim < tx_resume_threshold(eq))
+ return (0);
+ eq->flags &= ~EQ_STALLED;
+ eq->unstalled++;
+ }
+ eq->cidx += can_reclaim;
+ eq->avail += can_reclaim;
+ if (__predict_false(eq->cidx >= eq->cap))
+ eq->cidx -= eq->cap;
+
+ for (wr = wrq->head; wr; wr = next) {
+ int ndesc;
+ struct mbuf *m;
+
+ next = wr->m_nextpkt;
+ wr->m_nextpkt = NULL;
+
+ M_ASSERTPKTHDR(wr);
+ KASSERT(wr->m_pkthdr.len > 0 && (wr->m_pkthdr.len & 0x7) == 0,
+ ("%s: work request len %d.", __func__, wr->m_pkthdr.len));
+
+ if (wr->m_pkthdr.len > SGE_MAX_WR_LEN) {
+#ifdef INVARIANTS
+ panic("%s: oversized work request", __func__);
+#else
+ log(LOG_ERR, "%s: %s work request too long (%d)",
+ device_get_nameunit(sc->dev), __func__,
+ wr->m_pkthdr.len);
+ m_freem(wr);
+ continue;
+#endif
+ }
+
+ ndesc = howmany(wr->m_pkthdr.len, EQ_ESIZE);
+ if (eq->avail < ndesc) {
+ wr->m_nextpkt = next;
+ wrq->no_desc++;
+ break;
+ }
+
+ dst = (void *)&eq->desc[eq->pidx];
+ for (m = wr; m; m = m->m_next)
+ copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
+
+ eq->pidx += ndesc;
+ eq->avail -= ndesc;
+ if (__predict_false(eq->pidx >= eq->cap))
+ eq->pidx -= eq->cap;
+
+ eq->pending += ndesc;
+ if (eq->pending > 16)
+ ring_eq_db(sc, eq);
+
+ wrq->tx_wrs++;
+ m_freem(wr);
+
+ if (eq->avail < 8) {
+ can_reclaim = reclaimable(eq);
+ eq->cidx += can_reclaim;
+ eq->avail += can_reclaim;
+ if (__predict_false(eq->cidx >= eq->cap))
+ eq->cidx -= eq->cap;
+ }
+ }
+
+ if (eq->pending)
+ ring_eq_db(sc, eq);
+
+ if (wr == NULL)
+ wrq->head = wrq->tail = NULL;
+ else {
+ wrq->head = wr;
+
+ KASSERT(wrq->tail->m_nextpkt == NULL,
+ ("%s: wrq->tail grew a tail of its own", __func__));
+
+ eq->flags |= EQ_STALLED;
+ if (callout_pending(&eq->tx_callout) == 0)
+ callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
+ }
+
+ return (0);
}
/* Per-packet header in a coalesced tx WR, before the SGL starts (in flits) */
@@ -852,6 +1233,8 @@ t4_eth_tx(struct ifnet *ifp, struct sge_txq *txq, struct mbuf *m)
TXQ_LOCK_ASSERT_OWNED(txq);
KASSERT(m, ("%s: called with nothing to do.", __func__));
+ KASSERT((eq->flags & EQ_TYPEMASK) == EQ_ETH,
+ ("%s: eq type %d", __func__, eq->flags & EQ_TYPEMASK));
prefetch(&eq->desc[eq->pidx]);
prefetch(&txq->sdesc[eq->pidx]);
@@ -859,8 +1242,25 @@ t4_eth_tx(struct ifnet *ifp, struct sge_txq *txq, struct mbuf *m)
txpkts.npkt = 0;/* indicates there's nothing in txpkts */
coalescing = 0;
- if (eq->avail < 8)
- reclaim_tx_descs(txq, 0, 8);
+ can_reclaim = reclaimable(eq);
+ if (__predict_false(eq->flags & EQ_STALLED)) {
+ if (can_reclaim < tx_resume_threshold(eq)) {
+ txq->m = m;
+ return (0);
+ }
+ eq->flags &= ~EQ_STALLED;
+ eq->unstalled++;
+ }
+
+ if (__predict_false(eq->flags & EQ_DOOMED)) {
+ m_freem(m);
+ while ((m = buf_ring_dequeue_sc(txq->br)) != NULL)
+ m_freem(m);
+ return (ENETDOWN);
+ }
+
+ if (eq->avail < 8 && can_reclaim)
+ reclaim_tx_descs(txq, can_reclaim, 32);
for (; m; m = next ? next : drbr_dequeue(ifp, br)) {
@@ -938,15 +1338,13 @@ t4_eth_tx(struct ifnet *ifp, struct sge_txq *txq, struct mbuf *m)
ETHER_BPF_MTAP(ifp, m);
if (sgl.nsegs == 0)
m_freem(m);
-
doorbell:
- /* Fewer and fewer doorbells as the queue fills up */
- if (eq->pending >= (1 << (fls(eq->qsize - eq->avail) / 2)))
+ if (eq->pending >= 64)
ring_eq_db(sc, eq);
can_reclaim = reclaimable(eq);
if (can_reclaim >= 32)
- reclaim_tx_descs(txq, can_reclaim, 32);
+ reclaim_tx_descs(txq, can_reclaim, 64);
}
if (txpkts.npkt > 0)
@@ -957,14 +1355,17 @@ doorbell:
* This can happen when we're short of tx descriptors (no_desc) or maybe
* even DMA maps (no_dmamap). Either way, a credit flush and reclaim
* will get things going again.
- *
- * If eq->avail is already 0 we know a credit flush was requested in the
- * WR that reduced it to 0 so we don't need another flush (we don't have
- * any descriptor for a flush WR anyway, duh).
*/
- if (m && eq->avail > 0 && !(eq->flags & EQ_CRFLUSHED)) {
+ if (m && !(eq->flags & EQ_CRFLUSHED)) {
struct tx_sdesc *txsd = &txq->sdesc[eq->pidx];
+ /*
+ * If EQ_CRFLUSHED is not set then we know we have at least one
+ * available descriptor because any WR that reduces eq->avail to
+ * 0 also sets EQ_CRFLUSHED.
+ */
+ KASSERT(eq->avail > 0, ("%s: no space for eqflush.", __func__));
+
txsd->desc_used = 1;
txsd->credits = 0;
write_eqflush_wr(eq);
@@ -974,9 +1375,10 @@ doorbell:
if (eq->pending)
ring_eq_db(sc, eq);
- can_reclaim = reclaimable(eq);
- if (can_reclaim >= 32)
- reclaim_tx_descs(txq, can_reclaim, 128);
+ reclaim_tx_descs(txq, 0, 128);
+
+ if (eq->flags & EQ_STALLED && callout_pending(&eq->tx_callout) == 0)
+ callout_reset(&eq->tx_callout, 1, t4_tx_callout, eq);
return (0);
}
@@ -987,24 +1389,29 @@ t4_update_fl_bufsize(struct ifnet *ifp)
struct port_info *pi = ifp->if_softc;
struct sge_rxq *rxq;
struct sge_fl *fl;
- int i;
+ int i, bufsize;
+ /* large enough for a frame even when VLAN extraction is disabled */
+ bufsize = FL_PKTSHIFT + ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN +
+ ifp->if_mtu;
for_each_rxq(pi, i, rxq) {
fl = &rxq->fl;
FL_LOCK(fl);
- set_fl_tag_idx(fl, ifp->if_mtu);
+ set_fl_tag_idx(fl, bufsize);
FL_UNLOCK(fl);
}
}
-/*
- * A non-NULL handler indicates this iq will not receive direct interrupts, the
- * handler will be invoked by an interrupt queue.
- */
+int
+can_resume_tx(struct sge_eq *eq)
+{
+ return (reclaimable(eq) >= tx_resume_threshold(eq));
+}
+
static inline void
init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
- int qsize, int esize, iq_intr_handler_t *handler, char *name)
+ int qsize, int esize, char *name)
{
KASSERT(tmr_idx >= 0 && tmr_idx < SGE_NTIMERS,
("%s: bad tmr_idx %d", __func__, tmr_idx));
@@ -1018,22 +1425,32 @@ init_iq(struct sge_iq *iq, struct adapter *sc, int tmr_idx, int pktc_idx,
iq->intr_pktc_idx = pktc_idx;
iq->qsize = roundup(qsize, 16); /* See FW_IQ_CMD/iqsize */
iq->esize = max(esize, 16); /* See FW_IQ_CMD/iqesize */
- iq->handler = handler;
strlcpy(iq->lockname, name, sizeof(iq->lockname));
}
static inline void
-init_fl(struct sge_fl *fl, int qsize, char *name)
+init_fl(struct sge_fl *fl, int qsize, int bufsize, char *name)
{
fl->qsize = qsize;
strlcpy(fl->lockname, name, sizeof(fl->lockname));
+ set_fl_tag_idx(fl, bufsize);
}
static inline void
-init_eq(struct sge_eq *eq, int qsize, char *name)
+init_eq(struct sge_eq *eq, int eqtype, int qsize, uint8_t tx_chan,
+ uint16_t iqid, char *name)
{
+ KASSERT(tx_chan < NCHAN, ("%s: bad tx channel %d", __func__, tx_chan));
+ KASSERT(eqtype <= EQ_TYPEMASK, ("%s: bad qtype %d", __func__, eqtype));
+
+ eq->flags = eqtype & EQ_TYPEMASK;
+ eq->tx_chan = tx_chan;
+ eq->iqid = iqid;
eq->qsize = qsize;
strlcpy(eq->lockname, name, sizeof(eq->lockname));
+
+ TASK_INIT(&eq->tx_task, 0, t4_tx_task, eq);
+ callout_init(&eq->tx_callout, CALLOUT_MPSAFE);
}
static int
@@ -1090,9 +1507,9 @@ free_ring(struct adapter *sc, bus_dma_tag_t tag, bus_dmamap_t map,
* Returns errno on failure. Resources allocated up to that point may still be
* allocated. Caller is responsible for cleanup in case this function fails.
*
- * If the ingress queue will take interrupts directly (iq->handler == NULL) then
+ * If the ingress queue will take interrupts directly (iq->flags & IQ_INTR) then
* the intr_idx specifies the vector, starting from 0. Otherwise it specifies
- * the index of the interrupt queue to which its interrupts will be forwarded.
+ * the abs_id of the ingress queue to which its interrupts should be forwarded.
*/
static int
alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
@@ -1122,16 +1539,12 @@ alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
if (iq == &sc->sge.fwq)
v |= F_FW_IQ_CMD_IQASYNCH;
- if (iq->handler) {
- KASSERT(intr_idx < NINTRQ(sc),
- ("%s: invalid indirect intr_idx %d", __func__, intr_idx));
- v |= F_FW_IQ_CMD_IQANDST;
- v |= V_FW_IQ_CMD_IQANDSTINDEX(sc->sge.intrq[intr_idx].abs_id);
- } else {
+ if (iq->flags & IQ_INTR) {
KASSERT(intr_idx < sc->intr_count,
("%s: invalid direct intr_idx %d", __func__, intr_idx));
- v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
- }
+ } else
+ v |= F_FW_IQ_CMD_IQANDST;
+ v |= V_FW_IQ_CMD_IQANDSTINDEX(intr_idx);
c.type_to_iqandstindex = htobe32(v |
V_FW_IQ_CMD_TYPE(FW_IQ_TYPE_FL_INT_CAP) |
@@ -1176,7 +1589,6 @@ alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
/* Allocate space for one software descriptor per buffer. */
fl->cap = (fl->qsize - SPG_LEN / RX_FL_ESIZE) * 8;
FL_LOCK(fl);
- set_fl_tag_idx(fl, pi->ifp->if_mtu);
rc = alloc_fl_sdesc(fl);
FL_UNLOCK(fl);
if (rc != 0) {
@@ -1186,6 +1598,7 @@ alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
return (rc);
}
fl->needed = fl->cap;
+ fl->lowat = roundup(sc->sge.fl_starve_threshold, 8);
c.iqns_to_fl0congen |=
htobe32(V_FW_IQ_CMD_FL0HOSTFCMODE(X_HOSTFCMODE_NONE) |
@@ -1217,12 +1630,13 @@ alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
iq->intr_next = iq->intr_params;
iq->cntxt_id = be16toh(c.iqid);
iq->abs_id = be16toh(c.physiqid);
- iq->flags |= (IQ_ALLOCATED | IQ_STARTED);
+ iq->flags |= IQ_ALLOCATED;
cntxt_id = iq->cntxt_id - sc->sge.iq_start;
- KASSERT(cntxt_id < sc->sge.niq,
- ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
- cntxt_id, sc->sge.niq - 1));
+ if (cntxt_id >= sc->sge.niq) {
+ panic ("%s: iq->cntxt_id (%d) more than the max (%d)", __func__,
+ cntxt_id, sc->sge.niq - 1);
+ }
sc->sge.iqmap[cntxt_id] = iq;
if (fl) {
@@ -1230,29 +1644,28 @@ alloc_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl,
fl->pidx = fl->cidx = 0;
cntxt_id = fl->cntxt_id - sc->sge.eq_start;
- KASSERT(cntxt_id < sc->sge.neq,
- ("%s: fl->cntxt_id (%d) more than the max (%d)", __func__,
- cntxt_id, sc->sge.neq - 1));
+ if (cntxt_id >= sc->sge.neq) {
+ panic("%s: fl->cntxt_id (%d) more than the max (%d)",
+ __func__, cntxt_id, sc->sge.neq - 1);
+ }
sc->sge.eqmap[cntxt_id] = (void *)fl;
FL_LOCK(fl);
- /* Just enough to make sure it doesn't starve right away. */
- refill_fl(sc, fl, roundup(sc->sge.fl_starve_threshold, 8), 8);
+ /* Enough to make sure the SGE doesn't think it's starved */
+ refill_fl(sc, fl, fl->lowat);
FL_UNLOCK(fl);
+
+ iq->flags |= IQ_HAS_FL;
}
/* Enable IQ interrupts */
- atomic_store_rel_32(&iq->state, IQS_IDLE);
+ atomic_store_rel_int(&iq->state, IQS_IDLE);
t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), V_SEINTARM(iq->intr_params) |
V_INGRESSQID(iq->cntxt_id));
return (0);
}
-/*
- * This can be called with the iq/fl in any state - fully allocated and
- * functional, partially allocated, even all-zeroed out.
- */
static int
free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
{
@@ -1265,23 +1678,7 @@ free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
dev = pi ? pi->dev : sc->dev;
- if (iq->flags & IQ_STARTED) {
- rc = -t4_iq_start_stop(sc, sc->mbox, 0, sc->pf, 0,
- iq->cntxt_id, fl ? fl->cntxt_id : 0xffff, 0xffff);
- if (rc != 0) {
- device_printf(dev,
- "failed to stop queue %p: %d\n", iq, rc);
- return (rc);
- }
- iq->flags &= ~IQ_STARTED;
-
- /* Synchronize with the interrupt handler */
- while (!atomic_cmpset_32(&iq->state, IQS_IDLE, IQS_DISABLED))
- pause("iqfree", hz / 1000);
- }
-
if (iq->flags & IQ_ALLOCATED) {
-
rc = -t4_iq_free(sc, sc->mbox, sc->pf, 0,
FW_IQ_TYPE_FL_INT_CAP, iq->cntxt_id,
fl ? fl->cntxt_id : 0xffff, 0xffff);
@@ -1322,76 +1719,84 @@ free_iq_fl(struct port_info *pi, struct sge_iq *iq, struct sge_fl *fl)
}
static int
-alloc_intrq(struct adapter *sc, int port_idx, int intrq_idx, int intr_idx)
+alloc_fwq(struct adapter *sc)
{
- int rc;
- struct sysctl_oid *oid;
- struct sysctl_oid_list *children;
+ int rc, intr_idx;
+ struct sge_iq *fwq = &sc->sge.fwq;
char name[16];
- struct sge_iq *intrq = &sc->sge.intrq[intrq_idx];
+ struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
+ struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
- rc = alloc_iq_fl(sc->port[port_idx], intrq, NULL, intr_idx, -1);
- if (rc != 0)
+ snprintf(name, sizeof(name), "%s fwq", device_get_nameunit(sc->dev));
+ init_iq(fwq, sc, 0, 0, FW_IQ_QSIZE, FW_IQ_ESIZE, name);
+ fwq->flags |= IQ_INTR; /* always */
+ intr_idx = sc->intr_count > 1 ? 1 : 0;
+ rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to create firmware event queue: %d\n", rc);
return (rc);
+ }
- children = SYSCTL_CHILDREN(sc->oid_intrq);
-
- snprintf(name, sizeof(name), "%d", intrq_idx);
- oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, name, CTLFLAG_RD,
- NULL, "interrupt queue");
+ oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "fwq", CTLFLAG_RD,
+ NULL, "firmware event queue");
children = SYSCTL_CHILDREN(oid);
+ SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "abs_id",
+ CTLTYPE_INT | CTLFLAG_RD, &fwq->abs_id, 0, sysctl_uint16, "I",
+ "absolute id of the queue");
+ SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cntxt_id",
+ CTLTYPE_INT | CTLFLAG_RD, &fwq->cntxt_id, 0, sysctl_uint16, "I",
+ "SGE context id of the queue");
SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cidx",
- CTLTYPE_INT | CTLFLAG_RD, &intrq->cidx, 0, sysctl_uint16, "I",
+ CTLTYPE_INT | CTLFLAG_RD, &fwq->cidx, 0, sysctl_uint16, "I",
"consumer index");
- return (rc);
+ return (0);
}
static int
-free_intrq(struct sge_iq *iq)
+free_fwq(struct adapter *sc)
{
- return free_iq_fl(NULL, iq, NULL);
-
+ return free_iq_fl(NULL, &sc->sge.fwq, NULL);
}
static int
-alloc_fwq(struct adapter *sc, int intr_idx)
+alloc_mgmtq(struct adapter *sc)
{
int rc;
- struct sysctl_oid_list *children;
- struct sge_iq *fwq = &sc->sge.fwq;
-
- rc = alloc_iq_fl(sc->port[0], fwq, NULL, intr_idx, -1);
- if (rc != 0)
- return (rc);
+ struct sge_wrq *mgmtq = &sc->sge.mgmtq;
+ char name[16];
+ struct sysctl_oid *oid = device_get_sysctl_tree(sc->dev);
+ struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
- children = SYSCTL_CHILDREN(sc->oid_fwq);
+ oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, "mgmtq", CTLFLAG_RD,
+ NULL, "management queue");
- SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "abs_id",
- CTLTYPE_INT | CTLFLAG_RD, &fwq->abs_id, 0, sysctl_uint16, "I",
- "absolute id of the queue");
- SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cntxt_id",
- CTLTYPE_INT | CTLFLAG_RD, &fwq->cntxt_id, 0, sysctl_uint16, "I",
- "SGE context id of the queue");
- SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "cidx",
- CTLTYPE_INT | CTLFLAG_RD, &fwq->cidx, 0, sysctl_uint16, "I",
- "consumer index");
+ snprintf(name, sizeof(name), "%s mgmtq", device_get_nameunit(sc->dev));
+ init_eq(&mgmtq->eq, EQ_CTRL, CTRL_EQ_QSIZE, sc->port[0]->tx_chan,
+ sc->sge.fwq.cntxt_id, name);
+ rc = alloc_wrq(sc, NULL, mgmtq, oid);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to create management queue: %d\n", rc);
+ return (rc);
+ }
- return (rc);
+ return (0);
}
static int
-free_fwq(struct sge_iq *iq)
+free_mgmtq(struct adapter *sc)
{
- return free_iq_fl(NULL, iq, NULL);
+ return free_wrq(sc, &sc->sge.mgmtq);
}
static int
-alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int idx)
+alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int idx,
+ struct sysctl_oid *oid)
{
int rc;
- struct sysctl_oid *oid;
struct sysctl_oid_list *children;
char name[16];
@@ -1400,7 +1805,7 @@ alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int idx)
return (rc);
FL_LOCK(&rxq->fl);
- refill_fl(pi->adapter, &rxq->fl, rxq->fl.needed / 8, 8);
+ refill_fl(pi->adapter, &rxq->fl, rxq->fl.needed / 8);
FL_UNLOCK(&rxq->fl);
#ifdef INET
@@ -1410,11 +1815,11 @@ alloc_rxq(struct port_info *pi, struct sge_rxq *rxq, int intr_idx, int idx)
rxq->lro.ifp = pi->ifp; /* also indicates LRO init'ed */
if (pi->ifp->if_capenable & IFCAP_LRO)
- rxq->flags |= RXQ_LRO_ENABLED;
+ rxq->iq.flags |= IQ_LRO_ENABLED;
#endif
rxq->ifp = pi->ifp;
- children = SYSCTL_CHILDREN(pi->oid_rxq);
+ children = SYSCTL_CHILDREN(oid);
snprintf(name, sizeof(name), "%d", idx);
oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
@@ -1477,32 +1882,71 @@ free_rxq(struct port_info *pi, struct sge_rxq *rxq)
return (rc);
}
+#ifndef TCP_OFFLOAD_DISABLE
static int
-alloc_ctrlq(struct adapter *sc, struct sge_ctrlq *ctrlq, int idx)
+alloc_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq,
+ int intr_idx, int idx, struct sysctl_oid *oid)
{
- int rc, cntxt_id;
- size_t len;
- struct fw_eq_ctrl_cmd c;
- struct sge_eq *eq = &ctrlq->eq;
- char name[16];
- struct sysctl_oid *oid;
+ int rc;
struct sysctl_oid_list *children;
+ char name[16];
- mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
-
- len = eq->qsize * CTRL_EQ_ESIZE;
- rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map,
- &eq->ba, (void **)&eq->desc);
- if (rc)
+ rc = alloc_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl, intr_idx,
+ 1 << pi->tx_chan);
+ if (rc != 0)
return (rc);
- eq->cap = eq->qsize - SPG_LEN / CTRL_EQ_ESIZE;
- eq->spg = (void *)&eq->desc[eq->cap];
- eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
- if (sc->flags & INTR_SHARED)
- eq->iqid = sc->sge.intrq[idx % NINTRQ(sc)].cntxt_id;
- else
- eq->iqid = sc->sge.intrq[sc->port[idx]->first_rxq].cntxt_id;
+ children = SYSCTL_CHILDREN(oid);
+
+ snprintf(name, sizeof(name), "%d", idx);
+ oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
+ NULL, "rx queue");
+ children = SYSCTL_CHILDREN(oid);
+
+ SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "abs_id",
+ CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.abs_id, 0, sysctl_uint16,
+ "I", "absolute id of the queue");
+ SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cntxt_id",
+ CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cntxt_id, 0, sysctl_uint16,
+ "I", "SGE context id of the queue");
+ SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cidx",
+ CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->iq.cidx, 0, sysctl_uint16, "I",
+ "consumer index");
+
+ children = SYSCTL_CHILDREN(oid);
+ oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, "fl", CTLFLAG_RD,
+ NULL, "freelist");
+ children = SYSCTL_CHILDREN(oid);
+
+ SYSCTL_ADD_PROC(&pi->ctx, children, OID_AUTO, "cntxt_id",
+ CTLTYPE_INT | CTLFLAG_RD, &ofld_rxq->fl.cntxt_id, 0, sysctl_uint16,
+ "I", "SGE context id of the queue");
+ SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "cidx", CTLFLAG_RD,
+ &ofld_rxq->fl.cidx, 0, "consumer index");
+ SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "pidx", CTLFLAG_RD,
+ &ofld_rxq->fl.pidx, 0, "producer index");
+
+ return (rc);
+}
+
+static int
+free_ofld_rxq(struct port_info *pi, struct sge_ofld_rxq *ofld_rxq)
+{
+ int rc;
+
+ rc = free_iq_fl(pi, &ofld_rxq->iq, &ofld_rxq->fl);
+ if (rc == 0)
+ bzero(ofld_rxq, sizeof(*ofld_rxq));
+
+ return (rc);
+}
+#endif
+
+static int
+ctrl_eq_alloc(struct adapter *sc, struct sge_eq *eq)
+{
+ int rc, cntxt_id;
+ struct fw_eq_ctrl_cmd c;
bzero(&c, sizeof(c));
@@ -1515,7 +1959,7 @@ alloc_ctrlq(struct adapter *sc, struct sge_ctrlq *ctrlq, int idx)
c.physeqid_pkd = htobe32(0);
c.fetchszm_to_iqid =
htobe32(V_FW_EQ_CTRL_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
- V_FW_EQ_CTRL_CMD_PCIECHN(sc->port[idx]->tx_chan) |
+ V_FW_EQ_CTRL_CMD_PCIECHN(eq->tx_chan) |
F_FW_EQ_CTRL_CMD_FETCHRO | V_FW_EQ_CTRL_CMD_IQID(eq->iqid));
c.dcaen_to_eqsize =
htobe32(V_FW_EQ_CTRL_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
@@ -1527,51 +1971,191 @@ alloc_ctrlq(struct adapter *sc, struct sge_ctrlq *ctrlq, int idx)
rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
if (rc != 0) {
device_printf(sc->dev,
- "failed to create control queue %d: %d\n", idx, rc);
+ "failed to create control queue %d: %d\n", eq->tx_chan, rc);
return (rc);
}
+ eq->flags |= EQ_ALLOCATED;
- eq->pidx = eq->cidx = 0;
eq->cntxt_id = G_FW_EQ_CTRL_CMD_EQID(be32toh(c.cmpliqid_eqid));
- eq->flags |= (EQ_ALLOCATED | EQ_STARTED);
+ cntxt_id = eq->cntxt_id - sc->sge.eq_start;
+ if (cntxt_id >= sc->sge.neq)
+ panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
+ cntxt_id, sc->sge.neq - 1);
+ sc->sge.eqmap[cntxt_id] = eq;
+
+ return (rc);
+}
+
+static int
+eth_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
+{
+ int rc, cntxt_id;
+ struct fw_eq_eth_cmd c;
+
+ bzero(&c, sizeof(c));
+ c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
+ F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
+ V_FW_EQ_ETH_CMD_VFN(0));
+ c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
+ F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
+ c.viid_pkd = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->viid));
+ c.fetchszm_to_iqid =
+ htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
+ V_FW_EQ_ETH_CMD_PCIECHN(eq->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
+ V_FW_EQ_ETH_CMD_IQID(eq->iqid));
+ c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
+ V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
+ V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
+ V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
+ c.eqaddr = htobe64(eq->ba);
+
+ rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
+ if (rc != 0) {
+ device_printf(pi->dev,
+ "failed to create Ethernet egress queue: %d\n", rc);
+ return (rc);
+ }
+ eq->flags |= EQ_ALLOCATED;
+
+ eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
cntxt_id = eq->cntxt_id - sc->sge.eq_start;
- KASSERT(cntxt_id < sc->sge.neq,
- ("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
- cntxt_id, sc->sge.neq - 1));
+ if (cntxt_id >= sc->sge.neq)
+ panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
+ cntxt_id, sc->sge.neq - 1);
sc->sge.eqmap[cntxt_id] = eq;
- children = SYSCTL_CHILDREN(sc->oid_ctrlq);
+ return (rc);
+}
- snprintf(name, sizeof(name), "%d", idx);
- oid = SYSCTL_ADD_NODE(&sc->ctx, children, OID_AUTO, name, CTLFLAG_RD,
- NULL, "ctrl queue");
- children = SYSCTL_CHILDREN(oid);
+#ifndef TCP_OFFLOAD_DISABLE
+static int
+ofld_eq_alloc(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
+{
+ int rc, cntxt_id;
+ struct fw_eq_ofld_cmd c;
- SYSCTL_ADD_PROC(&sc->ctx, children, OID_AUTO, "pidx",
- CTLTYPE_INT | CTLFLAG_RD, &ctrlq->eq.pidx, 0, sysctl_uint16, "I",
- "producer index");
- SYSCTL_ADD_UINT(&sc->ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
- &ctrlq->no_desc, 0,
- "# of times ctrlq ran out of hardware descriptors");
+ bzero(&c, sizeof(c));
+
+ c.op_to_vfn = htonl(V_FW_CMD_OP(FW_EQ_OFLD_CMD) | F_FW_CMD_REQUEST |
+ F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_OFLD_CMD_PFN(sc->pf) |
+ V_FW_EQ_OFLD_CMD_VFN(0));
+ c.alloc_to_len16 = htonl(F_FW_EQ_OFLD_CMD_ALLOC |
+ F_FW_EQ_OFLD_CMD_EQSTART | FW_LEN16(c));
+ c.fetchszm_to_iqid =
+ htonl(V_FW_EQ_OFLD_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
+ V_FW_EQ_OFLD_CMD_PCIECHN(eq->tx_chan) |
+ F_FW_EQ_OFLD_CMD_FETCHRO | V_FW_EQ_OFLD_CMD_IQID(eq->iqid));
+ c.dcaen_to_eqsize =
+ htobe32(V_FW_EQ_OFLD_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
+ V_FW_EQ_OFLD_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
+ V_FW_EQ_OFLD_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
+ V_FW_EQ_OFLD_CMD_EQSIZE(eq->qsize));
+ c.eqaddr = htobe64(eq->ba);
+
+ rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
+ if (rc != 0) {
+ device_printf(pi->dev,
+ "failed to create egress queue for TCP offload: %d\n", rc);
+ return (rc);
+ }
+ eq->flags |= EQ_ALLOCATED;
+
+ eq->cntxt_id = G_FW_EQ_OFLD_CMD_EQID(be32toh(c.eqid_pkd));
+ cntxt_id = eq->cntxt_id - sc->sge.eq_start;
+ if (cntxt_id >= sc->sge.neq)
+ panic("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
+ cntxt_id, sc->sge.neq - 1);
+ sc->sge.eqmap[cntxt_id] = eq;
+
+ return (rc);
+}
+#endif
+
+static int
+alloc_eq(struct adapter *sc, struct port_info *pi, struct sge_eq *eq)
+{
+ int rc;
+ size_t len;
+
+ mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
+
+ len = eq->qsize * EQ_ESIZE;
+ rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map,
+ &eq->ba, (void **)&eq->desc);
+ if (rc)
+ return (rc);
+
+ eq->cap = eq->qsize - SPG_LEN / EQ_ESIZE;
+ eq->spg = (void *)&eq->desc[eq->cap];
+ eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
+ eq->pidx = eq->cidx = 0;
+
+ switch (eq->flags & EQ_TYPEMASK) {
+ case EQ_CTRL:
+ rc = ctrl_eq_alloc(sc, eq);
+ break;
+
+ case EQ_ETH:
+ rc = eth_eq_alloc(sc, pi, eq);
+ break;
+
+#ifndef TCP_OFFLOAD_DISABLE
+ case EQ_OFLD:
+ rc = ofld_eq_alloc(sc, pi, eq);
+ break;
+#endif
+
+ default:
+ panic("%s: invalid eq type %d.", __func__,
+ eq->flags & EQ_TYPEMASK);
+ }
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to allocate egress queue(%d): %d",
+ eq->flags & EQ_TYPEMASK, rc);
+ }
+
+ eq->tx_callout.c_cpu = eq->cntxt_id % mp_ncpus;
return (rc);
}
static int
-free_ctrlq(struct adapter *sc, struct sge_ctrlq *ctrlq)
+free_eq(struct adapter *sc, struct sge_eq *eq)
{
int rc;
- struct sge_eq *eq = &ctrlq->eq;
- if (eq->flags & (EQ_ALLOCATED | EQ_STARTED)) {
- rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
+ if (eq->flags & EQ_ALLOCATED) {
+ switch (eq->flags & EQ_TYPEMASK) {
+ case EQ_CTRL:
+ rc = -t4_ctrl_eq_free(sc, sc->mbox, sc->pf, 0,
+ eq->cntxt_id);
+ break;
+
+ case EQ_ETH:
+ rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0,
+ eq->cntxt_id);
+ break;
+
+#ifndef TCP_OFFLOAD_DISABLE
+ case EQ_OFLD:
+ rc = -t4_ofld_eq_free(sc, sc->mbox, sc->pf, 0,
+ eq->cntxt_id);
+ break;
+#endif
+
+ default:
+ panic("%s: invalid eq type %d.", __func__,
+ eq->flags & EQ_TYPEMASK);
+ }
if (rc != 0) {
device_printf(sc->dev,
- "failed to free ctrl queue %p: %d\n", eq, rc);
+ "failed to free egress queue (%d): %d\n",
+ eq->flags & EQ_TYPEMASK, rc);
return (rc);
}
- eq->flags &= ~(EQ_ALLOCATED | EQ_STARTED);
+ eq->flags &= ~EQ_ALLOCATED;
}
free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc);
@@ -1579,47 +2163,77 @@ free_ctrlq(struct adapter *sc, struct sge_ctrlq *ctrlq)
if (mtx_initialized(&eq->eq_lock))
mtx_destroy(&eq->eq_lock);
- bzero(ctrlq, sizeof(*ctrlq));
+ bzero(eq, sizeof(*eq));
return (0);
}
static int
-alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
+alloc_wrq(struct adapter *sc, struct port_info *pi, struct sge_wrq *wrq,
+ struct sysctl_oid *oid)
{
- int rc, cntxt_id;
- size_t len;
+ int rc;
+ struct sysctl_ctx_list *ctx = pi ? &pi->ctx : &sc->ctx;
+ struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
+
+ rc = alloc_eq(sc, pi, &wrq->eq);
+ if (rc)
+ return (rc);
+
+ wrq->adapter = sc;
+
+ SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "cntxt_id", CTLFLAG_RD,
+ &wrq->eq.cntxt_id, 0, "SGE context id of the queue");
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "cidx",
+ CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.cidx, 0, sysctl_uint16, "I",
+ "consumer index");
+ SYSCTL_ADD_PROC(ctx, children, OID_AUTO, "pidx",
+ CTLTYPE_INT | CTLFLAG_RD, &wrq->eq.pidx, 0, sysctl_uint16, "I",
+ "producer index");
+ SYSCTL_ADD_UQUAD(ctx, children, OID_AUTO, "tx_wrs", CTLFLAG_RD,
+ &wrq->tx_wrs, "# of work requests");
+ SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
+ &wrq->no_desc, 0,
+ "# of times queue ran out of hardware descriptors");
+ SYSCTL_ADD_UINT(ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD,
+ &wrq->eq.unstalled, 0, "# of times queue recovered after stall");
+
+
+ return (rc);
+}
+
+static int
+free_wrq(struct adapter *sc, struct sge_wrq *wrq)
+{
+ int rc;
+
+ rc = free_eq(sc, &wrq->eq);
+ if (rc)
+ return (rc);
+
+ bzero(wrq, sizeof(*wrq));
+ return (0);
+}
+
+static int
+alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx,
+ struct sysctl_oid *oid)
+{
+ int rc;
struct adapter *sc = pi->adapter;
- struct fw_eq_eth_cmd c;
struct sge_eq *eq = &txq->eq;
char name[16];
- struct sysctl_oid *oid;
- struct sysctl_oid_list *children;
- struct sge_iq *intrq;
-
- txq->ifp = pi->ifp;
- TASK_INIT(&txq->resume_tx, 0, cxgbe_txq_start, txq);
-
- mtx_init(&eq->eq_lock, eq->lockname, NULL, MTX_DEF);
+ struct sysctl_oid_list *children = SYSCTL_CHILDREN(oid);
- len = eq->qsize * TX_EQ_ESIZE;
- rc = alloc_ring(sc, len, &eq->desc_tag, &eq->desc_map,
- &eq->ba, (void **)&eq->desc);
+ rc = alloc_eq(sc, pi, eq);
if (rc)
return (rc);
- eq->cap = eq->qsize - SPG_LEN / TX_EQ_ESIZE;
- eq->spg = (void *)&eq->desc[eq->cap];
- eq->avail = eq->cap - 1; /* one less to avoid cidx = pidx */
+ txq->ifp = pi->ifp;
+
txq->sdesc = malloc(eq->cap * sizeof(struct tx_sdesc), M_CXGBE,
M_ZERO | M_WAITOK);
txq->br = buf_ring_alloc(eq->qsize, M_CXGBE, M_WAITOK, &eq->eq_lock);
- intrq = &sc->sge.intrq[0];
- if (sc->flags & INTR_SHARED)
- eq->iqid = intrq[(pi->first_txq + idx) % NINTRQ(sc)].cntxt_id;
- else
- eq->iqid = intrq[pi->first_rxq + (idx % pi->nrxq)].cntxt_id;
-
rc = bus_dma_tag_create(sc->dmat, 1, 0, BUS_SPACE_MAXADDR,
BUS_SPACE_MAXADDR, NULL, NULL, 64 * 1024, TX_SGL_SEGS,
BUS_SPACE_MAXSIZE, BUS_DMA_ALLOCNOW, NULL, NULL, &txq->tx_tag);
@@ -1629,49 +2243,18 @@ alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
return (rc);
}
- rc = alloc_tx_maps(txq);
+ /*
+ * We can stuff ~10 frames in an 8-descriptor txpkts WR (8 is the SGE
+ * limit for any WR). txq->no_dmamap events shouldn't occur if maps is
+ * sized for the worst case.
+ */
+ rc = t4_alloc_tx_maps(&txq->txmaps, txq->tx_tag, eq->qsize * 10 / 8,
+ M_WAITOK);
if (rc != 0) {
device_printf(sc->dev, "failed to setup tx DMA maps: %d\n", rc);
return (rc);
}
- bzero(&c, sizeof(c));
-
- c.op_to_vfn = htobe32(V_FW_CMD_OP(FW_EQ_ETH_CMD) | F_FW_CMD_REQUEST |
- F_FW_CMD_WRITE | F_FW_CMD_EXEC | V_FW_EQ_ETH_CMD_PFN(sc->pf) |
- V_FW_EQ_ETH_CMD_VFN(0));
- c.alloc_to_len16 = htobe32(F_FW_EQ_ETH_CMD_ALLOC |
- F_FW_EQ_ETH_CMD_EQSTART | FW_LEN16(c));
- c.viid_pkd = htobe32(V_FW_EQ_ETH_CMD_VIID(pi->viid));
- c.fetchszm_to_iqid =
- htobe32(V_FW_EQ_ETH_CMD_HOSTFCMODE(X_HOSTFCMODE_STATUS_PAGE) |
- V_FW_EQ_ETH_CMD_PCIECHN(pi->tx_chan) | F_FW_EQ_ETH_CMD_FETCHRO |
- V_FW_EQ_ETH_CMD_IQID(eq->iqid));
- c.dcaen_to_eqsize = htobe32(V_FW_EQ_ETH_CMD_FBMIN(X_FETCHBURSTMIN_64B) |
- V_FW_EQ_ETH_CMD_FBMAX(X_FETCHBURSTMAX_512B) |
- V_FW_EQ_ETH_CMD_CIDXFTHRESH(X_CIDXFLUSHTHRESH_32) |
- V_FW_EQ_ETH_CMD_EQSIZE(eq->qsize));
- c.eqaddr = htobe64(eq->ba);
-
- rc = -t4_wr_mbox(sc, sc->mbox, &c, sizeof(c), &c);
- if (rc != 0) {
- device_printf(pi->dev,
- "failed to create egress queue: %d\n", rc);
- return (rc);
- }
-
- eq->pidx = eq->cidx = 0;
- eq->cntxt_id = G_FW_EQ_ETH_CMD_EQID(be32toh(c.eqid_pkd));
- eq->flags |= (EQ_ALLOCATED | EQ_STARTED);
-
- cntxt_id = eq->cntxt_id - sc->sge.eq_start;
- KASSERT(cntxt_id < sc->sge.neq,
- ("%s: eq->cntxt_id (%d) more than the max (%d)", __func__,
- cntxt_id, sc->sge.neq - 1));
- sc->sge.eqmap[cntxt_id] = eq;
-
- children = SYSCTL_CHILDREN(pi->oid_txq);
-
snprintf(name, sizeof(name), "%d", idx);
oid = SYSCTL_ADD_NODE(&pi->ctx, children, OID_AUTO, name, CTLFLAG_RD,
NULL, "tx queue");
@@ -1709,7 +2292,9 @@ alloc_txq(struct port_info *pi, struct sge_txq *txq, int idx)
SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "no_desc", CTLFLAG_RD,
&txq->no_desc, 0, "# of times txq ran out of hardware descriptors");
SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "egr_update", CTLFLAG_RD,
- &txq->egr_update, 0, "egress update notifications from the SGE");
+ &eq->egr_update, 0, "egress update notifications from the SGE");
+ SYSCTL_ADD_UINT(&pi->ctx, children, OID_AUTO, "unstalled", CTLFLAG_RD,
+ &eq->unstalled, 0, "# of times txq recovered after stall");
return (rc);
}
@@ -1721,52 +2306,20 @@ free_txq(struct port_info *pi, struct sge_txq *txq)
struct adapter *sc = pi->adapter;
struct sge_eq *eq = &txq->eq;
- if (eq->flags & (EQ_ALLOCATED | EQ_STARTED)) {
-
- /*
- * Wait for the response to a credit flush if there's one
- * pending. Clearing the flag tells handle_sge_egr_update or
- * cxgbe_txq_start (depending on how far the response has made
- * it) that they should ignore the response and wake up free_txq
- * instead.
- *
- * The interface has been marked down by the time we get here
- * (both IFF_UP and IFF_DRV_RUNNING cleared). qflush has
- * emptied the tx buf_rings and we know nothing new is being
- * queued for tx so we don't have to worry about a new credit
- * flush request.
- */
- TXQ_LOCK(txq);
- if (eq->flags & EQ_CRFLUSHED) {
- eq->flags &= ~EQ_CRFLUSHED;
- msleep(txq, &eq->eq_lock, 0, "crflush", 0);
- }
- TXQ_UNLOCK(txq);
-
- rc = -t4_eth_eq_free(sc, sc->mbox, sc->pf, 0, eq->cntxt_id);
- if (rc != 0) {
- device_printf(pi->dev,
- "failed to free egress queue %p: %d\n", eq, rc);
- return (rc);
- }
- eq->flags &= ~(EQ_ALLOCATED | EQ_STARTED);
- }
-
- free_ring(sc, eq->desc_tag, eq->desc_map, eq->ba, eq->desc);
+ rc = free_eq(sc, eq);
+ if (rc)
+ return (rc);
free(txq->sdesc, M_CXGBE);
- if (txq->maps)
- free_tx_maps(txq);
+ if (txq->txmaps.maps)
+ t4_free_tx_maps(&txq->txmaps, txq->tx_tag);
buf_ring_free(txq->br, M_CXGBE);
if (txq->tx_tag)
bus_dma_tag_destroy(txq->tx_tag);
- if (mtx_initialized(&eq->eq_lock))
- mtx_destroy(&eq->eq_lock);
-
bzero(txq, sizeof(*txq));
return (0);
}
@@ -1822,11 +2375,13 @@ ring_fl_db(struct adapter *sc, struct sge_fl *fl)
}
/*
- * Fill up the freelist by upto nbufs and ring its doorbell if the number of
- * buffers ready to be handed to the hardware >= dbthresh.
+ * Fill up the freelist by upto nbufs and maybe ring its doorbell.
+ *
+ * Returns non-zero to indicate that it should be added to the list of starving
+ * freelists.
*/
-static void
-refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs, int dbthresh)
+static int
+refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs)
{
__be64 *d = &fl->desc[fl->pidx];
struct fl_sdesc *sd = &fl->sdesc[fl->pidx];
@@ -1837,7 +2392,7 @@ refill_fl(struct adapter *sc, struct sge_fl *fl, int nbufs, int dbthresh)
FL_LOCK_ASSERT_OWNED(fl);
- if (nbufs < 0 || nbufs > fl->needed)
+ if (nbufs > fl->needed)
nbufs = fl->needed;
while (nbufs--) {
@@ -1918,8 +2473,35 @@ recycled:
}
}
- if (fl->pending >= dbthresh)
+ if (fl->pending >= 8)
ring_fl_db(sc, fl);
+
+ return (FL_RUNNING_LOW(fl) && !(fl->flags & FL_STARVING));
+}
+
+/*
+ * Attempt to refill all starving freelists.
+ */
+static void
+refill_sfl(void *arg)
+{
+ struct adapter *sc = arg;
+ struct sge_fl *fl, *fl_temp;
+
+ mtx_lock(&sc->sfl_lock);
+ TAILQ_FOREACH_SAFE(fl, &sc->sfl, link, fl_temp) {
+ FL_LOCK(fl);
+ refill_fl(sc, fl, 64);
+ if (FL_NOT_RUNNING_LOW(fl) || fl->flags & FL_DOOMED) {
+ TAILQ_REMOVE(&sc->sfl, fl, link);
+ fl->flags &= ~FL_STARVING;
+ }
+ FL_UNLOCK(fl);
+ }
+
+ if (!TAILQ_EMPTY(&sc->sfl))
+ callout_schedule(&sc->sfl_callout, hz / 5);
+ mtx_unlock(&sc->sfl_lock);
}
static int
@@ -1993,27 +2575,22 @@ free_fl_sdesc(struct sge_fl *fl)
fl->sdesc = NULL;
}
-static int
-alloc_tx_maps(struct sge_txq *txq)
+int
+t4_alloc_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag, int count,
+ int flags)
{
struct tx_map *txm;
- int i, rc, count;
+ int i, rc;
- /*
- * We can stuff ~10 frames in an 8-descriptor txpkts WR (8 is the SGE
- * limit for any WR). txq->no_dmamap events shouldn't occur if maps is
- * sized for the worst case.
- */
- count = txq->eq.qsize * 10 / 8;
- txq->map_total = txq->map_avail = count;
- txq->map_cidx = txq->map_pidx = 0;
+ txmaps->map_total = txmaps->map_avail = count;
+ txmaps->map_cidx = txmaps->map_pidx = 0;
- txq->maps = malloc(count * sizeof(struct tx_map), M_CXGBE,
- M_ZERO | M_WAITOK);
+ txmaps->maps = malloc(count * sizeof(struct tx_map), M_CXGBE,
+ M_ZERO | flags);
- txm = txq->maps;
+ txm = txmaps->maps;
for (i = 0; i < count; i++, txm++) {
- rc = bus_dmamap_create(txq->tx_tag, 0, &txm->map);
+ rc = bus_dmamap_create(tx_tag, 0, &txm->map);
if (rc != 0)
goto failed;
}
@@ -2022,36 +2599,36 @@ alloc_tx_maps(struct sge_txq *txq)
failed:
while (--i >= 0) {
txm--;
- bus_dmamap_destroy(txq->tx_tag, txm->map);
+ bus_dmamap_destroy(tx_tag, txm->map);
}
- KASSERT(txm == txq->maps, ("%s: EDOOFUS", __func__));
+ KASSERT(txm == txmaps->maps, ("%s: EDOOFUS", __func__));
- free(txq->maps, M_CXGBE);
- txq->maps = NULL;
+ free(txmaps->maps, M_CXGBE);
+ txmaps->maps = NULL;
return (rc);
}
-static void
-free_tx_maps(struct sge_txq *txq)
+void
+t4_free_tx_maps(struct tx_maps *txmaps, bus_dma_tag_t tx_tag)
{
struct tx_map *txm;
int i;
- txm = txq->maps;
- for (i = 0; i < txq->map_total; i++, txm++) {
+ txm = txmaps->maps;
+ for (i = 0; i < txmaps->map_total; i++, txm++) {
if (txm->m) {
- bus_dmamap_unload(txq->tx_tag, txm->map);
+ bus_dmamap_unload(tx_tag, txm->map);
m_freem(txm->m);
txm->m = NULL;
}
- bus_dmamap_destroy(txq->tx_tag, txm->map);
+ bus_dmamap_destroy(tx_tag, txm->map);
}
- free(txq->maps, M_CXGBE);
- txq->maps = NULL;
+ free(txmaps->maps, M_CXGBE);
+ txmaps->maps = NULL;
}
/*
@@ -2060,7 +2637,7 @@ free_tx_maps(struct sge_txq *txq)
* of immediate data.
*/
#define IMM_LEN ( \
- 2 * TX_EQ_ESIZE \
+ 2 * EQ_ESIZE \
- sizeof(struct fw_eth_tx_pkt_wr) \
- sizeof(struct cpl_tx_pkt_core))
@@ -2078,6 +2655,7 @@ get_pkt_sgl(struct sge_txq *txq, struct mbuf **fp, struct sgl *sgl,
int sgl_only)
{
struct mbuf *m = *fp;
+ struct tx_maps *txmaps;
struct tx_map *txm;
int rc, defragged = 0, n;
@@ -2091,11 +2669,12 @@ start: sgl->nsegs = 0;
if (m->m_pkthdr.len <= IMM_LEN && !sgl_only)
return (0); /* nsegs = 0 tells caller to use imm. tx */
- if (txq->map_avail == 0) {
+ txmaps = &txq->txmaps;
+ if (txmaps->map_avail == 0) {
txq->no_dmamap++;
return (ENOMEM);
}
- txm = &txq->maps[txq->map_pidx];
+ txm = &txmaps->maps[txmaps->map_pidx];
if (m->m_pkthdr.tso_segsz && m->m_len < 50) {
*fp = m_pullup(m, 50);
@@ -2119,9 +2698,9 @@ start: sgl->nsegs = 0;
return (rc);
txm->m = m;
- txq->map_avail--;
- if (++txq->map_pidx == txq->map_total)
- txq->map_pidx = 0;
+ txmaps->map_avail--;
+ if (++txmaps->map_pidx == txmaps->map_total)
+ txmaps->map_pidx = 0;
KASSERT(sgl->nsegs > 0 && sgl->nsegs <= TX_SGL_SEGS,
("%s: bad DMA mapping (%d segments)", __func__, sgl->nsegs));
@@ -2145,6 +2724,7 @@ start: sgl->nsegs = 0;
static int
free_pkt_sgl(struct sge_txq *txq, struct sgl *sgl)
{
+ struct tx_maps *txmaps;
struct tx_map *txm;
TXQ_LOCK_ASSERT_OWNED(txq);
@@ -2152,15 +2732,17 @@ free_pkt_sgl(struct sge_txq *txq, struct sgl *sgl)
if (sgl->nsegs == 0)
return (0); /* didn't use any map */
+ txmaps = &txq->txmaps;
+
/* 1 pkt uses exactly 1 map, back it out */
- txq->map_avail++;
- if (txq->map_pidx > 0)
- txq->map_pidx--;
+ txmaps->map_avail++;
+ if (txmaps->map_pidx > 0)
+ txmaps->map_pidx--;
else
- txq->map_pidx = txq->map_total - 1;
+ txmaps->map_pidx = txmaps->map_total - 1;
- txm = &txq->maps[txq->map_pidx];
+ txm = &txmaps->maps[txmaps->map_pidx];
bus_dmamap_unload(txq->tx_tag, txm->map);
txm->m = NULL;
@@ -2206,11 +2788,14 @@ write_txpkt_wr(struct port_info *pi, struct sge_txq *txq, struct mbuf *m,
/* Firmware work request header */
wr = (void *)&eq->desc[eq->pidx];
wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKT_WR) |
- V_FW_WR_IMMDLEN(ctrl));
+ V_FW_ETH_TX_PKT_WR_IMMDLEN(ctrl));
ctrl = V_FW_WR_LEN16(howmany(nflits, 2));
- if (eq->avail == ndesc && !(eq->flags & EQ_CRFLUSHED)) {
- ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
- eq->flags |= EQ_CRFLUSHED;
+ if (eq->avail == ndesc) {
+ if (!(eq->flags & EQ_CRFLUSHED)) {
+ ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
+ eq->flags |= EQ_CRFLUSHED;
+ }
+ eq->flags |= EQ_STALLED;
}
wr->equiq_to_len16 = htobe32(ctrl);
@@ -2325,6 +2910,8 @@ add_to_txpkts(struct port_info *pi, struct sge_txq *txq, struct txpkts *txpkts,
TXQ_LOCK_ASSERT_OWNED(txq);
+ KASSERT(sgl->nsegs, ("%s: can't coalesce imm data", __func__));
+
if (txpkts->npkt > 0) {
flits = TXPKTS_PKT_HDR + sgl->nflits;
can_coalesce = m->m_pkthdr.tso_segsz == 0 &&
@@ -2397,12 +2984,14 @@ write_txpkts_wr(struct sge_txq *txq, struct txpkts *txpkts)
ndesc = howmany(txpkts->nflits, 8);
wr = (void *)&eq->desc[eq->pidx];
- wr->op_immdlen = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR) |
- V_FW_WR_IMMDLEN(0)); /* immdlen does not matter in this WR */
+ wr->op_pkd = htobe32(V_FW_WR_OP(FW_ETH_TX_PKTS_WR));
ctrl = V_FW_WR_LEN16(howmany(txpkts->nflits, 2));
- if (eq->avail == ndesc && !(eq->flags & EQ_CRFLUSHED)) {
- ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
- eq->flags |= EQ_CRFLUSHED;
+ if (eq->avail == ndesc) {
+ if (!(eq->flags & EQ_CRFLUSHED)) {
+ ctrl |= F_FW_WR_EQUEQ | F_FW_WR_EQUIQ;
+ eq->flags |= EQ_CRFLUSHED;
+ }
+ eq->flags |= EQ_STALLED;
}
wr->equiq_to_len16 = htobe32(ctrl);
wr->plen = htobe16(txpkts->plen);
@@ -2616,7 +3205,7 @@ reclaimable(struct sge_eq *eq)
unsigned int cidx;
cidx = eq->spg->cidx; /* stable snapshot */
- cidx = be16_to_cpu(cidx);
+ cidx = be16toh(cidx);
if (cidx >= eq->cidx)
return (cidx - eq->cidx);
@@ -2634,11 +3223,12 @@ static int
reclaim_tx_descs(struct sge_txq *txq, int can_reclaim, int n)
{
struct tx_sdesc *txsd;
+ struct tx_maps *txmaps;
struct tx_map *txm;
unsigned int reclaimed, maps;
struct sge_eq *eq = &txq->eq;
- EQ_LOCK_ASSERT_OWNED(eq);
+ TXQ_LOCK_ASSERT_OWNED(txq);
if (can_reclaim == 0)
can_reclaim = reclaimable(eq);
@@ -2665,7 +3255,8 @@ reclaim_tx_descs(struct sge_txq *txq, int can_reclaim, int n)
eq->cidx -= eq->cap;
}
- txm = &txq->maps[txq->map_cidx];
+ txmaps = &txq->txmaps;
+ txm = &txmaps->maps[txmaps->map_cidx];
if (maps)
prefetch(txm->m);
@@ -2673,16 +3264,16 @@ reclaim_tx_descs(struct sge_txq *txq, int can_reclaim, int n)
KASSERT(eq->avail < eq->cap, /* avail tops out at (cap - 1) */
("%s: too many descriptors available", __func__));
- txq->map_avail += maps;
- KASSERT(txq->map_avail <= txq->map_total,
+ txmaps->map_avail += maps;
+ KASSERT(txmaps->map_avail <= txmaps->map_total,
("%s: too many maps available", __func__));
while (maps--) {
struct tx_map *next;
next = txm + 1;
- if (__predict_false(txq->map_cidx + 1 == txq->map_total))
- next = txq->maps;
+ if (__predict_false(txmaps->map_cidx + 1 == txmaps->map_total))
+ next = txmaps->maps;
prefetch(next->m);
bus_dmamap_unload(txq->tx_tag, txm->map);
@@ -2690,8 +3281,8 @@ reclaim_tx_descs(struct sge_txq *txq, int can_reclaim, int n)
txm->m = NULL;
txm = next;
- if (__predict_false(++txq->map_cidx == txq->map_total))
- txq->map_cidx = 0;
+ if (__predict_false(++txmaps->map_cidx == txmaps->map_total))
+ txmaps->map_cidx = 0;
}
return (reclaimed);
@@ -2704,6 +3295,7 @@ write_eqflush_wr(struct sge_eq *eq)
EQ_LOCK_ASSERT_OWNED(eq);
KASSERT(eq->avail > 0, ("%s: no descriptors left.", __func__));
+ KASSERT(!(eq->flags & EQ_CRFLUSHED), ("%s: flushed already", __func__));
wr = (void *)&eq->desc[eq->pidx];
bzero(wr, sizeof(*wr));
@@ -2711,7 +3303,7 @@ write_eqflush_wr(struct sge_eq *eq)
wr->equiq_to_len16 = htobe32(V_FW_WR_LEN16(sizeof(*wr) / 16) |
F_FW_WR_EQUEQ | F_FW_WR_EQUIQ);
- eq->flags |= EQ_CRFLUSHED;
+ eq->flags |= (EQ_CRFLUSHED | EQ_STALLED);
eq->pending++;
eq->avail--;
if (++eq->pidx == eq->cap)
@@ -2743,118 +3335,73 @@ get_flit(bus_dma_segment_t *sgl, int nsegs, int idx)
}
static void
-set_fl_tag_idx(struct sge_fl *fl, int mtu)
+set_fl_tag_idx(struct sge_fl *fl, int bufsize)
{
int i;
- FL_LOCK_ASSERT_OWNED(fl);
-
for (i = 0; i < FL_BUF_SIZES - 1; i++) {
- if (FL_BUF_SIZE(i) >= (mtu + FL_PKTSHIFT))
+ if (FL_BUF_SIZE(i) >= bufsize)
break;
}
fl->tag_idx = i;
}
-static int
-handle_sge_egr_update(struct adapter *sc, const struct cpl_sge_egr_update *cpl)
-{
- unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
- struct sge *s = &sc->sge;
- struct sge_txq *txq;
- struct port_info *pi;
-
- txq = (void *)s->eqmap[qid - s->eq_start];
- TXQ_LOCK(txq);
- if (txq->eq.flags & EQ_CRFLUSHED) {
- pi = txq->ifp->if_softc;
- taskqueue_enqueue(pi->tq, &txq->resume_tx);
- txq->egr_update++;
- } else
- wakeup_one(txq); /* txq is going away, wakeup free_txq */
- TXQ_UNLOCK(txq);
-
- return (0);
-}
-
static void
-handle_cpl(struct adapter *sc, struct sge_iq *iq)
+add_fl_to_sfl(struct adapter *sc, struct sge_fl *fl)
{
- const struct rss_header *rss = (const void *)iq->cdesc;
- const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
-
- switch (rss->opcode) {
- case CPL_FW4_MSG:
- case CPL_FW6_MSG:
- if (cpl->type == FW6_TYPE_CMD_RPL)
- t4_handle_fw_rpl(sc, cpl->data);
- break;
-
- case CPL_SGE_EGR_UPDATE:
- handle_sge_egr_update(sc, (const void *)cpl);
- break;
-
- case CPL_SET_TCB_RPL:
- filter_rpl(sc, (const void *)cpl);
- break;
-
- default:
- panic("%s: unexpected CPL opcode 0x%x", __func__, rss->opcode);
+ mtx_lock(&sc->sfl_lock);
+ FL_LOCK(fl);
+ if ((fl->flags & FL_DOOMED) == 0) {
+ fl->flags |= FL_STARVING;
+ TAILQ_INSERT_TAIL(&sc->sfl, fl, link);
+ callout_reset(&sc->sfl_callout, hz / 5, refill_sfl, sc);
}
+ FL_UNLOCK(fl);
+ mtx_unlock(&sc->sfl_lock);
}
-/*
- * m0 is freed on successful transmission.
- */
static int
-ctrl_tx(struct adapter *sc, struct sge_ctrlq *ctrlq, struct mbuf *m0)
+handle_sge_egr_update(struct sge_iq *iq, const struct rss_header *rss,
+ struct mbuf *m)
{
- struct sge_eq *eq = &ctrlq->eq;
- int rc = 0, ndesc;
- int can_reclaim;
- caddr_t dst;
- struct mbuf *m;
-
- M_ASSERTPKTHDR(m0);
+ const struct cpl_sge_egr_update *cpl = (const void *)(rss + 1);
+ unsigned int qid = G_EGR_QID(ntohl(cpl->opcode_qid));
+ struct adapter *sc = iq->adapter;
+ struct sge *s = &sc->sge;
+ struct sge_eq *eq;
- if (m0->m_pkthdr.len > SGE_MAX_WR_LEN) {
- log(LOG_ERR, "%s: %s work request too long (%d)",
- device_get_nameunit(sc->dev), __func__, m0->m_pkthdr.len);
- return (EMSGSIZE);
- }
- ndesc = howmany(m0->m_pkthdr.len, CTRL_EQ_ESIZE);
+ KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
+ rss->opcode));
+ eq = s->eqmap[qid - s->eq_start];
EQ_LOCK(eq);
+ KASSERT(eq->flags & EQ_CRFLUSHED,
+ ("%s: unsolicited egress update", __func__));
+ eq->flags &= ~EQ_CRFLUSHED;
+ eq->egr_update++;
+
+ if (__predict_false(eq->flags & EQ_DOOMED))
+ wakeup_one(eq);
+ else if (eq->flags & EQ_STALLED && can_resume_tx(eq))
+ taskqueue_enqueue(sc->tq[eq->tx_chan], &eq->tx_task);
+ EQ_UNLOCK(eq);
- can_reclaim = reclaimable(eq);
- eq->cidx += can_reclaim;
- eq->avail += can_reclaim;
- if (__predict_false(eq->cidx >= eq->cap))
- eq->cidx -= eq->cap;
-
- if (eq->avail < ndesc) {
- rc = EAGAIN;
- ctrlq->no_desc++;
- goto failed;
- }
+ return (0);
+}
- dst = (void *)&eq->desc[eq->pidx];
- for (m = m0; m; m = m->m_next)
- copy_to_txd(eq, mtod(m, caddr_t), &dst, m->m_len);
+static int
+handle_fw_rpl(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
+{
+ const struct cpl_fw6_msg *cpl = (const void *)(rss + 1);
- eq->pidx += ndesc;
- if (__predict_false(eq->pidx >= eq->cap))
- eq->pidx -= eq->cap;
+ KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
+ rss->opcode));
- eq->pending += ndesc;
- ring_eq_db(sc, eq);
-failed:
- EQ_UNLOCK(eq);
- if (rc == 0)
- m_freem(m0);
+ if (cpl->type == FW6_TYPE_CMD_RPL)
+ t4_handle_fw_rpl(iq->adapter, cpl->data);
- return (rc);
+ return (0);
}
static int
diff --git a/sys/modules/cxgbe/Makefile b/sys/modules/cxgbe/Makefile
index 2e83446..1d69f76 100644
--- a/sys/modules/cxgbe/Makefile
+++ b/sys/modules/cxgbe/Makefile
@@ -3,5 +3,6 @@
#
SUBDIR = if_cxgbe
+SUBDIR+= firmware
.include <bsd.subdir.mk>
diff --git a/sys/modules/cxgbe/firmware/Makefile b/sys/modules/cxgbe/firmware/Makefile
new file mode 100644
index 0000000..035de02
--- /dev/null
+++ b/sys/modules/cxgbe/firmware/Makefile
@@ -0,0 +1,27 @@
+#
+# $FreeBSD$
+#
+
+T4FW = ${.CURDIR}/../../../dev/cxgbe/firmware
+.PATH: ${T4FW}
+
+KMOD = t4fw_cfg
+FIRMWS = ${KMOD}.txt:${KMOD}:1.0.0.0
+
+# You can have additional configuration files in the ${T4FW} directory.
+# t4fw_cfg_<name>.txt
+CFG_FILES != cd ${T4FW} && echo ${KMOD}_*.txt
+.for F in ${CFG_FILES}
+.if exists(${F})
+FIRMWS += ${F}:${F:C/.txt//}:1.0.0.0
+.endif
+.endfor
+
+# The firmware binary is optional.
+# t4fw-<a>.<b>.<c>.<d>.bin
+FW_BIN != cd ${T4FW} && echo t4fw-*.bin
+.if exists(${FW_BIN})
+FIRMWS += ${FW_BIN}:t4fw:${FW_BIN:C/t4fw-//:C/.bin//}
+.endif
+
+.include <bsd.kmod.mk>
diff --git a/tools/tools/cxgbetool/cxgbetool.c b/tools/tools/cxgbetool/cxgbetool.c
index da6bfba..06fa50b 100644
--- a/tools/tools/cxgbetool/cxgbetool.c
+++ b/tools/tools/cxgbetool/cxgbetool.c
@@ -396,12 +396,12 @@ do_show_info_header(uint32_t mode)
printf (" Port");
break;
- case T4_FILTER_OVLAN:
- printf (" vld:oVLAN");
+ case T4_FILTER_VNIC:
+ printf (" vld:VNIC");
break;
- case T4_FILTER_IVLAN:
- printf (" vld:iVLAN");
+ case T4_FILTER_VLAN:
+ printf (" vld:VLAN");
break;
case T4_FILTER_IP_TOS:
@@ -653,18 +653,18 @@ do_show_one_filter_info(struct t4_filter *t, uint32_t mode)
printf(" %1d/%1d", t->fs.val.iport, t->fs.mask.iport);
break;
- case T4_FILTER_OVLAN:
+ case T4_FILTER_VNIC:
printf(" %1d:%1x:%02x/%1d:%1x:%02x",
- t->fs.val.ovlan_vld, (t->fs.val.ovlan >> 7) & 0x7,
- t->fs.val.ovlan & 0x7f, t->fs.mask.ovlan_vld,
- (t->fs.mask.ovlan >> 7) & 0x7,
- t->fs.mask.ovlan & 0x7f);
+ t->fs.val.vnic_vld, (t->fs.val.vnic >> 7) & 0x7,
+ t->fs.val.vnic & 0x7f, t->fs.mask.vnic_vld,
+ (t->fs.mask.vnic >> 7) & 0x7,
+ t->fs.mask.vnic & 0x7f);
break;
- case T4_FILTER_IVLAN:
+ case T4_FILTER_VLAN:
printf(" %1d:%04x/%1d:%04x",
- t->fs.val.ivlan_vld, t->fs.val.ivlan,
- t->fs.mask.ivlan_vld, t->fs.mask.ivlan);
+ t->fs.val.vlan_vld, t->fs.val.vlan,
+ t->fs.mask.vlan_vld, t->fs.mask.vlan);
break;
case T4_FILTER_IP_TOS:
@@ -830,11 +830,11 @@ get_filter_mode(void)
if (mode & T4_FILTER_IP_TOS)
printf("tos ");
- if (mode & T4_FILTER_IVLAN)
- printf("ivlan ");
+ if (mode & T4_FILTER_VLAN)
+ printf("vlan ");
- if (mode & T4_FILTER_OVLAN)
- printf("ovlan ");
+ if (mode & T4_FILTER_VNIC)
+ printf("vnic ");
if (mode & T4_FILTER_PORT)
printf("iport ");
@@ -868,11 +868,12 @@ set_filter_mode(int argc, const char *argv[])
if (!strcmp(argv[0], "tos"))
mode |= T4_FILTER_IP_TOS;
- if (!strcmp(argv[0], "ivlan"))
- mode |= T4_FILTER_IVLAN;
+ if (!strcmp(argv[0], "vlan"))
+ mode |= T4_FILTER_VLAN;
- if (!strcmp(argv[0], "ovlan"))
- mode |= T4_FILTER_OVLAN;
+ if (!strcmp(argv[0], "ovlan") ||
+ !strcmp(argv[0], "vnic"))
+ mode |= T4_FILTER_VNIC;
if (!strcmp(argv[0], "iport"))
mode |= T4_FILTER_PORT;
@@ -936,15 +937,20 @@ set_filter(uint32_t idx, int argc, const char *argv[])
t.fs.val.iport = val;
t.fs.mask.iport = mask;
} else if (!parse_val_mask("ovlan", args, &val, &mask)) {
- t.fs.val.ovlan = val;
- t.fs.mask.ovlan = mask;
- t.fs.val.ovlan_vld = 1;
- t.fs.mask.ovlan_vld = 1;
- } else if (!parse_val_mask("ivlan", args, &val, &mask)) {
- t.fs.val.ivlan = val;
- t.fs.mask.ivlan = mask;
- t.fs.val.ivlan_vld = 1;
- t.fs.mask.ivlan_vld = 1;
+ t.fs.val.vnic = val;
+ t.fs.mask.vnic = mask;
+ t.fs.val.vnic_vld = 1;
+ t.fs.mask.vnic_vld = 1;
+ } else if (!parse_val_mask("vnic", args, &val, &mask)) {
+ t.fs.val.vnic = val;
+ t.fs.mask.vnic = mask;
+ t.fs.val.vnic_vld = 1;
+ t.fs.mask.vnic_vld = 1;
+ } else if (!parse_val_mask("vlan", args, &val, &mask)) {
+ t.fs.val.vlan = val;
+ t.fs.mask.vlan = mask;
+ t.fs.val.vlan_vld = 1;
+ t.fs.mask.vlan_vld = 1;
} else if (!parse_val_mask("tos", args, &val, &mask)) {
t.fs.val.tos = val;
t.fs.mask.tos = mask;
OpenPOWER on IntegriCloud