- update firmware to 5.0

- add support for T3C
- add DDP support (zero-copy receive)
- fix TOE transmit of large requests
- fix shutdown so that sockets don't remain in CLOSING state indefinitely
- register listeners when an interface is brought up after tom is loaded
- fix setting of multicast filter
- enable link at device attach
- exit tick handler if shutdown is in progress
- add helper for logging TCB
- add sysctls for dumping transmit queues

- note that TOE wxill not be MFC'd until after 7.0 has been finalized

MFC after: 3 days

13 files changed, 1881 insertions, 644 deletions

diff --git a/sys/dev/cxgb/common/cxgb_ael1002.c b/sys/dev/cxgb/common/cxgb_ael1002.c
index c570ed3..a5a258b 100644
--- a/sys/dev/cxgb/common/cxgb_ael1002.c
+++ b/sys/dev/cxgb/common/cxgb_ael1002.c
@@ -36,6 +36,9 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/cxgb_include.h>
 #endif
 
+#undef msleep
+#define msleep t3_os_sleep
+
 enum {
 	AEL100X_TX_DISABLE  = 9,
 	AEL100X_TX_CONFIG1  = 0xc002,
@@ -52,9 +55,9 @@ static void ael100x_txon(struct cphy *phy)
 {
 	int tx_on_gpio = phy->addr == 0 ? F_GPIO7_OUT_VAL : F_GPIO2_OUT_VAL;
 
-	t3_os_sleep(100);
+	msleep(100);
 	t3_set_reg_field(phy->adapter, A_T3DBG_GPIO_EN, 0, tx_on_gpio);
-	t3_os_sleep(30);
+	msleep(30);
 }
 
 static int ael1002_power_down(struct cphy *phy, int enable)
@@ -115,7 +118,6 @@ static int ael100x_get_link_status(struct cphy *phy, int *link_ok,
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops ael1002_ops = {
-	NULL,
 	ael1002_reset,
 	ael1002_intr_noop,
 	ael1002_intr_noop,
@@ -141,11 +143,14 @@ static struct cphy_ops ael1002_ops = {
 };
 #endif
 
-void t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			 const struct mdio_ops *mdio_ops)
+int t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+			const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &ael1002_ops, mdio_ops);
+	cphy_init(phy, adapter, phy_addr, &ael1002_ops, mdio_ops,
+		  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE,
+		  "10GBASE-XR");
 	ael100x_txon(phy);
+	return 0;
 }
 
 static int ael1006_reset(struct cphy *phy, int wait)
@@ -188,7 +193,6 @@ static int ael1006_power_down(struct cphy *phy, int enable)
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops ael1006_ops = {
-	NULL,
 	ael1006_reset,
 	ael1006_intr_enable,
 	ael1006_intr_disable,
@@ -214,16 +218,18 @@ static struct cphy_ops ael1006_ops = {
 };
 #endif
 
-void t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			 const struct mdio_ops *mdio_ops)
+int t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+			const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &ael1006_ops, mdio_ops);
+	cphy_init(phy, adapter, phy_addr, &ael1006_ops, mdio_ops,
+		  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_FIBRE,
+		  "10GBASE-SR");
 	ael100x_txon(phy);
+	return 0;
 }
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops qt2045_ops = {
-	NULL,
 	ael1006_reset,
 	ael1006_intr_enable,
 	ael1006_intr_disable,
@@ -249,12 +255,14 @@ static struct cphy_ops qt2045_ops = {
 };
 #endif
 
-void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			const struct mdio_ops *mdio_ops)
+int t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+		       const struct mdio_ops *mdio_ops)
 {
 	unsigned int stat;
 
-	cphy_init(phy, adapter, phy_addr, &qt2045_ops, mdio_ops);
+	cphy_init(phy, adapter, phy_addr, &qt2045_ops, mdio_ops,
+		  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_TP,
+		  "10GBASE-CX4");
 
 	/*
 	 * Some cards where the PHY is supposed to be at address 0 actually
@@ -263,6 +271,7 @@ void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 	if (!phy_addr && !mdio_read(phy, MDIO_DEV_PMA_PMD, MII_BMSR, &stat) &&
 	    stat == 0xffff)
 		phy->addr = 1;
+	return 0;
 }
 
 static int xaui_direct_reset(struct cphy *phy, int wait)
@@ -300,7 +309,6 @@ static int xaui_direct_power_down(struct cphy *phy, int enable)
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops xaui_direct_ops = {
-	NULL,
 	xaui_direct_reset,
 	ael1002_intr_noop,
 	ael1002_intr_noop,
@@ -326,8 +334,11 @@ static struct cphy_ops xaui_direct_ops = {
 };
 #endif
 
-void t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			     const struct mdio_ops *mdio_ops)
+int t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+			    const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &xaui_direct_ops, mdio_ops);
+	cphy_init(phy, adapter, phy_addr, &xaui_direct_ops, mdio_ops,
+		  SUPPORTED_10000baseT_Full | SUPPORTED_AUI | SUPPORTED_TP,
+		  "10GBASE-CX4");
+	return 0;
 }
diff --git a/sys/dev/cxgb/common/cxgb_common.h b/sys/dev/cxgb/common/cxgb_common.h
index 0b4b6aa..f1b5075 100644
--- a/sys/dev/cxgb/common/cxgb_common.h
+++ b/sys/dev/cxgb/common/cxgb_common.h
@@ -98,8 +98,8 @@ enum {
 	    (((x) >> S_TP_VERSION_MICRO) & M_TP_VERSION_MICRO)
 
 enum {
-	FW_VERSION_MAJOR = 4,
-	FW_VERSION_MINOR = 7,
+	FW_VERSION_MAJOR = 5,
+	FW_VERSION_MINOR = 0,
 	FW_VERSION_MICRO = 0
 };
 
@@ -157,10 +157,10 @@ struct adapter_info {
 };
 
 struct port_type_info {
-	void (*phy_prep)(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			 const struct mdio_ops *ops);
-	unsigned int caps;
-	const char *desc;
+	int (*phy_prep)(struct cphy *phy, adapter_t *adapter, int phy_addr,
+			const struct mdio_ops *ops);
+
+
 };
 
 struct mc5_stats {
@@ -508,7 +508,6 @@ enum {
 
 /* PHY operations */
 struct cphy_ops {
-	void (*destroy)(struct cphy *phy);
 	int (*reset)(struct cphy *phy, int wait);
 
 	int (*intr_enable)(struct cphy *phy);
@@ -530,7 +529,9 @@ struct cphy_ops {
 /* A PHY instance */
 struct cphy {
 	int addr;                            /* PHY address */
+	unsigned int caps;                   /* PHY capabilities */
 	adapter_t *adapter;                  /* associated adapter */
+	const char *desc;                    /* PHY description */
 	unsigned long fifo_errors;           /* FIFO over/under-flows */
 	const struct cphy_ops *ops;          /* PHY operations */
 	int (*mdio_read)(adapter_t *adapter, int phy_addr, int mmd_addr,
@@ -555,10 +556,13 @@ static inline int mdio_write(struct cphy *phy, int mmd, int reg,
 /* Convenience initializer */
 static inline void cphy_init(struct cphy *phy, adapter_t *adapter,
 			     int phy_addr, struct cphy_ops *phy_ops,
-			     const struct mdio_ops *mdio_ops)
+                             const struct mdio_ops *mdio_ops, unsigned int caps,
+                             const char *desc)
 {
 	phy->adapter = adapter;
 	phy->addr    = phy_addr;
+	phy->caps    = caps;
+	phy->desc    = desc;
 	phy->ops     = phy_ops;
 	if (mdio_ops) {
 		phy->mdio_read  = mdio_ops->read;
@@ -667,11 +671,12 @@ int t3_seeprom_wp(adapter_t *adapter, int enable);
 int t3_read_flash(adapter_t *adapter, unsigned int addr, unsigned int nwords,
 		  u32 *data, int byte_oriented);
 int t3_get_tp_version(adapter_t *adapter, u32 *vers);
-int t3_check_tpsram_version(adapter_t *adapter);
+int t3_check_tpsram_version(adapter_t *adapter, int *must_load);
 int t3_check_tpsram(adapter_t *adapter, const u8 *tp_ram, unsigned int size);
 int t3_load_fw(adapter_t *adapter, const const u8 *fw_data, unsigned int size);
+int t3_load_boot(adapter_t *adapter, u8 *boot_data, unsigned int size);
 int t3_get_fw_version(adapter_t *adapter, u32 *vers);
-int t3_check_fw_version(adapter_t *adapter);
+int t3_check_fw_version(adapter_t *adapter, int *must_load);
 int t3_init_hw(adapter_t *adapter, u32 fw_params);
 void mac_prep(struct cmac *mac, adapter_t *adapter, int index);
 void early_hw_init(adapter_t *adapter, const struct adapter_info *ai);
@@ -769,18 +774,21 @@ int t3_vsc7323_set_mtu(adapter_t *adap, unsigned int mtu, int port);
 int t3_vsc7323_set_addr(adapter_t *adap, u8 addr[6], int port);
 int t3_vsc7323_enable(adapter_t *adap, int port, int which);
 int t3_vsc7323_disable(adapter_t *adap, int port, int which);
+
+int t3_phy_advertise_fiber(struct cphy *phy, unsigned int advert);
+
 const struct mac_stats *t3_vsc7323_update_stats(struct cmac *mac);
 
-void t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			   const struct mdio_ops *mdio_ops);
-void t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			 const struct mdio_ops *mdio_ops);
-void t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_ael1002_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			 const struct mdio_ops *mdio_ops);
-void t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_ael1006_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			 const struct mdio_ops *mdio_ops);
-void t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_qt2045_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			const struct mdio_ops *mdio_ops);
-void t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_xaui_direct_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			     const struct mdio_ops *mdio_ops);
 #endif /* __CHELSIO_COMMON_H */
diff --git a/sys/dev/cxgb/common/cxgb_ctl_defs.h b/sys/dev/cxgb/common/cxgb_ctl_defs.h
index 3a2eb4f..11ed65a 100644
--- a/sys/dev/cxgb/common/cxgb_ctl_defs.h
+++ b/sys/dev/cxgb/common/cxgb_ctl_defs.h
@@ -125,8 +125,8 @@ struct rdma_info {
 	unsigned int rqt_top;	 /* RQT last entry address */
 	unsigned int udbell_len; /* user doorbell region length */
 	unsigned long udbell_physbase;  /* user doorbell physical start addr */
-	void volatile *kdb_addr;  /* kernel doorbell register address */
-	struct pci_dev *pdev;    /* associated PCI device */
+	void *kdb_addr;  /* kernel doorbell register address */
+	struct device *pdev;    /* associated PCI device */
 };
 
 /*
diff --git a/sys/dev/cxgb/common/cxgb_firmware_exports.h b/sys/dev/cxgb/common/cxgb_firmware_exports.h
index e361c95..55c5078 100644
--- a/sys/dev/cxgb/common/cxgb_firmware_exports.h
+++ b/sys/dev/cxgb/common/cxgb_firmware_exports.h
@@ -74,6 +74,8 @@ $FreeBSD$
 
 #define FW_WROPCODE_MNGT			0x1D
 #define FW_MNGTOPCODE_PKTSCHED_SET		0x00
+#define FW_MNGTOPCODE_WRC_SET			0x01
+#define FW_MNGTOPCODE_TUNNEL_CR_FLUSH		0x02
 
 /* Maximum size of a WR sent from the host, limited by the SGE. 
  *
diff --git a/sys/dev/cxgb/common/cxgb_mc5.c b/sys/dev/cxgb/common/cxgb_mc5.c
index d3eed4a..0e40aca 100644
--- a/sys/dev/cxgb/common/cxgb_mc5.c
+++ b/sys/dev/cxgb/common/cxgb_mc5.c
@@ -384,7 +384,7 @@ int t3_mc5_init(struct mc5 *mc5, unsigned int nservers, unsigned int nfilters,
 	return err;
 }
 
-/*
+/**
  *	read_mc5_range - dump a part of the memory managed by MC5
  *	@mc5: the MC5 handle
  *	@start: the start address for the dump
@@ -425,8 +425,11 @@ int t3_read_mc5_range(const struct mc5 *mc5, unsigned int start,
 
 #define MC5_INT_FATAL (F_PARITYERR | F_REQQPARERR | F_DISPQPARERR)
 
-/*
- * MC5 interrupt handler
+/**
+ *	t3_mc5_intr_handler - MC5 interrupt handler
+ *	@mc5: the MC5 handle
+ *
+ *	The MC5 interrupt handler.
  */
 void t3_mc5_intr_handler(struct mc5 *mc5)
 {
@@ -462,6 +465,16 @@ void t3_mc5_intr_handler(struct mc5 *mc5)
 	t3_write_reg(adap, A_MC5_DB_INT_CAUSE, cause);
 }
 
+
+/**
+ *	t3_mc5_prep - initialize the SW state for MC5
+ *	@adapter: the adapter
+ *	@mc5: the MC5 handle
+ *	@mode: whether the TCAM will be in 72- or 144-bit mode
+ *
+ *	Initialize the SW state associated with MC5.  Among other things
+ *	this determines the size of the attached TCAM.
+ */
 void __devinit t3_mc5_prep(adapter_t *adapter, struct mc5 *mc5, int mode)
 {
 #define K * 1024
diff --git a/sys/dev/cxgb/common/cxgb_mv88e1xxx.c b/sys/dev/cxgb/common/cxgb_mv88e1xxx.c
index 6cee581..8777b82 100644
--- a/sys/dev/cxgb/common/cxgb_mv88e1xxx.c
+++ b/sys/dev/cxgb/common/cxgb_mv88e1xxx.c
@@ -221,6 +221,16 @@ static int mv88e1xxx_get_link_status(struct cphy *cphy, int *link_ok,
 	return 0;
 }
 
+static int mv88e1xxx_set_speed_duplex(struct cphy *phy, int speed, int duplex)
+{
+	int err = t3_set_phy_speed_duplex(phy, speed, duplex);
+
+	/* PHY needs reset for new settings to take effect */
+	if (!err)
+		err = mv88e1xxx_reset(phy, 0);
+	return err;
+}
+
 static int mv88e1xxx_downshift_set(struct cphy *cphy, int downshift_enable)
 {
 	/*
@@ -258,7 +268,6 @@ static int mv88e1xxx_intr_handler(struct cphy *cphy)
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops mv88e1xxx_ops = {
-	NULL,
 	mv88e1xxx_reset,
 	mv88e1xxx_intr_enable,
 	mv88e1xxx_intr_disable,
@@ -268,7 +277,7 @@ static struct cphy_ops mv88e1xxx_ops = {
 	mv88e1xxx_autoneg_restart,
 	t3_phy_advertise,
 	mv88e1xxx_set_loopback,
-	t3_set_phy_speed_duplex,
+	mv88e1xxx_set_speed_duplex,
 	mv88e1xxx_get_link_status,
 	mv88e1xxx_power_down,
 };
@@ -283,20 +292,28 @@ static struct cphy_ops mv88e1xxx_ops = {
 	.autoneg_restart   = mv88e1xxx_autoneg_restart,
 	.advertise         = t3_phy_advertise,
 	.set_loopback      = mv88e1xxx_set_loopback,
-	.set_speed_duplex  = t3_set_phy_speed_duplex,
+	.set_speed_duplex  = mv88e1xxx_set_speed_duplex,
 	.get_link_status   = mv88e1xxx_get_link_status,
 	.power_down        = mv88e1xxx_power_down,
 };
 #endif
 
-void t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+int t3_mv88e1xxx_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
 			   const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &mv88e1xxx_ops, mdio_ops);
+	int err;
+
+	cphy_init(phy, adapter, phy_addr, &mv88e1xxx_ops, mdio_ops,
+		  SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full |
+		  SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII |
+		  SUPPORTED_TP | SUPPORTED_IRQ, "10/100/1000BASE-T");
 
 	/* Configure copper PHY transmitter as class A to reduce EMI. */
-	mdio_write(phy, 0, MV88E1XXX_EXTENDED_ADDR, 0xb);
-	mdio_write(phy, 0, MV88E1XXX_EXTENDED_DATA, 0x8004);
-		
-	mv88e1xxx_downshift_set(phy, 1);   /* Enable downshift */
+	err = mdio_write(phy, 0, MV88E1XXX_EXTENDED_ADDR, 0xb);
+
+	if (!err)
+		err = mdio_write(phy, 0, MV88E1XXX_EXTENDED_DATA, 0x8004);
+	if (!err)
+		err = mv88e1xxx_downshift_set(phy, 1);   /* Enable downshift */
+	return err;
 }
diff --git a/sys/dev/cxgb/common/cxgb_regs.h b/sys/dev/cxgb/common/cxgb_regs.h
index 63744c4..dd8db9a 100644
--- a/sys/dev/cxgb/common/cxgb_regs.h
+++ b/sys/dev/cxgb/common/cxgb_regs.h
@@ -35,6 +35,38 @@ $FreeBSD$
 
 #define A_SG_CONTROL 0x0
 
+#define S_CONGMODE    29
+#define V_CONGMODE(x) ((x) << S_CONGMODE)
+#define F_CONGMODE    V_CONGMODE(1U)
+
+#define S_TNLFLMODE    28
+#define V_TNLFLMODE(x) ((x) << S_TNLFLMODE)
+#define F_TNLFLMODE    V_TNLFLMODE(1U)
+
+#define S_FATLPERREN    27
+#define V_FATLPERREN(x) ((x) << S_FATLPERREN)
+#define F_FATLPERREN    V_FATLPERREN(1U)
+
+#define S_URGTNL    26
+#define V_URGTNL(x) ((x) << S_URGTNL)
+#define F_URGTNL    V_URGTNL(1U)
+
+#define S_NEWNOTIFY    25
+#define V_NEWNOTIFY(x) ((x) << S_NEWNOTIFY)
+#define F_NEWNOTIFY    V_NEWNOTIFY(1U)
+
+#define S_AVOIDCQOVFL    24
+#define V_AVOIDCQOVFL(x) ((x) << S_AVOIDCQOVFL)
+#define F_AVOIDCQOVFL    V_AVOIDCQOVFL(1U)
+
+#define S_OPTONEINTMULTQ    23
+#define V_OPTONEINTMULTQ(x) ((x) << S_OPTONEINTMULTQ)
+#define F_OPTONEINTMULTQ    V_OPTONEINTMULTQ(1U)
+
+#define S_CQCRDTCTRL    22
+#define V_CQCRDTCTRL(x) ((x) << S_CQCRDTCTRL)
+#define F_CQCRDTCTRL    V_CQCRDTCTRL(1U)
+
 #define S_EGRENUPBP    21
 #define V_EGRENUPBP(x) ((x) << S_EGRENUPBP)
 #define F_EGRENUPBP    V_EGRENUPBP(1U)
@@ -94,26 +126,6 @@ $FreeBSD$
 #define V_GLOBALENABLE(x) ((x) << S_GLOBALENABLE)
 #define F_GLOBALENABLE    V_GLOBALENABLE(1U)
 
-#define S_URGTNL    26
-#define V_URGTNL(x) ((x) << S_URGTNL)
-#define F_URGTNL    V_URGTNL(1U)
-
-#define S_NEWNOTIFY    25
-#define V_NEWNOTIFY(x) ((x) << S_NEWNOTIFY)
-#define F_NEWNOTIFY    V_NEWNOTIFY(1U)
-
-#define S_AVOIDCQOVFL    24
-#define V_AVOIDCQOVFL(x) ((x) << S_AVOIDCQOVFL)
-#define F_AVOIDCQOVFL    V_AVOIDCQOVFL(1U)
-
-#define S_OPTONEINTMULTQ    23
-#define V_OPTONEINTMULTQ(x) ((x) << S_OPTONEINTMULTQ)
-#define F_OPTONEINTMULTQ    V_OPTONEINTMULTQ(1U)
-
-#define S_CQCRDTCTRL    22
-#define V_CQCRDTCTRL(x) ((x) << S_CQCRDTCTRL)
-#define F_CQCRDTCTRL    V_CQCRDTCTRL(1U)
-
 #define A_SG_KDOORBELL 0x4
 
 #define S_SELEGRCNTX    31
@@ -366,11 +378,6 @@ $FreeBSD$
 
 #define A_SG_EGR_PRI_CNT 0x50
 
-#define S_EGRPRICNT    0
-#define M_EGRPRICNT    0x1f
-#define V_EGRPRICNT(x) ((x) << S_EGRPRICNT)
-#define G_EGRPRICNT(x) (((x) >> S_EGRPRICNT) & M_EGRPRICNT)
-
 #define S_EGRERROPCODE    24
 #define M_EGRERROPCODE    0xff
 #define V_EGRERROPCODE(x) ((x) << S_EGRERROPCODE)
@@ -386,6 +393,11 @@ $FreeBSD$
 #define V_EGRLOOPCODE(x) ((x) << S_EGRLOOPCODE)
 #define G_EGRLOOPCODE(x) (((x) >> S_EGRLOOPCODE) & M_EGRLOOPCODE)
 
+#define S_EGRPRICNT    0
+#define M_EGRPRICNT    0x1f
+#define V_EGRPRICNT(x) ((x) << S_EGRPRICNT)
+#define G_EGRPRICNT(x) (((x) >> S_EGRPRICNT) & M_EGRPRICNT)
+
 #define A_SG_EGR_RCQ_DRB_THRSH 0x54
 
 #define S_HIRCQDRBTHRSH    16
@@ -407,6 +419,56 @@ $FreeBSD$
 
 #define A_SG_INT_CAUSE 0x5c
 
+#define S_HIRCQPARITYERROR    31
+#define V_HIRCQPARITYERROR(x) ((x) << S_HIRCQPARITYERROR)
+#define F_HIRCQPARITYERROR    V_HIRCQPARITYERROR(1U)
+
+#define S_LORCQPARITYERROR    30
+#define V_LORCQPARITYERROR(x) ((x) << S_LORCQPARITYERROR)
+#define F_LORCQPARITYERROR    V_LORCQPARITYERROR(1U)
+
+#define S_HIDRBPARITYERROR    29
+#define V_HIDRBPARITYERROR(x) ((x) << S_HIDRBPARITYERROR)
+#define F_HIDRBPARITYERROR    V_HIDRBPARITYERROR(1U)
+
+#define S_LODRBPARITYERROR    28
+#define V_LODRBPARITYERROR(x) ((x) << S_LODRBPARITYERROR)
+#define F_LODRBPARITYERROR    V_LODRBPARITYERROR(1U)
+
+#define S_FLPARITYERROR    22
+#define M_FLPARITYERROR    0x3f
+#define V_FLPARITYERROR(x) ((x) << S_FLPARITYERROR)
+#define G_FLPARITYERROR(x) (((x) >> S_FLPARITYERROR) & M_FLPARITYERROR)
+
+#define S_ITPARITYERROR    20
+#define M_ITPARITYERROR    0x3
+#define V_ITPARITYERROR(x) ((x) << S_ITPARITYERROR)
+#define G_ITPARITYERROR(x) (((x) >> S_ITPARITYERROR) & M_ITPARITYERROR)
+
+#define S_IRPARITYERROR    19
+#define V_IRPARITYERROR(x) ((x) << S_IRPARITYERROR)
+#define F_IRPARITYERROR    V_IRPARITYERROR(1U)
+
+#define S_RCPARITYERROR    18
+#define V_RCPARITYERROR(x) ((x) << S_RCPARITYERROR)
+#define F_RCPARITYERROR    V_RCPARITYERROR(1U)
+
+#define S_OCPARITYERROR    17
+#define V_OCPARITYERROR(x) ((x) << S_OCPARITYERROR)
+#define F_OCPARITYERROR    V_OCPARITYERROR(1U)
+
+#define S_CPPARITYERROR    16
+#define V_CPPARITYERROR(x) ((x) << S_CPPARITYERROR)
+#define F_CPPARITYERROR    V_CPPARITYERROR(1U)
+
+#define S_R_REQ_FRAMINGERROR    15
+#define V_R_REQ_FRAMINGERROR(x) ((x) << S_R_REQ_FRAMINGERROR)
+#define F_R_REQ_FRAMINGERROR    V_R_REQ_FRAMINGERROR(1U)
+
+#define S_UC_REQ_FRAMINGERROR    14
+#define V_UC_REQ_FRAMINGERROR(x) ((x) << S_UC_REQ_FRAMINGERROR)
+#define F_UC_REQ_FRAMINGERROR    V_UC_REQ_FRAMINGERROR(1U)
+
 #define S_HICTLDRBDROPERR    13
 #define V_HICTLDRBDROPERR(x) ((x) << S_HICTLDRBDROPERR)
 #define F_HICTLDRBDROPERR    V_HICTLDRBDROPERR(1U)
@@ -582,6 +644,10 @@ $FreeBSD$
 #define A_PCIX_INT_CAUSE 0x84
 #define A_PCIX_CFG 0x88
 
+#define S_DMASTOPEN    19
+#define V_DMASTOPEN(x) ((x) << S_DMASTOPEN)
+#define F_DMASTOPEN    V_DMASTOPEN(1U)
+
 #define S_CLIDECEN    18
 #define V_CLIDECEN(x) ((x) << S_CLIDECEN)
 #define F_CLIDECEN    V_CLIDECEN(1U)
@@ -721,16 +787,175 @@ $FreeBSD$
 #define V_SLEEPMODE0(x) ((x) << S_SLEEPMODE0)
 #define F_SLEEPMODE0    V_SLEEPMODE0(1U)
 
+#define A_PCIX_STAT0 0x98
+
+#define S_PIOREQFIFOLEVEL    26
+#define M_PIOREQFIFOLEVEL    0x3f
+#define V_PIOREQFIFOLEVEL(x) ((x) << S_PIOREQFIFOLEVEL)
+#define G_PIOREQFIFOLEVEL(x) (((x) >> S_PIOREQFIFOLEVEL) & M_PIOREQFIFOLEVEL)
+
+#define S_RFINIST    24
+#define M_RFINIST    0x3
+#define V_RFINIST(x) ((x) << S_RFINIST)
+#define G_RFINIST(x) (((x) >> S_RFINIST) & M_RFINIST)
+
+#define S_RFRESPRDST    22
+#define M_RFRESPRDST    0x3
+#define V_RFRESPRDST(x) ((x) << S_RFRESPRDST)
+#define G_RFRESPRDST(x) (((x) >> S_RFRESPRDST) & M_RFRESPRDST)
+
+#define S_TARCST    19
+#define M_TARCST    0x7
+#define V_TARCST(x) ((x) << S_TARCST)
+#define G_TARCST(x) (((x) >> S_TARCST) & M_TARCST)
+
+#define S_TARXST    16
+#define M_TARXST    0x7
+#define V_TARXST(x) ((x) << S_TARXST)
+#define G_TARXST(x) (((x) >> S_TARXST) & M_TARXST)
+
+#define S_WFREQWRST    13
+#define M_WFREQWRST    0x7
+#define V_WFREQWRST(x) ((x) << S_WFREQWRST)
+#define G_WFREQWRST(x) (((x) >> S_WFREQWRST) & M_WFREQWRST)
+
+#define S_WFRESPFIFOEMPTY    12
+#define V_WFRESPFIFOEMPTY(x) ((x) << S_WFRESPFIFOEMPTY)
+#define F_WFRESPFIFOEMPTY    V_WFRESPFIFOEMPTY(1U)
+
+#define S_WFREQFIFOEMPTY    11
+#define V_WFREQFIFOEMPTY(x) ((x) << S_WFREQFIFOEMPTY)
+#define F_WFREQFIFOEMPTY    V_WFREQFIFOEMPTY(1U)
+
+#define S_RFRESPFIFOEMPTY    10
+#define V_RFRESPFIFOEMPTY(x) ((x) << S_RFRESPFIFOEMPTY)
+#define F_RFRESPFIFOEMPTY    V_RFRESPFIFOEMPTY(1U)
+
+#define S_RFREQFIFOEMPTY    9
+#define V_RFREQFIFOEMPTY(x) ((x) << S_RFREQFIFOEMPTY)
+#define F_RFREQFIFOEMPTY    V_RFREQFIFOEMPTY(1U)
+
+#define S_PIORESPFIFOLEVEL    7
+#define M_PIORESPFIFOLEVEL    0x3
+#define V_PIORESPFIFOLEVEL(x) ((x) << S_PIORESPFIFOLEVEL)
+#define G_PIORESPFIFOLEVEL(x) (((x) >> S_PIORESPFIFOLEVEL) & M_PIORESPFIFOLEVEL)
+
+#define S_CFRESPFIFOEMPTY    6
+#define V_CFRESPFIFOEMPTY(x) ((x) << S_CFRESPFIFOEMPTY)
+#define F_CFRESPFIFOEMPTY    V_CFRESPFIFOEMPTY(1U)
+
+#define S_CFREQFIFOEMPTY    5
+#define V_CFREQFIFOEMPTY(x) ((x) << S_CFREQFIFOEMPTY)
+#define F_CFREQFIFOEMPTY    V_CFREQFIFOEMPTY(1U)
+
+#define S_VPDRESPFIFOEMPTY    4
+#define V_VPDRESPFIFOEMPTY(x) ((x) << S_VPDRESPFIFOEMPTY)
+#define F_VPDRESPFIFOEMPTY    V_VPDRESPFIFOEMPTY(1U)
+
+#define S_VPDREQFIFOEMPTY    3
+#define V_VPDREQFIFOEMPTY(x) ((x) << S_VPDREQFIFOEMPTY)
+#define F_VPDREQFIFOEMPTY    V_VPDREQFIFOEMPTY(1U)
+
+#define S_PIO_RSPPND    2
+#define V_PIO_RSPPND(x) ((x) << S_PIO_RSPPND)
+#define F_PIO_RSPPND    V_PIO_RSPPND(1U)
+
+#define S_DLYTRNPND    1
+#define V_DLYTRNPND(x) ((x) << S_DLYTRNPND)
+#define F_DLYTRNPND    V_DLYTRNPND(1U)
+
+#define S_SPLTRNPND    0
+#define V_SPLTRNPND(x) ((x) << S_SPLTRNPND)
+#define F_SPLTRNPND    V_SPLTRNPND(1U)
+
+#define A_PCIX_STAT1 0x9c
+
+#define S_WFINIST    26
+#define M_WFINIST    0xf
+#define V_WFINIST(x) ((x) << S_WFINIST)
+#define G_WFINIST(x) (((x) >> S_WFINIST) & M_WFINIST)
+
+#define S_ARBST    23
+#define M_ARBST    0x7
+#define V_ARBST(x) ((x) << S_ARBST)
+#define G_ARBST(x) (((x) >> S_ARBST) & M_ARBST)
+
+#define S_PMIST    21
+#define M_PMIST    0x3
+#define V_PMIST(x) ((x) << S_PMIST)
+#define G_PMIST(x) (((x) >> S_PMIST) & M_PMIST)
+
+#define S_CALST    19
+#define M_CALST    0x3
+#define V_CALST(x) ((x) << S_CALST)
+#define G_CALST(x) (((x) >> S_CALST) & M_CALST)
+
+#define S_CFREQRDST    17
+#define M_CFREQRDST    0x3
+#define V_CFREQRDST(x) ((x) << S_CFREQRDST)
+#define G_CFREQRDST(x) (((x) >> S_CFREQRDST) & M_CFREQRDST)
+
+#define S_CFINIST    15
+#define M_CFINIST    0x3
+#define V_CFINIST(x) ((x) << S_CFINIST)
+#define G_CFINIST(x) (((x) >> S_CFINIST) & M_CFINIST)
+
+#define S_CFRESPRDST    13
+#define M_CFRESPRDST    0x3
+#define V_CFRESPRDST(x) ((x) << S_CFRESPRDST)
+#define G_CFRESPRDST(x) (((x) >> S_CFRESPRDST) & M_CFRESPRDST)
+
+#define S_INICST    10
+#define M_INICST    0x7
+#define V_INICST(x) ((x) << S_INICST)
+#define G_INICST(x) (((x) >> S_INICST) & M_INICST)
+
+#define S_INIXST    7
+#define M_INIXST    0x7
+#define V_INIXST(x) ((x) << S_INIXST)
+#define G_INIXST(x) (((x) >> S_INIXST) & M_INIXST)
+
+#define S_INTST    4
+#define M_INTST    0x7
+#define V_INTST(x) ((x) << S_INTST)
+#define G_INTST(x) (((x) >> S_INTST) & M_INTST)
+
+#define S_PIOST    2
+#define M_PIOST    0x3
+#define V_PIOST(x) ((x) << S_PIOST)
+#define G_PIOST(x) (((x) >> S_PIOST) & M_PIOST)
+
+#define S_RFREQRDST    0
+#define M_RFREQRDST    0x3
+#define V_RFREQRDST(x) ((x) << S_RFREQRDST)
+#define G_RFREQRDST(x) (((x) >> S_RFREQRDST) & M_RFREQRDST)
+
 /* registers for module PCIE0 */
 #define PCIE0_BASE_ADDR 0x80
 
 #define A_PCIE_INT_ENABLE 0x80
 
-#define S_BISTERR    15
+#define S_BISTERR    19
 #define M_BISTERR    0xff
 #define V_BISTERR(x) ((x) << S_BISTERR)
 #define G_BISTERR(x) (((x) >> S_BISTERR) & M_BISTERR)
 
+#define S_TXPARERR    18
+#define V_TXPARERR(x) ((x) << S_TXPARERR)
+#define F_TXPARERR    V_TXPARERR(1U)
+
+#define S_RXPARERR    17
+#define V_RXPARERR(x) ((x) << S_RXPARERR)
+#define F_RXPARERR    V_RXPARERR(1U)
+
+#define S_RETRYLUTPARERR    16
+#define V_RETRYLUTPARERR(x) ((x) << S_RETRYLUTPARERR)
+#define F_RETRYLUTPARERR    V_RETRYLUTPARERR(1U)
+
+#define S_RETRYBUFPARERR    15
+#define V_RETRYBUFPARERR(x) ((x) << S_RETRYBUFPARERR)
+#define F_RETRYBUFPARERR    V_RETRYBUFPARERR(1U)
+
 #define S_PCIE_MSIXPARERR    12
 #define M_PCIE_MSIXPARERR    0x7
 #define V_PCIE_MSIXPARERR(x) ((x) << S_PCIE_MSIXPARERR)
@@ -787,6 +1012,18 @@ $FreeBSD$
 #define A_PCIE_INT_CAUSE 0x84
 #define A_PCIE_CFG 0x88
 
+#define S_PCIE_DMASTOPEN    24
+#define V_PCIE_DMASTOPEN(x) ((x) << S_PCIE_DMASTOPEN)
+#define F_PCIE_DMASTOPEN    V_PCIE_DMASTOPEN(1U)
+
+#define S_PRIORITYINTA    23
+#define V_PRIORITYINTA(x) ((x) << S_PRIORITYINTA)
+#define F_PRIORITYINTA    V_PRIORITYINTA(1U)
+
+#define S_INIFULLPKT    22
+#define V_INIFULLPKT(x) ((x) << S_INIFULLPKT)
+#define F_INIFULLPKT    V_INIFULLPKT(1U)
+
 #define S_ENABLELINKDWNDRST    21
 #define V_ENABLELINKDWNDRST(x) ((x) << S_ENABLELINKDWNDRST)
 #define F_ENABLELINKDWNDRST    V_ENABLELINKDWNDRST(1U)
@@ -825,15 +1062,37 @@ $FreeBSD$
 #define V_CRSTWRMMODE(x) ((x) << S_CRSTWRMMODE)
 #define F_CRSTWRMMODE    V_CRSTWRMMODE(1U)
 
-#define S_PRIORITYINTA    23
-#define V_PRIORITYINTA(x) ((x) << S_PRIORITYINTA)
-#define F_PRIORITYINTA    V_PRIORITYINTA(1U)
+#define A_PCIE_MODE 0x8c
 
-#define S_INIFULLPKT    22
-#define V_INIFULLPKT(x) ((x) << S_INIFULLPKT)
-#define F_INIFULLPKT    V_INIFULLPKT(1U)
+#define S_TAR_STATE    29
+#define M_TAR_STATE    0x7
+#define V_TAR_STATE(x) ((x) << S_TAR_STATE)
+#define G_TAR_STATE(x) (((x) >> S_TAR_STATE) & M_TAR_STATE)
 
-#define A_PCIE_MODE 0x8c
+#define S_RF_STATEINI    26
+#define M_RF_STATEINI    0x7
+#define V_RF_STATEINI(x) ((x) << S_RF_STATEINI)
+#define G_RF_STATEINI(x) (((x) >> S_RF_STATEINI) & M_RF_STATEINI)
+
+#define S_CF_STATEINI    23
+#define M_CF_STATEINI    0x7
+#define V_CF_STATEINI(x) ((x) << S_CF_STATEINI)
+#define G_CF_STATEINI(x) (((x) >> S_CF_STATEINI) & M_CF_STATEINI)
+
+#define S_PIO_STATEPL    20
+#define M_PIO_STATEPL    0x7
+#define V_PIO_STATEPL(x) ((x) << S_PIO_STATEPL)
+#define G_PIO_STATEPL(x) (((x) >> S_PIO_STATEPL) & M_PIO_STATEPL)
+
+#define S_PIO_STATEISC    18
+#define M_PIO_STATEISC    0x3
+#define V_PIO_STATEISC(x) ((x) << S_PIO_STATEISC)
+#define G_PIO_STATEISC(x) (((x) >> S_PIO_STATEISC) & M_PIO_STATEISC)
+
+#define S_NUMFSTTRNSEQRX    10
+#define M_NUMFSTTRNSEQRX    0xff
+#define V_NUMFSTTRNSEQRX(x) ((x) << S_NUMFSTTRNSEQRX)
+#define G_NUMFSTTRNSEQRX(x) (((x) >> S_NUMFSTTRNSEQRX) & M_NUMFSTTRNSEQRX)
 
 #define S_LNKCNTLSTATE    2
 #define M_LNKCNTLSTATE    0xff
@@ -848,10 +1107,76 @@ $FreeBSD$
 #define V_LNKINITIAL(x) ((x) << S_LNKINITIAL)
 #define F_LNKINITIAL    V_LNKINITIAL(1U)
 
-#define S_NUMFSTTRNSEQRX    10
-#define M_NUMFSTTRNSEQRX    0xff
-#define V_NUMFSTTRNSEQRX(x) ((x) << S_NUMFSTTRNSEQRX)
-#define G_NUMFSTTRNSEQRX(x) (((x) >> S_NUMFSTTRNSEQRX) & M_NUMFSTTRNSEQRX)
+#define A_PCIE_STAT 0x90
+
+#define S_INI_STATE    28
+#define M_INI_STATE    0xf
+#define V_INI_STATE(x) ((x) << S_INI_STATE)
+#define G_INI_STATE(x) (((x) >> S_INI_STATE) & M_INI_STATE)
+
+#define S_WF_STATEINI    24
+#define M_WF_STATEINI    0xf
+#define V_WF_STATEINI(x) ((x) << S_WF_STATEINI)
+#define G_WF_STATEINI(x) (((x) >> S_WF_STATEINI) & M_WF_STATEINI)
+
+#define S_PLM_REQFIFOCNT    22
+#define M_PLM_REQFIFOCNT    0x3
+#define V_PLM_REQFIFOCNT(x) ((x) << S_PLM_REQFIFOCNT)
+#define G_PLM_REQFIFOCNT(x) (((x) >> S_PLM_REQFIFOCNT) & M_PLM_REQFIFOCNT)
+
+#define S_ER_REQFIFOEMPTY    21
+#define V_ER_REQFIFOEMPTY(x) ((x) << S_ER_REQFIFOEMPTY)
+#define F_ER_REQFIFOEMPTY    V_ER_REQFIFOEMPTY(1U)
+
+#define S_WF_RSPFIFOEMPTY    20
+#define V_WF_RSPFIFOEMPTY(x) ((x) << S_WF_RSPFIFOEMPTY)
+#define F_WF_RSPFIFOEMPTY    V_WF_RSPFIFOEMPTY(1U)
+
+#define S_WF_REQFIFOEMPTY    19
+#define V_WF_REQFIFOEMPTY(x) ((x) << S_WF_REQFIFOEMPTY)
+#define F_WF_REQFIFOEMPTY    V_WF_REQFIFOEMPTY(1U)
+
+#define S_RF_RSPFIFOEMPTY    18
+#define V_RF_RSPFIFOEMPTY(x) ((x) << S_RF_RSPFIFOEMPTY)
+#define F_RF_RSPFIFOEMPTY    V_RF_RSPFIFOEMPTY(1U)
+
+#define S_RF_REQFIFOEMPTY    17
+#define V_RF_REQFIFOEMPTY(x) ((x) << S_RF_REQFIFOEMPTY)
+#define F_RF_REQFIFOEMPTY    V_RF_REQFIFOEMPTY(1U)
+
+#define S_RF_ACTEMPTY    16
+#define V_RF_ACTEMPTY(x) ((x) << S_RF_ACTEMPTY)
+#define F_RF_ACTEMPTY    V_RF_ACTEMPTY(1U)
+
+#define S_PIO_RSPFIFOCNT    11
+#define M_PIO_RSPFIFOCNT    0x1f
+#define V_PIO_RSPFIFOCNT(x) ((x) << S_PIO_RSPFIFOCNT)
+#define G_PIO_RSPFIFOCNT(x) (((x) >> S_PIO_RSPFIFOCNT) & M_PIO_RSPFIFOCNT)
+
+#define S_PIO_REQFIFOCNT    5
+#define M_PIO_REQFIFOCNT    0x3f
+#define V_PIO_REQFIFOCNT(x) ((x) << S_PIO_REQFIFOCNT)
+#define G_PIO_REQFIFOCNT(x) (((x) >> S_PIO_REQFIFOCNT) & M_PIO_REQFIFOCNT)
+
+#define S_CF_RSPFIFOEMPTY    4
+#define V_CF_RSPFIFOEMPTY(x) ((x) << S_CF_RSPFIFOEMPTY)
+#define F_CF_RSPFIFOEMPTY    V_CF_RSPFIFOEMPTY(1U)
+
+#define S_CF_REQFIFOEMPTY    3
+#define V_CF_REQFIFOEMPTY(x) ((x) << S_CF_REQFIFOEMPTY)
+#define F_CF_REQFIFOEMPTY    V_CF_REQFIFOEMPTY(1U)
+
+#define S_CF_ACTEMPTY    2
+#define V_CF_ACTEMPTY(x) ((x) << S_CF_ACTEMPTY)
+#define F_CF_ACTEMPTY    V_CF_ACTEMPTY(1U)
+
+#define S_VPD_RSPFIFOEMPTY    1
+#define V_VPD_RSPFIFOEMPTY(x) ((x) << S_VPD_RSPFIFOEMPTY)
+#define F_VPD_RSPFIFOEMPTY    V_VPD_RSPFIFOEMPTY(1U)
+
+#define S_VPD_REQFIFOEMPTY    0
+#define V_VPD_REQFIFOEMPTY(x) ((x) << S_VPD_REQFIFOEMPTY)
+#define F_VPD_REQFIFOEMPTY    V_VPD_REQFIFOEMPTY(1U)
 
 #define A_PCIE_CAL 0x90
 
@@ -883,8 +1208,37 @@ $FreeBSD$
 #define G_ZIN(x) (((x) >> S_ZIN) & M_ZIN)
 
 #define A_PCIE_WOL 0x94
+
+#define S_CF_RSPSTATE    12
+#define M_CF_RSPSTATE    0x3
+#define V_CF_RSPSTATE(x) ((x) << S_CF_RSPSTATE)
+#define G_CF_RSPSTATE(x) (((x) >> S_CF_RSPSTATE) & M_CF_RSPSTATE)
+
+#define S_RF_RSPSTATE    10
+#define M_RF_RSPSTATE    0x3
+#define V_RF_RSPSTATE(x) ((x) << S_RF_RSPSTATE)
+#define G_RF_RSPSTATE(x) (((x) >> S_RF_RSPSTATE) & M_RF_RSPSTATE)
+
+#define S_PME_STATE    7
+#define M_PME_STATE    0x7
+#define V_PME_STATE(x) ((x) << S_PME_STATE)
+#define G_PME_STATE(x) (((x) >> S_PME_STATE) & M_PME_STATE)
+
+#define S_INT_STATE    4
+#define M_INT_STATE    0x7
+#define V_INT_STATE(x) ((x) << S_INT_STATE)
+#define G_INT_STATE(x) (((x) >> S_INT_STATE) & M_INT_STATE)
+
 #define A_PCIE_PEX_CTRL0 0x98
 
+#define S_CPLTIMEOUTRETRY    31
+#define V_CPLTIMEOUTRETRY(x) ((x) << S_CPLTIMEOUTRETRY)
+#define F_CPLTIMEOUTRETRY    V_CPLTIMEOUTRETRY(1U)
+
+#define S_STRICTTSMN    30
+#define V_STRICTTSMN(x) ((x) << S_STRICTTSMN)
+#define F_STRICTTSMN    V_STRICTTSMN(1U)
+
 #define S_NUMFSTTRNSEQ    22
 #define M_NUMFSTTRNSEQ    0xff
 #define V_NUMFSTTRNSEQ(x) ((x) << S_NUMFSTTRNSEQ)
@@ -903,26 +1257,8 @@ $FreeBSD$
 #define V_CPLPNDCHKEN(x) ((x) << S_CPLPNDCHKEN)
 #define F_CPLPNDCHKEN    V_CPLPNDCHKEN(1U)
 
-#define S_CPLTIMEOUTRETRY    31
-#define V_CPLTIMEOUTRETRY(x) ((x) << S_CPLTIMEOUTRETRY)
-#define F_CPLTIMEOUTRETRY    V_CPLTIMEOUTRETRY(1U)
-
-#define S_STRICTTSMN    30
-#define V_STRICTTSMN(x) ((x) << S_STRICTTSMN)
-#define F_STRICTTSMN    V_STRICTTSMN(1U)
-
 #define A_PCIE_PEX_CTRL1 0x9c
 
-#define S_T3A_DLLPTIMEOUTLMT    11
-#define M_T3A_DLLPTIMEOUTLMT    0xfffff
-#define V_T3A_DLLPTIMEOUTLMT(x) ((x) << S_T3A_DLLPTIMEOUTLMT)
-#define G_T3A_DLLPTIMEOUTLMT(x) (((x) >> S_T3A_DLLPTIMEOUTLMT) & M_T3A_DLLPTIMEOUTLMT)
-
-#define S_T3A_ACKLAT    0
-#define M_T3A_ACKLAT    0x7ff
-#define V_T3A_ACKLAT(x) ((x) << S_T3A_ACKLAT)
-#define G_T3A_ACKLAT(x) (((x) >> S_T3A_ACKLAT) & M_T3A_ACKLAT)
-
 #define S_RXPHYERREN    31
 #define V_RXPHYERREN(x) ((x) << S_RXPHYERREN)
 #define F_RXPHYERREN    V_RXPHYERREN(1U)
@@ -937,34 +1273,48 @@ $FreeBSD$
 #define V_ACKLAT(x) ((x) << S_ACKLAT)
 #define G_ACKLAT(x) (((x) >> S_ACKLAT) & M_ACKLAT)
 
+#define S_T3A_DLLPTIMEOUTLMT    11
+#define M_T3A_DLLPTIMEOUTLMT    0xfffff
+#define V_T3A_DLLPTIMEOUTLMT(x) ((x) << S_T3A_DLLPTIMEOUTLMT)
+#define G_T3A_DLLPTIMEOUTLMT(x) (((x) >> S_T3A_DLLPTIMEOUTLMT) & M_T3A_DLLPTIMEOUTLMT)
+
+#define S_T3A_ACKLAT    0
+#define M_T3A_ACKLAT    0x7ff
+#define V_T3A_ACKLAT(x) ((x) << S_T3A_ACKLAT)
+#define G_T3A_ACKLAT(x) (((x) >> S_T3A_ACKLAT) & M_T3A_ACKLAT)
+
 #define A_PCIE_PEX_CTRL2 0xa0
 
-#define S_PMEXITL1REQ    29
+#define S_LNKCNTLDETDIR    30
+#define V_LNKCNTLDETDIR(x) ((x) << S_LNKCNTLDETDIR)
+#define F_LNKCNTLDETDIR    V_LNKCNTLDETDIR(1U)
+
+#define S_ENTERL1REN    29
+#define V_ENTERL1REN(x) ((x) << S_ENTERL1REN)
+#define F_ENTERL1REN    V_ENTERL1REN(1U)
+
+#define S_PMEXITL1REQ    28
 #define V_PMEXITL1REQ(x) ((x) << S_PMEXITL1REQ)
 #define F_PMEXITL1REQ    V_PMEXITL1REQ(1U)
 
-#define S_PMTXIDLE    28
+#define S_PMTXIDLE    27
 #define V_PMTXIDLE(x) ((x) << S_PMTXIDLE)
 #define F_PMTXIDLE    V_PMTXIDLE(1U)
 
-#define S_PCIMODELOOP    27
+#define S_PCIMODELOOP    26
 #define V_PCIMODELOOP(x) ((x) << S_PCIMODELOOP)
 #define F_PCIMODELOOP    V_PCIMODELOOP(1U)
 
-#define S_L1ASPMTXRXL0STIME    15
+#define S_L1ASPMTXRXL0STIME    14
 #define M_L1ASPMTXRXL0STIME    0xfff
 #define V_L1ASPMTXRXL0STIME(x) ((x) << S_L1ASPMTXRXL0STIME)
 #define G_L1ASPMTXRXL0STIME(x) (((x) >> S_L1ASPMTXRXL0STIME) & M_L1ASPMTXRXL0STIME)
 
-#define S_L0SIDLETIME    4
+#define S_L0SIDLETIME    3
 #define M_L0SIDLETIME    0x7ff
 #define V_L0SIDLETIME(x) ((x) << S_L0SIDLETIME)
 #define G_L0SIDLETIME(x) (((x) >> S_L0SIDLETIME) & M_L0SIDLETIME)
 
-#define S_ENTERL23    3
-#define V_ENTERL23(x) ((x) << S_ENTERL23)
-#define F_ENTERL23    V_ENTERL23(1U)
-
 #define S_ENTERL1ASPMEN    2
 #define V_ENTERL1ASPMEN(x) ((x) << S_ENTERL1ASPMEN)
 #define F_ENTERL1ASPMEN    V_ENTERL1ASPMEN(1U)
@@ -977,16 +1327,17 @@ $FreeBSD$
 #define V_ENTERL0SEN(x) ((x) << S_ENTERL0SEN)
 #define F_ENTERL0SEN    V_ENTERL0SEN(1U)
 
-#define S_LNKCNTLDETDIR    30
-#define V_LNKCNTLDETDIR(x) ((x) << S_LNKCNTLDETDIR)
-#define F_LNKCNTLDETDIR    V_LNKCNTLDETDIR(1U)
-
-#define S_ENTERL1REN    29
-#define V_ENTERL1REN(x) ((x) << S_ENTERL1REN)
-#define F_ENTERL1REN    V_ENTERL1REN(1U)
+#define S_ENTERL23    3
+#define V_ENTERL23(x) ((x) << S_ENTERL23)
+#define F_ENTERL23    V_ENTERL23(1U)
 
 #define A_PCIE_PEX_ERR 0xa4
 
+#define S_CPLTIMEOUTID    18
+#define M_CPLTIMEOUTID    0x7f
+#define V_CPLTIMEOUTID(x) ((x) << S_CPLTIMEOUTID)
+#define G_CPLTIMEOUTID(x) (((x) >> S_CPLTIMEOUTID) & M_CPLTIMEOUTID)
+
 #define S_FLOWCTLOFLOWERR    17
 #define V_FLOWCTLOFLOWERR(x) ((x) << S_FLOWCTLOFLOWERR)
 #define F_FLOWCTLOFLOWERR    V_FLOWCTLOFLOWERR(1U)
@@ -1059,10 +1410,16 @@ $FreeBSD$
 #define V_PSNCPL(x) ((x) << S_PSNCPL)
 #define F_PSNCPL    V_PSNCPL(1U)
 
-#define S_CPLTIMEOUTID    18
-#define M_CPLTIMEOUTID    0x7f
-#define V_CPLTIMEOUTID(x) ((x) << S_CPLTIMEOUTID)
-#define G_CPLTIMEOUTID(x) (((x) >> S_CPLTIMEOUTID) & M_CPLTIMEOUTID)
+#define A_PCIE_SERDES_CTRL 0xa8
+
+#define S_PMASEL    3
+#define V_PMASEL(x) ((x) << S_PMASEL)
+#define F_PMASEL    V_PMASEL(1U)
+
+#define S_LANE    0
+#define M_LANE    0x7
+#define V_LANE(x) ((x) << S_LANE)
+#define G_LANE(x) (((x) >> S_LANE) & M_LANE)
 
 #define A_PCIE_PIPE_CTRL 0xa8
 
@@ -1093,16 +1450,25 @@ $FreeBSD$
 #define V_PCLKOFFINP1(x) ((x) << S_PCLKOFFINP1)
 #define F_PCLKOFFINP1    V_PCLKOFFINP1(1U)
 
-#define S_PMASEL    3
-#define V_PMASEL(x) ((x) << S_PMASEL)
-#define F_PMASEL    V_PMASEL(1U)
+#define A_PCIE_SERDES_QUAD_CTRL0 0xac
 
-#define S_LANE    0
-#define M_LANE    0x7
-#define V_LANE(x) ((x) << S_LANE)
-#define G_LANE(x) (((x) >> S_LANE) & M_LANE)
+#define S_TESTSIG    10
+#define M_TESTSIG    0x7ffff
+#define V_TESTSIG(x) ((x) << S_TESTSIG)
+#define G_TESTSIG(x) (((x) >> S_TESTSIG) & M_TESTSIG)
 
-#define A_PCIE_SERDES_CTRL 0xac
+#define S_OFFSET    2
+#define M_OFFSET    0xff
+#define V_OFFSET(x) ((x) << S_OFFSET)
+#define G_OFFSET(x) (((x) >> S_OFFSET) & M_OFFSET)
+
+#define S_OFFSETEN    1
+#define V_OFFSETEN(x) ((x) << S_OFFSETEN)
+#define F_OFFSETEN    V_OFFSETEN(1U)
+
+#define S_IDDQB    0
+#define V_IDDQB(x) ((x) << S_IDDQB)
+#define F_IDDQB    V_IDDQB(1U)
 
 #define S_MANMODE    31
 #define V_MANMODE(x) ((x) << S_MANMODE)
@@ -1193,68 +1559,6 @@ $FreeBSD$
 #define V_PREEMPH(x) ((x) << S_PREEMPH)
 #define G_PREEMPH(x) (((x) >> S_PREEMPH) & M_PREEMPH)
 
-#define A_PCIE_SERDES_QUAD_CTRL0 0xac
-
-#define S_TESTSIG    10
-#define M_TESTSIG    0x7ffff
-#define V_TESTSIG(x) ((x) << S_TESTSIG)
-#define G_TESTSIG(x) (((x) >> S_TESTSIG) & M_TESTSIG)
-
-#define S_OFFSET    2
-#define M_OFFSET    0xff
-#define V_OFFSET(x) ((x) << S_OFFSET)
-#define G_OFFSET(x) (((x) >> S_OFFSET) & M_OFFSET)
-
-#define S_OFFSETEN    1
-#define V_OFFSETEN(x) ((x) << S_OFFSETEN)
-#define F_OFFSETEN    V_OFFSETEN(1U)
-
-#define S_IDDQB    0
-#define V_IDDQB(x) ((x) << S_IDDQB)
-#define F_IDDQB    V_IDDQB(1U)
-
-#define A_PCIE_SERDES_STATUS0 0xb0
-
-#define S_RXERRLANE7    21
-#define M_RXERRLANE7    0x7
-#define V_RXERRLANE7(x) ((x) << S_RXERRLANE7)
-#define G_RXERRLANE7(x) (((x) >> S_RXERRLANE7) & M_RXERRLANE7)
-
-#define S_RXERRLANE6    18
-#define M_RXERRLANE6    0x7
-#define V_RXERRLANE6(x) ((x) << S_RXERRLANE6)
-#define G_RXERRLANE6(x) (((x) >> S_RXERRLANE6) & M_RXERRLANE6)
-
-#define S_RXERRLANE5    15
-#define M_RXERRLANE5    0x7
-#define V_RXERRLANE5(x) ((x) << S_RXERRLANE5)
-#define G_RXERRLANE5(x) (((x) >> S_RXERRLANE5) & M_RXERRLANE5)
-
-#define S_RXERRLANE4    12
-#define M_RXERRLANE4    0x7
-#define V_RXERRLANE4(x) ((x) << S_RXERRLANE4)
-#define G_RXERRLANE4(x) (((x) >> S_RXERRLANE4) & M_RXERRLANE4)
-
-#define S_PCIE_RXERRLANE3    9
-#define M_PCIE_RXERRLANE3    0x7
-#define V_PCIE_RXERRLANE3(x) ((x) << S_PCIE_RXERRLANE3)
-#define G_PCIE_RXERRLANE3(x) (((x) >> S_PCIE_RXERRLANE3) & M_PCIE_RXERRLANE3)
-
-#define S_PCIE_RXERRLANE2    6
-#define M_PCIE_RXERRLANE2    0x7
-#define V_PCIE_RXERRLANE2(x) ((x) << S_PCIE_RXERRLANE2)
-#define G_PCIE_RXERRLANE2(x) (((x) >> S_PCIE_RXERRLANE2) & M_PCIE_RXERRLANE2)
-
-#define S_PCIE_RXERRLANE1    3
-#define M_PCIE_RXERRLANE1    0x7
-#define V_PCIE_RXERRLANE1(x) ((x) << S_PCIE_RXERRLANE1)
-#define G_PCIE_RXERRLANE1(x) (((x) >> S_PCIE_RXERRLANE1) & M_PCIE_RXERRLANE1)
-
-#define S_PCIE_RXERRLANE0    0
-#define M_PCIE_RXERRLANE0    0x7
-#define V_PCIE_RXERRLANE0(x) ((x) << S_PCIE_RXERRLANE0)
-#define G_PCIE_RXERRLANE0(x) (((x) >> S_PCIE_RXERRLANE0) & M_PCIE_RXERRLANE0)
-
 #define A_PCIE_SERDES_QUAD_CTRL1 0xb0
 
 #define S_FASTINIT    28
@@ -1339,6 +1643,120 @@ $FreeBSD$
 #define V_PCLKDETECT(x) ((x) << S_PCLKDETECT)
 #define F_PCLKDETECT    V_PCLKDETECT(1U)
 
+#define A_PCIE_SERDES_STATUS0 0xb0
+
+#define S_RXERRLANE7    21
+#define M_RXERRLANE7    0x7
+#define V_RXERRLANE7(x) ((x) << S_RXERRLANE7)
+#define G_RXERRLANE7(x) (((x) >> S_RXERRLANE7) & M_RXERRLANE7)
+
+#define S_RXERRLANE6    18
+#define M_RXERRLANE6    0x7
+#define V_RXERRLANE6(x) ((x) << S_RXERRLANE6)
+#define G_RXERRLANE6(x) (((x) >> S_RXERRLANE6) & M_RXERRLANE6)
+
+#define S_RXERRLANE5    15
+#define M_RXERRLANE5    0x7
+#define V_RXERRLANE5(x) ((x) << S_RXERRLANE5)
+#define G_RXERRLANE5(x) (((x) >> S_RXERRLANE5) & M_RXERRLANE5)
+
+#define S_RXERRLANE4    12
+#define M_RXERRLANE4    0x7
+#define V_RXERRLANE4(x) ((x) << S_RXERRLANE4)
+#define G_RXERRLANE4(x) (((x) >> S_RXERRLANE4) & M_RXERRLANE4)
+
+#define S_PCIE_RXERRLANE3    9
+#define M_PCIE_RXERRLANE3    0x7
+#define V_PCIE_RXERRLANE3(x) ((x) << S_PCIE_RXERRLANE3)
+#define G_PCIE_RXERRLANE3(x) (((x) >> S_PCIE_RXERRLANE3) & M_PCIE_RXERRLANE3)
+
+#define S_PCIE_RXERRLANE2    6
+#define M_PCIE_RXERRLANE2    0x7
+#define V_PCIE_RXERRLANE2(x) ((x) << S_PCIE_RXERRLANE2)
+#define G_PCIE_RXERRLANE2(x) (((x) >> S_PCIE_RXERRLANE2) & M_PCIE_RXERRLANE2)
+
+#define S_PCIE_RXERRLANE1    3
+#define M_PCIE_RXERRLANE1    0x7
+#define V_PCIE_RXERRLANE1(x) ((x) << S_PCIE_RXERRLANE1)
+#define G_PCIE_RXERRLANE1(x) (((x) >> S_PCIE_RXERRLANE1) & M_PCIE_RXERRLANE1)
+
+#define S_PCIE_RXERRLANE0    0
+#define M_PCIE_RXERRLANE0    0x7
+#define V_PCIE_RXERRLANE0(x) ((x) << S_PCIE_RXERRLANE0)
+#define G_PCIE_RXERRLANE0(x) (((x) >> S_PCIE_RXERRLANE0) & M_PCIE_RXERRLANE0)
+
+#define A_PCIE_SERDES_LANE_CTRL 0xb4
+
+#define S_EXTBISTCHKERRCLR    22
+#define V_EXTBISTCHKERRCLR(x) ((x) << S_EXTBISTCHKERRCLR)
+#define F_EXTBISTCHKERRCLR    V_EXTBISTCHKERRCLR(1U)
+
+#define S_EXTBISTCHKEN    21
+#define V_EXTBISTCHKEN(x) ((x) << S_EXTBISTCHKEN)
+#define F_EXTBISTCHKEN    V_EXTBISTCHKEN(1U)
+
+#define S_EXTBISTGENEN    20
+#define V_EXTBISTGENEN(x) ((x) << S_EXTBISTGENEN)
+#define F_EXTBISTGENEN    V_EXTBISTGENEN(1U)
+
+#define S_EXTBISTPAT    17
+#define M_EXTBISTPAT    0x7
+#define V_EXTBISTPAT(x) ((x) << S_EXTBISTPAT)
+#define G_EXTBISTPAT(x) (((x) >> S_EXTBISTPAT) & M_EXTBISTPAT)
+
+#define S_EXTPARRESET    16
+#define V_EXTPARRESET(x) ((x) << S_EXTPARRESET)
+#define F_EXTPARRESET    V_EXTPARRESET(1U)
+
+#define S_EXTPARLPBK    15
+#define V_EXTPARLPBK(x) ((x) << S_EXTPARLPBK)
+#define F_EXTPARLPBK    V_EXTPARLPBK(1U)
+
+#define S_MANRXTERMEN    14
+#define V_MANRXTERMEN(x) ((x) << S_MANRXTERMEN)
+#define F_MANRXTERMEN    V_MANRXTERMEN(1U)
+
+#define S_MANBEACONTXEN    13
+#define V_MANBEACONTXEN(x) ((x) << S_MANBEACONTXEN)
+#define F_MANBEACONTXEN    V_MANBEACONTXEN(1U)
+
+#define S_MANRXDETECTEN    12
+#define V_MANRXDETECTEN(x) ((x) << S_MANRXDETECTEN)
+#define F_MANRXDETECTEN    V_MANRXDETECTEN(1U)
+
+#define S_MANTXIDLEEN    11
+#define V_MANTXIDLEEN(x) ((x) << S_MANTXIDLEEN)
+#define F_MANTXIDLEEN    V_MANTXIDLEEN(1U)
+
+#define S_MANRXIDLEEN    10
+#define V_MANRXIDLEEN(x) ((x) << S_MANRXIDLEEN)
+#define F_MANRXIDLEEN    V_MANRXIDLEEN(1U)
+
+#define S_MANL1PWRDN    9
+#define V_MANL1PWRDN(x) ((x) << S_MANL1PWRDN)
+#define F_MANL1PWRDN    V_MANL1PWRDN(1U)
+
+#define S_MANRESET    8
+#define V_MANRESET(x) ((x) << S_MANRESET)
+#define F_MANRESET    V_MANRESET(1U)
+
+#define S_MANFMOFFSET    3
+#define M_MANFMOFFSET    0x1f
+#define V_MANFMOFFSET(x) ((x) << S_MANFMOFFSET)
+#define G_MANFMOFFSET(x) (((x) >> S_MANFMOFFSET) & M_MANFMOFFSET)
+
+#define S_MANFMOFFSETEN    2
+#define V_MANFMOFFSETEN(x) ((x) << S_MANFMOFFSETEN)
+#define F_MANFMOFFSETEN    V_MANFMOFFSETEN(1U)
+
+#define S_MANLANEEN    1
+#define V_MANLANEEN(x) ((x) << S_MANLANEEN)
+#define F_MANLANEEN    V_MANLANEEN(1U)
+
+#define S_INTSERLPBK    0
+#define V_INTSERLPBK(x) ((x) << S_INTSERLPBK)
+#define F_INTSERLPBK    V_INTSERLPBK(1U)
+
 #define A_PCIE_SERDES_STATUS1 0xb4
 
 #define S_CMULOCK    31
@@ -1441,77 +1859,40 @@ $FreeBSD$
 #define V_PCIE_RXOFLOWLANE0(x) ((x) << S_PCIE_RXOFLOWLANE0)
 #define F_PCIE_RXOFLOWLANE0    V_PCIE_RXOFLOWLANE0(1U)
 
-#define A_PCIE_SERDES_LANE_CTRL 0xb4
-
-#define S_EXTBISTCHKERRCLR    22
-#define V_EXTBISTCHKERRCLR(x) ((x) << S_EXTBISTCHKERRCLR)
-#define F_EXTBISTCHKERRCLR    V_EXTBISTCHKERRCLR(1U)
-
-#define S_EXTBISTCHKEN    21
-#define V_EXTBISTCHKEN(x) ((x) << S_EXTBISTCHKEN)
-#define F_EXTBISTCHKEN    V_EXTBISTCHKEN(1U)
-
-#define S_EXTBISTGENEN    20
-#define V_EXTBISTGENEN(x) ((x) << S_EXTBISTGENEN)
-#define F_EXTBISTGENEN    V_EXTBISTGENEN(1U)
-
-#define S_EXTBISTPAT    17
-#define M_EXTBISTPAT    0x7
-#define V_EXTBISTPAT(x) ((x) << S_EXTBISTPAT)
-#define G_EXTBISTPAT(x) (((x) >> S_EXTBISTPAT) & M_EXTBISTPAT)
-
-#define S_EXTPARRESET    16
-#define V_EXTPARRESET(x) ((x) << S_EXTPARRESET)
-#define F_EXTPARRESET    V_EXTPARRESET(1U)
-
-#define S_EXTPARLPBK    15
-#define V_EXTPARLPBK(x) ((x) << S_EXTPARLPBK)
-#define F_EXTPARLPBK    V_EXTPARLPBK(1U)
-
-#define S_MANRXTERMEN    14
-#define V_MANRXTERMEN(x) ((x) << S_MANRXTERMEN)
-#define F_MANRXTERMEN    V_MANRXTERMEN(1U)
-
-#define S_MANBEACONTXEN    13
-#define V_MANBEACONTXEN(x) ((x) << S_MANBEACONTXEN)
-#define F_MANBEACONTXEN    V_MANBEACONTXEN(1U)
-
-#define S_MANRXDETECTEN    12
-#define V_MANRXDETECTEN(x) ((x) << S_MANRXDETECTEN)
-#define F_MANRXDETECTEN    V_MANRXDETECTEN(1U)
+#define A_PCIE_SERDES_LANE_STAT 0xb8
 
-#define S_MANTXIDLEEN    11
-#define V_MANTXIDLEEN(x) ((x) << S_MANTXIDLEEN)
-#define F_MANTXIDLEEN    V_MANTXIDLEEN(1U)
+#define S_EXTBISTCHKERRCNT    8
+#define M_EXTBISTCHKERRCNT    0xffffff
+#define V_EXTBISTCHKERRCNT(x) ((x) << S_EXTBISTCHKERRCNT)
+#define G_EXTBISTCHKERRCNT(x) (((x) >> S_EXTBISTCHKERRCNT) & M_EXTBISTCHKERRCNT)
 
-#define S_MANRXIDLEEN    10
-#define V_MANRXIDLEEN(x) ((x) << S_MANRXIDLEEN)
-#define F_MANRXIDLEEN    V_MANRXIDLEEN(1U)
+#define S_EXTBISTCHKFMD    7
+#define V_EXTBISTCHKFMD(x) ((x) << S_EXTBISTCHKFMD)
+#define F_EXTBISTCHKFMD    V_EXTBISTCHKFMD(1U)
 
-#define S_MANL1PWRDN    9
-#define V_MANL1PWRDN(x) ((x) << S_MANL1PWRDN)
-#define F_MANL1PWRDN    V_MANL1PWRDN(1U)
+#define S_BEACONDETECTCHG    6
+#define V_BEACONDETECTCHG(x) ((x) << S_BEACONDETECTCHG)
+#define F_BEACONDETECTCHG    V_BEACONDETECTCHG(1U)
 
-#define S_MANRESET    8
-#define V_MANRESET(x) ((x) << S_MANRESET)
-#define F_MANRESET    V_MANRESET(1U)
+#define S_RXDETECTCHG    5
+#define V_RXDETECTCHG(x) ((x) << S_RXDETECTCHG)
+#define F_RXDETECTCHG    V_RXDETECTCHG(1U)
 
-#define S_MANFMOFFSET    3
-#define M_MANFMOFFSET    0x1f
-#define V_MANFMOFFSET(x) ((x) << S_MANFMOFFSET)
-#define G_MANFMOFFSET(x) (((x) >> S_MANFMOFFSET) & M_MANFMOFFSET)
+#define S_TXIDLEDETECTCHG    4
+#define V_TXIDLEDETECTCHG(x) ((x) << S_TXIDLEDETECTCHG)
+#define F_TXIDLEDETECTCHG    V_TXIDLEDETECTCHG(1U)
 
-#define S_MANFMOFFSETEN    2
-#define V_MANFMOFFSETEN(x) ((x) << S_MANFMOFFSETEN)
-#define F_MANFMOFFSETEN    V_MANFMOFFSETEN(1U)
+#define S_BEACONDETECT    2
+#define V_BEACONDETECT(x) ((x) << S_BEACONDETECT)
+#define F_BEACONDETECT    V_BEACONDETECT(1U)
 
-#define S_MANLANEEN    1
-#define V_MANLANEEN(x) ((x) << S_MANLANEEN)
-#define F_MANLANEEN    V_MANLANEEN(1U)
+#define S_RXDETECT    1
+#define V_RXDETECT(x) ((x) << S_RXDETECT)
+#define F_RXDETECT    V_RXDETECT(1U)
 
-#define S_INTSERLPBK    0
-#define V_INTSERLPBK(x) ((x) << S_INTSERLPBK)
-#define F_INTSERLPBK    V_INTSERLPBK(1U)
+#define S_TXIDLEDETECT    0
+#define V_TXIDLEDETECT(x) ((x) << S_TXIDLEDETECT)
+#define F_TXIDLEDETECT    V_TXIDLEDETECT(1U)
 
 #define A_PCIE_SERDES_STATUS2 0xb8
 
@@ -1643,40 +2024,22 @@ $FreeBSD$
 #define V_PCIE_RXADDSKIPLANE0(x) ((x) << S_PCIE_RXADDSKIPLANE0)
 #define F_PCIE_RXADDSKIPLANE0    V_PCIE_RXADDSKIPLANE0(1U)
 
-#define A_PCIE_SERDES_LANE_STAT 0xb8
-
-#define S_EXTBISTCHKERRCNT    8
-#define M_EXTBISTCHKERRCNT    0xffffff
-#define V_EXTBISTCHKERRCNT(x) ((x) << S_EXTBISTCHKERRCNT)
-#define G_EXTBISTCHKERRCNT(x) (((x) >> S_EXTBISTCHKERRCNT) & M_EXTBISTCHKERRCNT)
-
-#define S_EXTBISTCHKFMD    7
-#define V_EXTBISTCHKFMD(x) ((x) << S_EXTBISTCHKFMD)
-#define F_EXTBISTCHKFMD    V_EXTBISTCHKFMD(1U)
-
-#define S_BEACONDETECTCHG    6
-#define V_BEACONDETECTCHG(x) ((x) << S_BEACONDETECTCHG)
-#define F_BEACONDETECTCHG    V_BEACONDETECTCHG(1U)
-
-#define S_RXDETECTCHG    5
-#define V_RXDETECTCHG(x) ((x) << S_RXDETECTCHG)
-#define F_RXDETECTCHG    V_RXDETECTCHG(1U)
+#define A_PCIE_PEX_WMARK 0xbc
 
-#define S_TXIDLEDETECTCHG    4
-#define V_TXIDLEDETECTCHG(x) ((x) << S_TXIDLEDETECTCHG)
-#define F_TXIDLEDETECTCHG    V_TXIDLEDETECTCHG(1U)
+#define S_P_WMARK    18
+#define M_P_WMARK    0x7ff
+#define V_P_WMARK(x) ((x) << S_P_WMARK)
+#define G_P_WMARK(x) (((x) >> S_P_WMARK) & M_P_WMARK)
 
-#define S_BEACONDETECT    2
-#define V_BEACONDETECT(x) ((x) << S_BEACONDETECT)
-#define F_BEACONDETECT    V_BEACONDETECT(1U)
+#define S_NP_WMARK    11
+#define M_NP_WMARK    0x7f
+#define V_NP_WMARK(x) ((x) << S_NP_WMARK)
+#define G_NP_WMARK(x) (((x) >> S_NP_WMARK) & M_NP_WMARK)
 
-#define S_RXDETECT    1
-#define V_RXDETECT(x) ((x) << S_RXDETECT)
-#define F_RXDETECT    V_RXDETECT(1U)
-
-#define S_TXIDLEDETECT    0
-#define V_TXIDLEDETECT(x) ((x) << S_TXIDLEDETECT)
-#define F_TXIDLEDETECT    V_TXIDLEDETECT(1U)
+#define S_CPL_WMARK    0
+#define M_CPL_WMARK    0x7ff
+#define V_CPL_WMARK(x) ((x) << S_CPL_WMARK)
+#define G_CPL_WMARK(x) (((x) >> S_CPL_WMARK) & M_CPL_WMARK)
 
 #define A_PCIE_SERDES_BIST 0xbc
 
@@ -1831,54 +2194,6 @@ $FreeBSD$
 
 #define A_T3DBG_GPIO_IN 0xd4
 
-#define S_GPIO11_IN    11
-#define V_GPIO11_IN(x) ((x) << S_GPIO11_IN)
-#define F_GPIO11_IN    V_GPIO11_IN(1U)
-
-#define S_GPIO10_IN    10
-#define V_GPIO10_IN(x) ((x) << S_GPIO10_IN)
-#define F_GPIO10_IN    V_GPIO10_IN(1U)
-
-#define S_GPIO9_IN    9
-#define V_GPIO9_IN(x) ((x) << S_GPIO9_IN)
-#define F_GPIO9_IN    V_GPIO9_IN(1U)
-
-#define S_GPIO8_IN    8
-#define V_GPIO8_IN(x) ((x) << S_GPIO8_IN)
-#define F_GPIO8_IN    V_GPIO8_IN(1U)
-
-#define S_GPIO7_IN    7
-#define V_GPIO7_IN(x) ((x) << S_GPIO7_IN)
-#define F_GPIO7_IN    V_GPIO7_IN(1U)
-
-#define S_GPIO6_IN    6
-#define V_GPIO6_IN(x) ((x) << S_GPIO6_IN)
-#define F_GPIO6_IN    V_GPIO6_IN(1U)
-
-#define S_GPIO5_IN    5
-#define V_GPIO5_IN(x) ((x) << S_GPIO5_IN)
-#define F_GPIO5_IN    V_GPIO5_IN(1U)
-
-#define S_GPIO4_IN    4
-#define V_GPIO4_IN(x) ((x) << S_GPIO4_IN)
-#define F_GPIO4_IN    V_GPIO4_IN(1U)
-
-#define S_GPIO3_IN    3
-#define V_GPIO3_IN(x) ((x) << S_GPIO3_IN)
-#define F_GPIO3_IN    V_GPIO3_IN(1U)
-
-#define S_GPIO2_IN    2
-#define V_GPIO2_IN(x) ((x) << S_GPIO2_IN)
-#define F_GPIO2_IN    V_GPIO2_IN(1U)
-
-#define S_GPIO1_IN    1
-#define V_GPIO1_IN(x) ((x) << S_GPIO1_IN)
-#define F_GPIO1_IN    V_GPIO1_IN(1U)
-
-#define S_GPIO0_IN    0
-#define V_GPIO0_IN(x) ((x) << S_GPIO0_IN)
-#define F_GPIO0_IN    V_GPIO0_IN(1U)
-
 #define S_GPIO11_CHG_DET    27
 #define V_GPIO11_CHG_DET(x) ((x) << S_GPIO11_CHG_DET)
 #define F_GPIO11_CHG_DET    V_GPIO11_CHG_DET(1U)
@@ -1927,6 +2242,54 @@ $FreeBSD$
 #define V_GPIO0_CHG_DET(x) ((x) << S_GPIO0_CHG_DET)
 #define F_GPIO0_CHG_DET    V_GPIO0_CHG_DET(1U)
 
+#define S_GPIO11_IN    11
+#define V_GPIO11_IN(x) ((x) << S_GPIO11_IN)
+#define F_GPIO11_IN    V_GPIO11_IN(1U)
+
+#define S_GPIO10_IN    10
+#define V_GPIO10_IN(x) ((x) << S_GPIO10_IN)
+#define F_GPIO10_IN    V_GPIO10_IN(1U)
+
+#define S_GPIO9_IN    9
+#define V_GPIO9_IN(x) ((x) << S_GPIO9_IN)
+#define F_GPIO9_IN    V_GPIO9_IN(1U)
+
+#define S_GPIO8_IN    8
+#define V_GPIO8_IN(x) ((x) << S_GPIO8_IN)
+#define F_GPIO8_IN    V_GPIO8_IN(1U)
+
+#define S_GPIO7_IN    7
+#define V_GPIO7_IN(x) ((x) << S_GPIO7_IN)
+#define F_GPIO7_IN    V_GPIO7_IN(1U)
+
+#define S_GPIO6_IN    6
+#define V_GPIO6_IN(x) ((x) << S_GPIO6_IN)
+#define F_GPIO6_IN    V_GPIO6_IN(1U)
+
+#define S_GPIO5_IN    5
+#define V_GPIO5_IN(x) ((x) << S_GPIO5_IN)
+#define F_GPIO5_IN    V_GPIO5_IN(1U)
+
+#define S_GPIO4_IN    4
+#define V_GPIO4_IN(x) ((x) << S_GPIO4_IN)
+#define F_GPIO4_IN    V_GPIO4_IN(1U)
+
+#define S_GPIO3_IN    3
+#define V_GPIO3_IN(x) ((x) << S_GPIO3_IN)
+#define F_GPIO3_IN    V_GPIO3_IN(1U)
+
+#define S_GPIO2_IN    2
+#define V_GPIO2_IN(x) ((x) << S_GPIO2_IN)
+#define F_GPIO2_IN    V_GPIO2_IN(1U)
+
+#define S_GPIO1_IN    1
+#define V_GPIO1_IN(x) ((x) << S_GPIO1_IN)
+#define F_GPIO1_IN    V_GPIO1_IN(1U)
+
+#define S_GPIO0_IN    0
+#define V_GPIO0_IN(x) ((x) << S_GPIO0_IN)
+#define F_GPIO0_IN    V_GPIO0_IN(1U)
+
 #define A_T3DBG_INT_ENABLE 0xd8
 
 #define S_C_LOCK    21
@@ -1949,10 +2312,6 @@ $FreeBSD$
 #define V_PX_LOCK(x) ((x) << S_PX_LOCK)
 #define F_PX_LOCK    V_PX_LOCK(1U)
 
-#define S_PE_LOCK    16
-#define V_PE_LOCK(x) ((x) << S_PE_LOCK)
-#define F_PE_LOCK    V_PE_LOCK(1U)
-
 #define S_GPIO11    11
 #define V_GPIO11(x) ((x) << S_GPIO11)
 #define F_GPIO11    V_GPIO11(1U)
@@ -2001,12 +2360,17 @@ $FreeBSD$
 #define V_GPIO0(x) ((x) << S_GPIO0)
 #define F_GPIO0    V_GPIO0(1U)
 
+#define S_PE_LOCK    16
+#define V_PE_LOCK(x) ((x) << S_PE_LOCK)
+#define F_PE_LOCK    V_PE_LOCK(1U)
+
 #define A_T3DBG_INT_CAUSE 0xdc
 #define A_T3DBG_DBG0_RST_VALUE 0xe0
 
 #define S_DEBUGDATA    0
+#define M_DEBUGDATA    0xff
 #define V_DEBUGDATA(x) ((x) << S_DEBUGDATA)
-#define F_DEBUGDATA    V_DEBUGDATA(1U)
+#define G_DEBUGDATA(x) (((x) >> S_DEBUGDATA) & M_DEBUGDATA)
 
 #define A_T3DBG_PLL_OCLK_PAD_EN 0xe4
 
@@ -2014,6 +2378,10 @@ $FreeBSD$
 #define V_PCIE_OCLK_EN(x) ((x) << S_PCIE_OCLK_EN)
 #define F_PCIE_OCLK_EN    V_PCIE_OCLK_EN(1U)
 
+#define S_PCLKTREE_DBG_EN    17
+#define V_PCLKTREE_DBG_EN(x) ((x) << S_PCLKTREE_DBG_EN)
+#define F_PCLKTREE_DBG_EN    V_PCLKTREE_DBG_EN(1U)
+
 #define S_PCIX_OCLK_EN    16
 #define V_PCIX_OCLK_EN(x) ((x) << S_PCIX_OCLK_EN)
 #define F_PCIX_OCLK_EN    V_PCIX_OCLK_EN(1U)
@@ -2034,16 +2402,8 @@ $FreeBSD$
 #define V_C_OCLK_EN(x) ((x) << S_C_OCLK_EN)
 #define F_C_OCLK_EN    V_C_OCLK_EN(1U)
 
-#define S_PCLKTREE_DBG_EN    17
-#define V_PCLKTREE_DBG_EN(x) ((x) << S_PCLKTREE_DBG_EN)
-#define F_PCLKTREE_DBG_EN    V_PCLKTREE_DBG_EN(1U)
-
 #define A_T3DBG_PLL_LOCK 0xe8
 
-#define S_PCIE_LOCK    20
-#define V_PCIE_LOCK(x) ((x) << S_PCIE_LOCK)
-#define F_PCIE_LOCK    V_PCIE_LOCK(1U)
-
 #define S_PCIX_LOCK    16
 #define V_PCIX_LOCK(x) ((x) << S_PCIX_LOCK)
 #define F_PCIX_LOCK    V_PCIX_LOCK(1U)
@@ -2064,11 +2424,16 @@ $FreeBSD$
 #define V_PLL_C_LOCK(x) ((x) << S_PLL_C_LOCK)
 #define F_PLL_C_LOCK    V_PLL_C_LOCK(1U)
 
+#define S_PCIE_LOCK    20
+#define V_PCIE_LOCK(x) ((x) << S_PCIE_LOCK)
+#define F_PCIE_LOCK    V_PCIE_LOCK(1U)
+
 #define A_T3DBG_SERDES_RBC_CFG 0xec
 
 #define S_X_RBC_LANE_SEL    16
+#define M_X_RBC_LANE_SEL    0x3
 #define V_X_RBC_LANE_SEL(x) ((x) << S_X_RBC_LANE_SEL)
-#define F_X_RBC_LANE_SEL    V_X_RBC_LANE_SEL(1U)
+#define G_X_RBC_LANE_SEL(x) (((x) >> S_X_RBC_LANE_SEL) & M_X_RBC_LANE_SEL)
 
 #define S_X_RBC_DBG_EN    12
 #define V_X_RBC_DBG_EN(x) ((x) << S_X_RBC_DBG_EN)
@@ -2079,8 +2444,9 @@ $FreeBSD$
 #define F_X_SERDES_SEL    V_X_SERDES_SEL(1U)
 
 #define S_PE_RBC_LANE_SEL    4
+#define M_PE_RBC_LANE_SEL    0x7
 #define V_PE_RBC_LANE_SEL(x) ((x) << S_PE_RBC_LANE_SEL)
-#define F_PE_RBC_LANE_SEL    V_PE_RBC_LANE_SEL(1U)
+#define G_PE_RBC_LANE_SEL(x) (((x) >> S_PE_RBC_LANE_SEL) & M_PE_RBC_LANE_SEL)
 
 #define S_PE_RBC_DBG_EN    0
 #define V_PE_RBC_DBG_EN(x) ((x) << S_PE_RBC_DBG_EN)
@@ -2108,10 +2474,6 @@ $FreeBSD$
 #define V_PX_LOCK_ACT_LOW(x) ((x) << S_PX_LOCK_ACT_LOW)
 #define F_PX_LOCK_ACT_LOW    V_PX_LOCK_ACT_LOW(1U)
 
-#define S_PE_LOCK_ACT_LOW    16
-#define V_PE_LOCK_ACT_LOW(x) ((x) << S_PE_LOCK_ACT_LOW)
-#define F_PE_LOCK_ACT_LOW    V_PE_LOCK_ACT_LOW(1U)
-
 #define S_GPIO11_ACT_LOW    11
 #define V_GPIO11_ACT_LOW(x) ((x) << S_GPIO11_ACT_LOW)
 #define F_GPIO11_ACT_LOW    V_GPIO11_ACT_LOW(1U)
@@ -2160,6 +2522,10 @@ $FreeBSD$
 #define V_GPIO0_ACT_LOW(x) ((x) << S_GPIO0_ACT_LOW)
 #define F_GPIO0_ACT_LOW    V_GPIO0_ACT_LOW(1U)
 
+#define S_PE_LOCK_ACT_LOW    16
+#define V_PE_LOCK_ACT_LOW(x) ((x) << S_PE_LOCK_ACT_LOW)
+#define F_PE_LOCK_ACT_LOW    V_PE_LOCK_ACT_LOW(1U)
+
 #define A_T3DBG_PMON_CFG 0xf4
 
 #define S_PMON_DONE    29
@@ -2171,20 +2537,24 @@ $FreeBSD$
 #define F_PMON_FAIL    V_PMON_FAIL(1U)
 
 #define S_PMON_FDEL_AUTO    22
+#define M_PMON_FDEL_AUTO    0x3f
 #define V_PMON_FDEL_AUTO(x) ((x) << S_PMON_FDEL_AUTO)
-#define F_PMON_FDEL_AUTO    V_PMON_FDEL_AUTO(1U)
+#define G_PMON_FDEL_AUTO(x) (((x) >> S_PMON_FDEL_AUTO) & M_PMON_FDEL_AUTO)
 
 #define S_PMON_CDEL_AUTO    16
+#define M_PMON_CDEL_AUTO    0x3f
 #define V_PMON_CDEL_AUTO(x) ((x) << S_PMON_CDEL_AUTO)
-#define F_PMON_CDEL_AUTO    V_PMON_CDEL_AUTO(1U)
+#define G_PMON_CDEL_AUTO(x) (((x) >> S_PMON_CDEL_AUTO) & M_PMON_CDEL_AUTO)
 
 #define S_PMON_FDEL_MANUAL    10
+#define M_PMON_FDEL_MANUAL    0x3f
 #define V_PMON_FDEL_MANUAL(x) ((x) << S_PMON_FDEL_MANUAL)
-#define F_PMON_FDEL_MANUAL    V_PMON_FDEL_MANUAL(1U)
+#define G_PMON_FDEL_MANUAL(x) (((x) >> S_PMON_FDEL_MANUAL) & M_PMON_FDEL_MANUAL)
 
 #define S_PMON_CDEL_MANUAL    4
+#define M_PMON_CDEL_MANUAL    0x3f
 #define V_PMON_CDEL_MANUAL(x) ((x) << S_PMON_CDEL_MANUAL)
-#define F_PMON_CDEL_MANUAL    V_PMON_CDEL_MANUAL(1U)
+#define G_PMON_CDEL_MANUAL(x) (((x) >> S_PMON_CDEL_MANUAL) & M_PMON_CDEL_MANUAL)
 
 #define S_PMON_MANUAL    1
 #define V_PMON_MANUAL(x) ((x) << S_PMON_MANUAL)
@@ -2740,6 +3110,54 @@ $FreeBSD$
 
 #define A_CIM_HOST_INT_ENABLE 0x298
 
+#define S_DTAGPARERR    28
+#define V_DTAGPARERR(x) ((x) << S_DTAGPARERR)
+#define F_DTAGPARERR    V_DTAGPARERR(1U)
+
+#define S_ITAGPARERR    27
+#define V_ITAGPARERR(x) ((x) << S_ITAGPARERR)
+#define F_ITAGPARERR    V_ITAGPARERR(1U)
+
+#define S_IBQTPPARERR    26
+#define V_IBQTPPARERR(x) ((x) << S_IBQTPPARERR)
+#define F_IBQTPPARERR    V_IBQTPPARERR(1U)
+
+#define S_IBQULPPARERR    25
+#define V_IBQULPPARERR(x) ((x) << S_IBQULPPARERR)
+#define F_IBQULPPARERR    V_IBQULPPARERR(1U)
+
+#define S_IBQSGEHIPARERR    24
+#define V_IBQSGEHIPARERR(x) ((x) << S_IBQSGEHIPARERR)
+#define F_IBQSGEHIPARERR    V_IBQSGEHIPARERR(1U)
+
+#define S_IBQSGELOPARERR    23
+#define V_IBQSGELOPARERR(x) ((x) << S_IBQSGELOPARERR)
+#define F_IBQSGELOPARERR    V_IBQSGELOPARERR(1U)
+
+#define S_OBQULPLOPARERR    22
+#define V_OBQULPLOPARERR(x) ((x) << S_OBQULPLOPARERR)
+#define F_OBQULPLOPARERR    V_OBQULPLOPARERR(1U)
+
+#define S_OBQULPHIPARERR    21
+#define V_OBQULPHIPARERR(x) ((x) << S_OBQULPHIPARERR)
+#define F_OBQULPHIPARERR    V_OBQULPHIPARERR(1U)
+
+#define S_OBQSGEPARERR    20
+#define V_OBQSGEPARERR(x) ((x) << S_OBQSGEPARERR)
+#define F_OBQSGEPARERR    V_OBQSGEPARERR(1U)
+
+#define S_DCACHEPARERR    19
+#define V_DCACHEPARERR(x) ((x) << S_DCACHEPARERR)
+#define F_DCACHEPARERR    V_DCACHEPARERR(1U)
+
+#define S_ICACHEPARERR    18
+#define V_ICACHEPARERR(x) ((x) << S_ICACHEPARERR)
+#define F_ICACHEPARERR    V_ICACHEPARERR(1U)
+
+#define S_DRAMPARERR    17
+#define V_DRAMPARERR(x) ((x) << S_DRAMPARERR)
+#define F_DRAMPARERR    V_DRAMPARERR(1U)
+
 #define S_TIMER1INTEN    15
 #define V_TIMER1INTEN(x) ((x) << S_TIMER1INTEN)
 #define F_TIMER1INTEN    V_TIMER1INTEN(1U)
@@ -3043,6 +3461,10 @@ $FreeBSD$
 #define V_DBMAXOPCNT(x) ((x) << S_DBMAXOPCNT)
 #define G_DBMAXOPCNT(x) (((x) >> S_DBMAXOPCNT) & M_DBMAXOPCNT)
 
+#define S_IPV6ENABLE    15
+#define V_IPV6ENABLE(x) ((x) << S_IPV6ENABLE)
+#define F_IPV6ENABLE    V_IPV6ENABLE(1U)
+
 #define S_NICMODE    14
 #define V_NICMODE(x) ((x) << S_NICMODE)
 #define F_NICMODE    V_NICMODE(1U)
@@ -3087,12 +3509,16 @@ $FreeBSD$
 #define V_CTUNNEL(x) ((x) << S_CTUNNEL)
 #define F_CTUNNEL    V_CTUNNEL(1U)
 
-#define S_IPV6ENABLE    15
-#define V_IPV6ENABLE(x) ((x) << S_IPV6ENABLE)
-#define F_IPV6ENABLE    V_IPV6ENABLE(1U)
-
 #define A_TP_OUT_CONFIG 0x304
 
+#define S_IPIDSPLITMODE    16
+#define V_IPIDSPLITMODE(x) ((x) << S_IPIDSPLITMODE)
+#define F_IPIDSPLITMODE    V_IPIDSPLITMODE(1U)
+
+#define S_VLANEXTRACTIONENABLE2NDPORT    13
+#define V_VLANEXTRACTIONENABLE2NDPORT(x) ((x) << S_VLANEXTRACTIONENABLE2NDPORT)
+#define F_VLANEXTRACTIONENABLE2NDPORT    V_VLANEXTRACTIONENABLE2NDPORT(1U)
+
 #define S_VLANEXTRACTIONENABLE    12
 #define V_VLANEXTRACTIONENABLE(x) ((x) << S_VLANEXTRACTIONENABLE)
 #define F_VLANEXTRACTIONENABLE    V_VLANEXTRACTIONENABLE(1U)
@@ -3129,16 +3555,13 @@ $FreeBSD$
 #define V_OUT_CETHERNET(x) ((x) << S_OUT_CETHERNET)
 #define F_OUT_CETHERNET    V_OUT_CETHERNET(1U)
 
-#define S_IPIDSPLITMODE    16
-#define V_IPIDSPLITMODE(x) ((x) << S_IPIDSPLITMODE)
-#define F_IPIDSPLITMODE    V_IPIDSPLITMODE(1U)
-
-#define S_VLANEXTRACTIONENABLE2NDPORT    13
-#define V_VLANEXTRACTIONENABLE2NDPORT(x) ((x) << S_VLANEXTRACTIONENABLE2NDPORT)
-#define F_VLANEXTRACTIONENABLE2NDPORT    V_VLANEXTRACTIONENABLE2NDPORT(1U)
-
 #define A_TP_GLOBAL_CONFIG 0x308
 
+#define S_SYNCOOKIEPARAMS    26
+#define M_SYNCOOKIEPARAMS    0x3f
+#define V_SYNCOOKIEPARAMS(x) ((x) << S_SYNCOOKIEPARAMS)
+#define G_SYNCOOKIEPARAMS(x) (((x) >> S_SYNCOOKIEPARAMS) & M_SYNCOOKIEPARAMS)
+
 #define S_RXFLOWCONTROLDISABLE    25
 #define V_RXFLOWCONTROLDISABLE(x) ((x) << S_RXFLOWCONTROLDISABLE)
 #define F_RXFLOWCONTROLDISABLE    V_RXFLOWCONTROLDISABLE(1U)
@@ -3206,11 +3629,6 @@ $FreeBSD$
 #define V_IPTTL(x) ((x) << S_IPTTL)
 #define G_IPTTL(x) (((x) >> S_IPTTL) & M_IPTTL)
 
-#define S_SYNCOOKIEPARAMS    26
-#define M_SYNCOOKIEPARAMS    0x3f
-#define V_SYNCOOKIEPARAMS(x) ((x) << S_SYNCOOKIEPARAMS)
-#define G_SYNCOOKIEPARAMS(x) (((x) >> S_SYNCOOKIEPARAMS) & M_SYNCOOKIEPARAMS)
-
 #define A_TP_GLOBAL_RX_CREDIT 0x30c
 #define A_TP_CMM_SIZE 0x310
 
@@ -3228,16 +3646,16 @@ $FreeBSD$
 
 #define A_TP_CMM_TIMER_BASE 0x318
 
-#define S_CMTIMERBASE    0
-#define M_CMTIMERBASE    0xfffffff
-#define V_CMTIMERBASE(x) ((x) << S_CMTIMERBASE)
-#define G_CMTIMERBASE(x) (((x) >> S_CMTIMERBASE) & M_CMTIMERBASE)
-
 #define S_CMTIMERMAXNUM    28
 #define M_CMTIMERMAXNUM    0x3
 #define V_CMTIMERMAXNUM(x) ((x) << S_CMTIMERMAXNUM)
 #define G_CMTIMERMAXNUM(x) (((x) >> S_CMTIMERMAXNUM) & M_CMTIMERMAXNUM)
 
+#define S_CMTIMERBASE    0
+#define M_CMTIMERBASE    0xfffffff
+#define V_CMTIMERBASE(x) ((x) << S_CMTIMERBASE)
+#define G_CMTIMERBASE(x) (((x) >> S_CMTIMERBASE) & M_CMTIMERBASE)
+
 #define A_TP_PMM_SIZE 0x31c
 
 #define S_PMSIZE    0
@@ -3339,6 +3757,26 @@ $FreeBSD$
 
 #define A_TP_PC_CONFIG 0x348
 
+#define S_CMCACHEDISABLE    31
+#define V_CMCACHEDISABLE(x) ((x) << S_CMCACHEDISABLE)
+#define F_CMCACHEDISABLE    V_CMCACHEDISABLE(1U)
+
+#define S_ENABLEOCSPIFULL    30
+#define V_ENABLEOCSPIFULL(x) ((x) << S_ENABLEOCSPIFULL)
+#define F_ENABLEOCSPIFULL    V_ENABLEOCSPIFULL(1U)
+
+#define S_ENABLEFLMERRORDDP    29
+#define V_ENABLEFLMERRORDDP(x) ((x) << S_ENABLEFLMERRORDDP)
+#define F_ENABLEFLMERRORDDP    V_ENABLEFLMERRORDDP(1U)
+
+#define S_LOCKTID    28
+#define V_LOCKTID(x) ((x) << S_LOCKTID)
+#define F_LOCKTID    V_LOCKTID(1U)
+
+#define S_FIXRCVWND    27
+#define V_FIXRCVWND(x) ((x) << S_FIXRCVWND)
+#define F_FIXRCVWND    V_FIXRCVWND(1U)
+
 #define S_TXTOSQUEUEMAPMODE    26
 #define V_TXTOSQUEUEMAPMODE(x) ((x) << S_TXTOSQUEUEMAPMODE)
 #define F_TXTOSQUEUEMAPMODE    V_TXTOSQUEUEMAPMODE(1U)
@@ -3436,27 +3874,23 @@ $FreeBSD$
 #define V_TABLELATENCYDELTA(x) ((x) << S_TABLELATENCYDELTA)
 #define G_TABLELATENCYDELTA(x) (((x) >> S_TABLELATENCYDELTA) & M_TABLELATENCYDELTA)
 
-#define S_CMCACHEDISABLE    31
-#define V_CMCACHEDISABLE(x) ((x) << S_CMCACHEDISABLE)
-#define F_CMCACHEDISABLE    V_CMCACHEDISABLE(1U)
+#define A_TP_PC_CONFIG2 0x34c
 
-#define S_ENABLEOCSPIFULL    30
-#define V_ENABLEOCSPIFULL(x) ((x) << S_ENABLEOCSPIFULL)
-#define F_ENABLEOCSPIFULL    V_ENABLEOCSPIFULL(1U)
+#define S_DISBLEDAPARBIT0    15
+#define V_DISBLEDAPARBIT0(x) ((x) << S_DISBLEDAPARBIT0)
+#define F_DISBLEDAPARBIT0    V_DISBLEDAPARBIT0(1U)
 
-#define S_ENABLEFLMERRORDDP    29
-#define V_ENABLEFLMERRORDDP(x) ((x) << S_ENABLEFLMERRORDDP)
-#define F_ENABLEFLMERRORDDP    V_ENABLEFLMERRORDDP(1U)
+#define S_ENABLEARPMISS    13
+#define V_ENABLEARPMISS(x) ((x) << S_ENABLEARPMISS)
+#define F_ENABLEARPMISS    V_ENABLEARPMISS(1U)
 
-#define S_LOCKTID    28
-#define V_LOCKTID(x) ((x) << S_LOCKTID)
-#define F_LOCKTID    V_LOCKTID(1U)
+#define S_ENABLENONOFDTNLSYN    12
+#define V_ENABLENONOFDTNLSYN(x) ((x) << S_ENABLENONOFDTNLSYN)
+#define F_ENABLENONOFDTNLSYN    V_ENABLENONOFDTNLSYN(1U)
 
-#define S_FIXRCVWND    27
-#define V_FIXRCVWND(x) ((x) << S_FIXRCVWND)
-#define F_FIXRCVWND    V_FIXRCVWND(1U)
-
-#define A_TP_PC_CONFIG2 0x34c
+#define S_ENABLEIPV6RSS    11
+#define V_ENABLEIPV6RSS(x) ((x) << S_ENABLEIPV6RSS)
+#define F_ENABLEIPV6RSS    V_ENABLEIPV6RSS(1U)
 
 #define S_ENABLEDROPRQEMPTYPKT    10
 #define V_ENABLEDROPRQEMPTYPKT(x) ((x) << S_ENABLEDROPRQEMPTYPKT)
@@ -3482,9 +3916,9 @@ $FreeBSD$
 #define V_ENABLETXPORTFROMDA(x) ((x) << S_ENABLETXPORTFROMDA)
 #define F_ENABLETXPORTFROMDA    V_ENABLETXPORTFROMDA(1U)
 
-#define S_CHDRAFULL    4
-#define V_CHDRAFULL(x) ((x) << S_CHDRAFULL)
-#define F_CHDRAFULL    V_CHDRAFULL(1U)
+#define S_ENABLECHDRAFULL    4
+#define V_ENABLECHDRAFULL(x) ((x) << S_ENABLECHDRAFULL)
+#define F_ENABLECHDRAFULL    V_ENABLECHDRAFULL(1U)
 
 #define S_ENABLENONOFDSCBBIT    3
 #define V_ENABLENONOFDSCBBIT(x) ((x) << S_ENABLENONOFDSCBBIT)
@@ -3502,6 +3936,10 @@ $FreeBSD$
 #define V_ENABLEOLDRXFORWARD(x) ((x) << S_ENABLEOLDRXFORWARD)
 #define F_ENABLEOLDRXFORWARD    V_ENABLEOLDRXFORWARD(1U)
 
+#define S_CHDRAFULL    4
+#define V_CHDRAFULL(x) ((x) << S_CHDRAFULL)
+#define F_CHDRAFULL    V_CHDRAFULL(1U)
+
 #define A_TP_TCP_BACKOFF_REG0 0x350
 
 #define S_TIMERBACKOFFINDEX3    24
@@ -3662,6 +4100,10 @@ $FreeBSD$
 #define V_TXPACEAUTO(x) ((x) << S_TXPACEAUTO)
 #define F_TXPACEAUTO    V_TXPACEAUTO(1U)
 
+#define S_RXURGTUNNEL    6
+#define V_RXURGTUNNEL(x) ((x) << S_RXURGTUNNEL)
+#define F_RXURGTUNNEL    V_RXURGTUNNEL(1U)
+
 #define S_RXURGMODE    5
 #define V_RXURGMODE(x) ((x) << S_RXURGMODE)
 #define F_RXURGMODE    V_RXURGMODE(1U)
@@ -3683,10 +4125,6 @@ $FreeBSD$
 #define V_RXCOALESCEPSHEN(x) ((x) << S_RXCOALESCEPSHEN)
 #define F_RXCOALESCEPSHEN    V_RXCOALESCEPSHEN(1U)
 
-#define S_RXURGTUNNEL    6
-#define V_RXURGTUNNEL(x) ((x) << S_RXURGTUNNEL)
-#define F_RXURGTUNNEL    V_RXURGTUNNEL(1U)
-
 #define A_TP_PARA_REG4 0x370
 
 #define S_HIGHSPEEDCFG    24
@@ -3720,6 +4158,10 @@ $FreeBSD$
 #define V_SCHDENABLE(x) ((x) << S_SCHDENABLE)
 #define F_SCHDENABLE    V_SCHDENABLE(1U)
 
+#define S_RXDDPOFFINIT    3
+#define V_RXDDPOFFINIT(x) ((x) << S_RXDDPOFFINIT)
+#define F_RXDDPOFFINIT    V_RXDDPOFFINIT(1U)
+
 #define S_ONFLYDDPENABLE    2
 #define V_ONFLYDDPENABLE(x) ((x) << S_ONFLYDDPENABLE)
 #define F_ONFLYDDPENABLE    V_ONFLYDDPENABLE(1U)
@@ -3739,33 +4181,33 @@ $FreeBSD$
 #define V_TXPDUSIZEADJ(x) ((x) << S_TXPDUSIZEADJ)
 #define G_TXPDUSIZEADJ(x) (((x) >> S_TXPDUSIZEADJ) & M_TXPDUSIZEADJ)
 
-#define S_ENABLEEPDU    14
-#define V_ENABLEEPDU(x) ((x) << S_ENABLEEPDU)
-#define F_ENABLEEPDU    V_ENABLEEPDU(1U)
+#define S_ENABLEDEFERACK    12
+#define V_ENABLEDEFERACK(x) ((x) << S_ENABLEDEFERACK)
+#define F_ENABLEDEFERACK    V_ENABLEDEFERACK(1U)
 
-#define S_T3A_ENABLEESND    13
-#define V_T3A_ENABLEESND(x) ((x) << S_T3A_ENABLEESND)
-#define F_T3A_ENABLEESND    V_T3A_ENABLEESND(1U)
+#define S_ENABLEESND    11
+#define V_ENABLEESND(x) ((x) << S_ENABLEESND)
+#define F_ENABLEESND    V_ENABLEESND(1U)
 
-#define S_T3A_ENABLECSND    12
-#define V_T3A_ENABLECSND(x) ((x) << S_T3A_ENABLECSND)
-#define F_T3A_ENABLECSND    V_T3A_ENABLECSND(1U)
+#define S_ENABLECSND    10
+#define V_ENABLECSND(x) ((x) << S_ENABLECSND)
+#define F_ENABLECSND    V_ENABLECSND(1U)
 
-#define S_T3A_ENABLEDEFERACK    9
-#define V_T3A_ENABLEDEFERACK(x) ((x) << S_T3A_ENABLEDEFERACK)
-#define F_T3A_ENABLEDEFERACK    V_T3A_ENABLEDEFERACK(1U)
+#define S_ENABLEPDUE    9
+#define V_ENABLEPDUE(x) ((x) << S_ENABLEPDUE)
+#define F_ENABLEPDUE    V_ENABLEPDUE(1U)
 
 #define S_ENABLEPDUC    8
 #define V_ENABLEPDUC(x) ((x) << S_ENABLEPDUC)
 #define F_ENABLEPDUC    V_ENABLEPDUC(1U)
 
-#define S_ENABLEPDUI    7
-#define V_ENABLEPDUI(x) ((x) << S_ENABLEPDUI)
-#define F_ENABLEPDUI    V_ENABLEPDUI(1U)
+#define S_ENABLEBUFI    7
+#define V_ENABLEBUFI(x) ((x) << S_ENABLEBUFI)
+#define F_ENABLEBUFI    V_ENABLEBUFI(1U)
 
-#define S_T3A_ENABLEPDUE    6
-#define V_T3A_ENABLEPDUE(x) ((x) << S_T3A_ENABLEPDUE)
-#define F_T3A_ENABLEPDUE    V_T3A_ENABLEPDUE(1U)
+#define S_ENABLEBUFE    6
+#define V_ENABLEBUFE(x) ((x) << S_ENABLEBUFE)
+#define F_ENABLEBUFE    V_ENABLEBUFE(1U)
 
 #define S_ENABLEDEFER    5
 #define V_ENABLEDEFER(x) ((x) << S_ENABLEDEFER)
@@ -3791,29 +4233,29 @@ $FreeBSD$
 #define V_DISABLEPDUXMT(x) ((x) << S_DISABLEPDUXMT)
 #define F_DISABLEPDUXMT    V_DISABLEPDUXMT(1U)
 
-#define S_ENABLEDEFERACK    12
-#define V_ENABLEDEFERACK(x) ((x) << S_ENABLEDEFERACK)
-#define F_ENABLEDEFERACK    V_ENABLEDEFERACK(1U)
+#define S_ENABLEEPDU    14
+#define V_ENABLEEPDU(x) ((x) << S_ENABLEEPDU)
+#define F_ENABLEEPDU    V_ENABLEEPDU(1U)
 
-#define S_ENABLEESND    11
-#define V_ENABLEESND(x) ((x) << S_ENABLEESND)
-#define F_ENABLEESND    V_ENABLEESND(1U)
+#define S_T3A_ENABLEESND    13
+#define V_T3A_ENABLEESND(x) ((x) << S_T3A_ENABLEESND)
+#define F_T3A_ENABLEESND    V_T3A_ENABLEESND(1U)
 
-#define S_ENABLECSND    10
-#define V_ENABLECSND(x) ((x) << S_ENABLECSND)
-#define F_ENABLECSND    V_ENABLECSND(1U)
+#define S_T3A_ENABLECSND    12
+#define V_T3A_ENABLECSND(x) ((x) << S_T3A_ENABLECSND)
+#define F_T3A_ENABLECSND    V_T3A_ENABLECSND(1U)
 
-#define S_ENABLEPDUE    9
-#define V_ENABLEPDUE(x) ((x) << S_ENABLEPDUE)
-#define F_ENABLEPDUE    V_ENABLEPDUE(1U)
+#define S_T3A_ENABLEDEFERACK    9
+#define V_T3A_ENABLEDEFERACK(x) ((x) << S_T3A_ENABLEDEFERACK)
+#define F_T3A_ENABLEDEFERACK    V_T3A_ENABLEDEFERACK(1U)
 
-#define S_ENABLEBUFI    7
-#define V_ENABLEBUFI(x) ((x) << S_ENABLEBUFI)
-#define F_ENABLEBUFI    V_ENABLEBUFI(1U)
+#define S_ENABLEPDUI    7
+#define V_ENABLEPDUI(x) ((x) << S_ENABLEPDUI)
+#define F_ENABLEPDUI    V_ENABLEPDUI(1U)
 
-#define S_ENABLEBUFE    6
-#define V_ENABLEBUFE(x) ((x) << S_ENABLEBUFE)
-#define F_ENABLEBUFE    V_ENABLEBUFE(1U)
+#define S_T3A_ENABLEPDUE    6
+#define V_T3A_ENABLEPDUE(x) ((x) << S_T3A_ENABLEPDUE)
+#define F_T3A_ENABLEPDUE    V_T3A_ENABLEPDUE(1U)
 
 #define A_TP_PARA_REG7 0x37c
 
@@ -4302,6 +4744,131 @@ $FreeBSD$
 #define G_CMMAXPSTRUCT(x) (((x) >> S_CMMAXPSTRUCT) & M_CMMAXPSTRUCT)
 
 #define A_TP_INT_ENABLE 0x470
+
+#define S_FLMTXFLSTEMPTY    30
+#define V_FLMTXFLSTEMPTY(x) ((x) << S_FLMTXFLSTEMPTY)
+#define F_FLMTXFLSTEMPTY    V_FLMTXFLSTEMPTY(1U)
+
+#define S_FLMRXFLSTEMPTY    29
+#define V_FLMRXFLSTEMPTY(x) ((x) << S_FLMRXFLSTEMPTY)
+#define F_FLMRXFLSTEMPTY    V_FLMRXFLSTEMPTY(1U)
+
+#define S_FLMPERRSET    28
+#define V_FLMPERRSET(x) ((x) << S_FLMPERRSET)
+#define F_FLMPERRSET    V_FLMPERRSET(1U)
+
+#define S_PROTOCOLSRAMPERR    27
+#define V_PROTOCOLSRAMPERR(x) ((x) << S_PROTOCOLSRAMPERR)
+#define F_PROTOCOLSRAMPERR    V_PROTOCOLSRAMPERR(1U)
+
+#define S_ARPLUTPERR    26
+#define V_ARPLUTPERR(x) ((x) << S_ARPLUTPERR)
+#define F_ARPLUTPERR    V_ARPLUTPERR(1U)
+
+#define S_CMRCFOPPERR    25
+#define V_CMRCFOPPERR(x) ((x) << S_CMRCFOPPERR)
+#define F_CMRCFOPPERR    V_CMRCFOPPERR(1U)
+
+#define S_CMCACHEPERR    24
+#define V_CMCACHEPERR(x) ((x) << S_CMCACHEPERR)
+#define F_CMCACHEPERR    V_CMCACHEPERR(1U)
+
+#define S_CMRCFDATAPERR    23
+#define V_CMRCFDATAPERR(x) ((x) << S_CMRCFDATAPERR)
+#define F_CMRCFDATAPERR    V_CMRCFDATAPERR(1U)
+
+#define S_DBL2TLUTPERR    22
+#define V_DBL2TLUTPERR(x) ((x) << S_DBL2TLUTPERR)
+#define F_DBL2TLUTPERR    V_DBL2TLUTPERR(1U)
+
+#define S_DBTXTIDPERR    21
+#define V_DBTXTIDPERR(x) ((x) << S_DBTXTIDPERR)
+#define F_DBTXTIDPERR    V_DBTXTIDPERR(1U)
+
+#define S_DBEXTPERR    20
+#define V_DBEXTPERR(x) ((x) << S_DBEXTPERR)
+#define F_DBEXTPERR    V_DBEXTPERR(1U)
+
+#define S_DBOPPERR    19
+#define V_DBOPPERR(x) ((x) << S_DBOPPERR)
+#define F_DBOPPERR    V_DBOPPERR(1U)
+
+#define S_TMCACHEPERR    18
+#define V_TMCACHEPERR(x) ((x) << S_TMCACHEPERR)
+#define F_TMCACHEPERR    V_TMCACHEPERR(1U)
+
+#define S_ETPOUTCPLFIFOPERR    17
+#define V_ETPOUTCPLFIFOPERR(x) ((x) << S_ETPOUTCPLFIFOPERR)
+#define F_ETPOUTCPLFIFOPERR    V_ETPOUTCPLFIFOPERR(1U)
+
+#define S_ETPOUTTCPFIFOPERR    16
+#define V_ETPOUTTCPFIFOPERR(x) ((x) << S_ETPOUTTCPFIFOPERR)
+#define F_ETPOUTTCPFIFOPERR    V_ETPOUTTCPFIFOPERR(1U)
+
+#define S_ETPOUTIPFIFOPERR    15
+#define V_ETPOUTIPFIFOPERR(x) ((x) << S_ETPOUTIPFIFOPERR)
+#define F_ETPOUTIPFIFOPERR    V_ETPOUTIPFIFOPERR(1U)
+
+#define S_ETPOUTETHFIFOPERR    14
+#define V_ETPOUTETHFIFOPERR(x) ((x) << S_ETPOUTETHFIFOPERR)
+#define F_ETPOUTETHFIFOPERR    V_ETPOUTETHFIFOPERR(1U)
+
+#define S_ETPINCPLFIFOPERR    13
+#define V_ETPINCPLFIFOPERR(x) ((x) << S_ETPINCPLFIFOPERR)
+#define F_ETPINCPLFIFOPERR    V_ETPINCPLFIFOPERR(1U)
+
+#define S_ETPINTCPOPTFIFOPERR    12
+#define V_ETPINTCPOPTFIFOPERR(x) ((x) << S_ETPINTCPOPTFIFOPERR)
+#define F_ETPINTCPOPTFIFOPERR    V_ETPINTCPOPTFIFOPERR(1U)
+
+#define S_ETPINTCPFIFOPERR    11
+#define V_ETPINTCPFIFOPERR(x) ((x) << S_ETPINTCPFIFOPERR)
+#define F_ETPINTCPFIFOPERR    V_ETPINTCPFIFOPERR(1U)
+
+#define S_ETPINIPFIFOPERR    10
+#define V_ETPINIPFIFOPERR(x) ((x) << S_ETPINIPFIFOPERR)
+#define F_ETPINIPFIFOPERR    V_ETPINIPFIFOPERR(1U)
+
+#define S_ETPINETHFIFOPERR    9
+#define V_ETPINETHFIFOPERR(x) ((x) << S_ETPINETHFIFOPERR)
+#define F_ETPINETHFIFOPERR    V_ETPINETHFIFOPERR(1U)
+
+#define S_CTPOUTCPLFIFOPERR    8
+#define V_CTPOUTCPLFIFOPERR(x) ((x) << S_CTPOUTCPLFIFOPERR)
+#define F_CTPOUTCPLFIFOPERR    V_CTPOUTCPLFIFOPERR(1U)
+
+#define S_CTPOUTTCPFIFOPERR    7
+#define V_CTPOUTTCPFIFOPERR(x) ((x) << S_CTPOUTTCPFIFOPERR)
+#define F_CTPOUTTCPFIFOPERR    V_CTPOUTTCPFIFOPERR(1U)
+
+#define S_CTPOUTIPFIFOPERR    6
+#define V_CTPOUTIPFIFOPERR(x) ((x) << S_CTPOUTIPFIFOPERR)
+#define F_CTPOUTIPFIFOPERR    V_CTPOUTIPFIFOPERR(1U)
+
+#define S_CTPOUTETHFIFOPERR    5
+#define V_CTPOUTETHFIFOPERR(x) ((x) << S_CTPOUTETHFIFOPERR)
+#define F_CTPOUTETHFIFOPERR    V_CTPOUTETHFIFOPERR(1U)
+
+#define S_CTPINCPLFIFOPERR    4
+#define V_CTPINCPLFIFOPERR(x) ((x) << S_CTPINCPLFIFOPERR)
+#define F_CTPINCPLFIFOPERR    V_CTPINCPLFIFOPERR(1U)
+
+#define S_CTPINTCPOPFIFOPERR    3
+#define V_CTPINTCPOPFIFOPERR(x) ((x) << S_CTPINTCPOPFIFOPERR)
+#define F_CTPINTCPOPFIFOPERR    V_CTPINTCPOPFIFOPERR(1U)
+
+#define S_CTPINTCPFIFOPERR    2
+#define V_CTPINTCPFIFOPERR(x) ((x) << S_CTPINTCPFIFOPERR)
+#define F_CTPINTCPFIFOPERR    V_CTPINTCPFIFOPERR(1U)
+
+#define S_CTPINIPFIFOPERR    1
+#define V_CTPINIPFIFOPERR(x) ((x) << S_CTPINIPFIFOPERR)
+#define F_CTPINIPFIFOPERR    V_CTPINIPFIFOPERR(1U)
+
+#define S_CTPINETHFIFOPERR    0
+#define V_CTPINETHFIFOPERR(x) ((x) << S_CTPINETHFIFOPERR)
+#define F_CTPINETHFIFOPERR    V_CTPINETHFIFOPERR(1U)
+
 #define A_TP_INT_CAUSE 0x474
 #define A_TP_FLM_FREE_PS_CNT 0x480
 
@@ -4334,16 +4901,6 @@ $FreeBSD$
 #define A_TP_DEBUG_SEL 0x4a8
 #define A_TP_DEBUG_FLAGS 0x4ac
 
-#define S_RXDEBUGFLAGS    16
-#define M_RXDEBUGFLAGS    0xffff
-#define V_RXDEBUGFLAGS(x) ((x) << S_RXDEBUGFLAGS)
-#define G_RXDEBUGFLAGS(x) (((x) >> S_RXDEBUGFLAGS) & M_RXDEBUGFLAGS)
-
-#define S_TXDEBUGFLAGS    0
-#define M_TXDEBUGFLAGS    0xffff
-#define V_TXDEBUGFLAGS(x) ((x) << S_TXDEBUGFLAGS)
-#define G_TXDEBUGFLAGS(x) (((x) >> S_TXDEBUGFLAGS) & M_TXDEBUGFLAGS)
-
 #define S_RXTIMERDACKFIRST    26
 #define V_RXTIMERDACKFIRST(x) ((x) << S_RXTIMERDACKFIRST)
 #define F_RXTIMERDACKFIRST    V_RXTIMERDACKFIRST(1U)
@@ -4436,13 +4993,23 @@ $FreeBSD$
 #define V_TXRCVADVLTMSS(x) ((x) << S_TXRCVADVLTMSS)
 #define F_TXRCVADVLTMSS    V_TXRCVADVLTMSS(1U)
 
+#define S_RXDEBUGFLAGS    16
+#define M_RXDEBUGFLAGS    0xffff
+#define V_RXDEBUGFLAGS(x) ((x) << S_RXDEBUGFLAGS)
+#define G_RXDEBUGFLAGS(x) (((x) >> S_RXDEBUGFLAGS) & M_RXDEBUGFLAGS)
+
+#define S_TXDEBUGFLAGS    0
+#define M_TXDEBUGFLAGS    0xffff
+#define V_TXDEBUGFLAGS(x) ((x) << S_TXDEBUGFLAGS)
+#define G_TXDEBUGFLAGS(x) (((x) >> S_TXDEBUGFLAGS) & M_TXDEBUGFLAGS)
+
+#define A_TP_PROXY_FLOW_CNTL 0x4b0
 #define A_TP_CM_FLOW_CNTL_MODE 0x4b0
 
 #define S_CMFLOWCACHEDISABLE    0
 #define V_CMFLOWCACHEDISABLE(x) ((x) << S_CMFLOWCACHEDISABLE)
 #define F_CMFLOWCACHEDISABLE    V_CMFLOWCACHEDISABLE(1U)
 
-#define A_TP_PROXY_FLOW_CNTL 0x4b0
 #define A_TP_PC_CONGESTION_CNTL 0x4b4
 
 #define S_EDROPTUNNEL    19
@@ -4811,6 +5378,38 @@ $FreeBSD$
 
 #define A_ULPRX_INT_ENABLE 0x504
 
+#define S_DATASELFRAMEERR0    7
+#define V_DATASELFRAMEERR0(x) ((x) << S_DATASELFRAMEERR0)
+#define F_DATASELFRAMEERR0    V_DATASELFRAMEERR0(1U)
+
+#define S_DATASELFRAMEERR1    6
+#define V_DATASELFRAMEERR1(x) ((x) << S_DATASELFRAMEERR1)
+#define F_DATASELFRAMEERR1    V_DATASELFRAMEERR1(1U)
+
+#define S_PCMDMUXPERR    5
+#define V_PCMDMUXPERR(x) ((x) << S_PCMDMUXPERR)
+#define F_PCMDMUXPERR    V_PCMDMUXPERR(1U)
+
+#define S_ARBFPERR    4
+#define V_ARBFPERR(x) ((x) << S_ARBFPERR)
+#define F_ARBFPERR    V_ARBFPERR(1U)
+
+#define S_ARBPF0PERR    3
+#define V_ARBPF0PERR(x) ((x) << S_ARBPF0PERR)
+#define F_ARBPF0PERR    V_ARBPF0PERR(1U)
+
+#define S_ARBPF1PERR    2
+#define V_ARBPF1PERR(x) ((x) << S_ARBPF1PERR)
+#define F_ARBPF1PERR    V_ARBPF1PERR(1U)
+
+#define S_PARERRPCMD    1
+#define V_PARERRPCMD(x) ((x) << S_PARERRPCMD)
+#define F_PARERRPCMD    V_PARERRPCMD(1U)
+
+#define S_PARERRDATA    0
+#define V_PARERRDATA(x) ((x) << S_PARERRDATA)
+#define F_PARERRDATA    V_PARERRDATA(1U)
+
 #define S_PARERR    0
 #define V_PARERR(x) ((x) << S_PARERR)
 #define F_PARERR    V_PARERR(1U)
@@ -4893,12 +5492,40 @@ $FreeBSD$
 
 #define A_ULPTX_CONFIG 0x580
 
+#define S_CFG_CQE_SOP_MASK    1
+#define V_CFG_CQE_SOP_MASK(x) ((x) << S_CFG_CQE_SOP_MASK)
+#define F_CFG_CQE_SOP_MASK    V_CFG_CQE_SOP_MASK(1U)
+
 #define S_CFG_RR_ARB    0
 #define V_CFG_RR_ARB(x) ((x) << S_CFG_RR_ARB)
 #define F_CFG_RR_ARB    V_CFG_RR_ARB(1U)
 
 #define A_ULPTX_INT_ENABLE 0x584
 
+#define S_CMD_FIFO_PERR_SET1    7
+#define V_CMD_FIFO_PERR_SET1(x) ((x) << S_CMD_FIFO_PERR_SET1)
+#define F_CMD_FIFO_PERR_SET1    V_CMD_FIFO_PERR_SET1(1U)
+
+#define S_CMD_FIFO_PERR_SET0    6
+#define V_CMD_FIFO_PERR_SET0(x) ((x) << S_CMD_FIFO_PERR_SET0)
+#define F_CMD_FIFO_PERR_SET0    V_CMD_FIFO_PERR_SET0(1U)
+
+#define S_LSO_HDR_SRAM_PERR_SET1    5
+#define V_LSO_HDR_SRAM_PERR_SET1(x) ((x) << S_LSO_HDR_SRAM_PERR_SET1)
+#define F_LSO_HDR_SRAM_PERR_SET1    V_LSO_HDR_SRAM_PERR_SET1(1U)
+
+#define S_LSO_HDR_SRAM_PERR_SET0    4
+#define V_LSO_HDR_SRAM_PERR_SET0(x) ((x) << S_LSO_HDR_SRAM_PERR_SET0)
+#define F_LSO_HDR_SRAM_PERR_SET0    V_LSO_HDR_SRAM_PERR_SET0(1U)
+
+#define S_IMM_DATA_PERR_SET_CH1    3
+#define V_IMM_DATA_PERR_SET_CH1(x) ((x) << S_IMM_DATA_PERR_SET_CH1)
+#define F_IMM_DATA_PERR_SET_CH1    V_IMM_DATA_PERR_SET_CH1(1U)
+
+#define S_IMM_DATA_PERR_SET_CH0    2
+#define V_IMM_DATA_PERR_SET_CH0(x) ((x) << S_IMM_DATA_PERR_SET_CH0)
+#define F_IMM_DATA_PERR_SET_CH0    V_IMM_DATA_PERR_SET_CH0(1U)
+
 #define S_PBL_BOUND_ERR_CH1    1
 #define V_PBL_BOUND_ERR_CH1(x) ((x) << S_PBL_BOUND_ERR_CH1)
 #define F_PBL_BOUND_ERR_CH1    V_PBL_BOUND_ERR_CH1(1U)
@@ -5118,6 +5745,10 @@ $FreeBSD$
 
 #define A_MPS_CFG 0x600
 
+#define S_ENFORCEPKT    11
+#define V_ENFORCEPKT(x) ((x) << S_ENFORCEPKT)
+#define F_ENFORCEPKT    V_ENFORCEPKT(1U)
+
 #define S_SGETPQID    8
 #define M_SGETPQID    0x7
 #define V_SGETPQID(x) ((x) << S_SGETPQID)
@@ -5155,10 +5786,6 @@ $FreeBSD$
 #define V_PORT0ACTIVE(x) ((x) << S_PORT0ACTIVE)
 #define F_PORT0ACTIVE    V_PORT0ACTIVE(1U)
 
-#define S_ENFORCEPKT    11
-#define V_ENFORCEPKT(x) ((x) << S_ENFORCEPKT)
-#define F_ENFORCEPKT    V_ENFORCEPKT(1U)
-
 #define A_MPS_DRR_CFG1 0x604
 
 #define S_RLDWTTPD1    11
@@ -5280,6 +5907,10 @@ $FreeBSD$
 #define V_CPL_PKT_TID(x) ((x) << S_CPL_PKT_TID)
 #define G_CPL_PKT_TID(x) (((x) >> S_CPL_PKT_TID) & M_CPL_PKT_TID)
 
+#define S_CIM_TO_UP_FULL_SIZE    4
+#define V_CIM_TO_UP_FULL_SIZE(x) ((x) << S_CIM_TO_UP_FULL_SIZE)
+#define F_CIM_TO_UP_FULL_SIZE    V_CIM_TO_UP_FULL_SIZE(1U)
+
 #define S_CPU_NO_3F_CIM_ENABLE    3
 #define V_CPU_NO_3F_CIM_ENABLE(x) ((x) << S_CPU_NO_3F_CIM_ENABLE)
 #define F_CPU_NO_3F_CIM_ENABLE    V_CPU_NO_3F_CIM_ENABLE(1U)
@@ -5313,6 +5944,10 @@ $FreeBSD$
 
 #define A_CPL_INTR_ENABLE 0x650
 
+#define S_CIM_OP_MAP_PERR    5
+#define V_CIM_OP_MAP_PERR(x) ((x) << S_CIM_OP_MAP_PERR)
+#define F_CIM_OP_MAP_PERR    V_CIM_OP_MAP_PERR(1U)
+
 #define S_CIM_OVFL_ERROR    4
 #define V_CIM_OVFL_ERROR(x) ((x) << S_CIM_OVFL_ERROR)
 #define F_CIM_OVFL_ERROR    V_CIM_OVFL_ERROR(1U)
@@ -5704,6 +6339,10 @@ $FreeBSD$
 
 #define A_PL_INT_ENABLE0 0x6e0
 
+#define S_SW    25
+#define V_SW(x) ((x) << S_SW)
+#define F_SW    V_SW(1U)
+
 #define S_EXT    24
 #define V_EXT(x) ((x) << S_EXT)
 #define F_EXT    V_EXT(1U)
@@ -5792,18 +6431,14 @@ $FreeBSD$
 #define V_SGE3(x) ((x) << S_SGE3)
 #define F_SGE3    V_SGE3(1U)
 
-#define S_SW    25
-#define V_SW(x) ((x) << S_SW)
-#define F_SW    V_SW(1U)
-
 #define A_PL_INT_CAUSE0 0x6e4
 #define A_PL_INT_ENABLE1 0x6e8
 #define A_PL_INT_CAUSE1 0x6ec
 #define A_PL_RST 0x6f0
 
-#define S_CRSTWRM    1
-#define V_CRSTWRM(x) ((x) << S_CRSTWRM)
-#define F_CRSTWRM    V_CRSTWRM(1U)
+#define S_FATALPERREN    4
+#define V_FATALPERREN(x) ((x) << S_FATALPERREN)
+#define F_FATALPERREN    V_FATALPERREN(1U)
 
 #define S_SWINT1    3
 #define V_SWINT1(x) ((x) << S_SWINT1)
@@ -5813,6 +6448,10 @@ $FreeBSD$
 #define V_SWINT0(x) ((x) << S_SWINT0)
 #define F_SWINT0    V_SWINT0(1U)
 
+#define S_CRSTWRM    1
+#define V_CRSTWRM(x) ((x) << S_CRSTWRM)
+#define F_CRSTWRM    V_CRSTWRM(1U)
+
 #define A_PL_REV 0x6f4
 
 #define S_REV    0
@@ -5861,9 +6500,13 @@ $FreeBSD$
 #define V_READ(x) ((x) << S_READ)
 #define F_READ    V_READ(1U)
 
-#define S_CAL_IMP_UPD    23
-#define V_CAL_IMP_UPD(x) ((x) << S_CAL_IMP_UPD)
-#define F_CAL_IMP_UPD    V_CAL_IMP_UPD(1U)
+#define S_IMP_SET_UPDATE    24
+#define V_IMP_SET_UPDATE(x) ((x) << S_IMP_SET_UPDATE)
+#define F_IMP_SET_UPDATE    V_IMP_SET_UPDATE(1U)
+
+#define S_CAL_UPDATE    23
+#define V_CAL_UPDATE(x) ((x) << S_CAL_UPDATE)
+#define F_CAL_UPDATE    V_CAL_UPDATE(1U)
 
 #define S_CAL_BUSY    22
 #define V_CAL_BUSY(x) ((x) << S_CAL_BUSY)
@@ -5915,13 +6558,9 @@ $FreeBSD$
 #define V_SET_PD(x) ((x) << S_SET_PD)
 #define G_SET_PD(x) (((x) >> S_SET_PD) & M_SET_PD)
 
-#define S_IMP_SET_UPDATE    24
-#define V_IMP_SET_UPDATE(x) ((x) << S_IMP_SET_UPDATE)
-#define F_IMP_SET_UPDATE    V_IMP_SET_UPDATE(1U)
-
-#define S_CAL_UPDATE    23
-#define V_CAL_UPDATE(x) ((x) << S_CAL_UPDATE)
-#define F_CAL_UPDATE    V_CAL_UPDATE(1U)
+#define S_CAL_IMP_UPD    23
+#define V_CAL_IMP_UPD(x) ((x) << S_CAL_IMP_UPD)
+#define F_CAL_IMP_UPD    V_CAL_IMP_UPD(1U)
 
 #define A_MC5_DB_CONFIG 0x704
 
@@ -5961,6 +6600,14 @@ $FreeBSD$
 #define V_BUILD(x) ((x) << S_BUILD)
 #define F_BUILD    V_BUILD(1U)
 
+#define S_FILTEREN    11
+#define V_FILTEREN(x) ((x) << S_FILTEREN)
+#define F_FILTEREN    V_FILTEREN(1U)
+
+#define S_CLIPUPDATE    10
+#define V_CLIPUPDATE(x) ((x) << S_CLIPUPDATE)
+#define F_CLIPUPDATE    V_CLIPUPDATE(1U)
+
 #define S_TM_IO_PDOWN    9
 #define V_TM_IO_PDOWN(x) ((x) << S_TM_IO_PDOWN)
 #define F_TM_IO_PDOWN    V_TM_IO_PDOWN(1U)
@@ -5982,6 +6629,10 @@ $FreeBSD$
 #define V_DBGIEN(x) ((x) << S_DBGIEN)
 #define F_DBGIEN    V_DBGIEN(1U)
 
+#define S_TCMCFGOVR    3
+#define V_TCMCFGOVR(x) ((x) << S_TCMCFGOVR)
+#define F_TCMCFGOVR    V_TCMCFGOVR(1U)
+
 #define S_TMRDY    2
 #define V_TMRDY(x) ((x) << S_TMRDY)
 #define F_TMRDY    V_TMRDY(1U)
@@ -5994,18 +6645,6 @@ $FreeBSD$
 #define V_TMMODE(x) ((x) << S_TMMODE)
 #define F_TMMODE    V_TMMODE(1U)
 
-#define S_FILTEREN    11
-#define V_FILTEREN(x) ((x) << S_FILTEREN)
-#define F_FILTEREN    V_FILTEREN(1U)
-
-#define S_CLIPUPDATE    10
-#define V_CLIPUPDATE(x) ((x) << S_CLIPUPDATE)
-#define F_CLIPUPDATE    V_CLIPUPDATE(1U)
-
-#define S_TCMCFGOVR    3
-#define V_TCMCFGOVR(x) ((x) << S_TCMCFGOVR)
-#define F_TCMCFGOVR    V_TCMCFGOVR(1U)
-
 #define A_MC5_MISC 0x708
 
 #define S_LIP_CMP_UNAVAILABLE    0
@@ -6021,13 +6660,13 @@ $FreeBSD$
 #define G_RTINDX(x) (((x) >> S_RTINDX) & M_RTINDX)
 
 #define A_MC5_DB_FILTER_TABLE 0x710
-#define A_MC5_DB_SERVER_INDEX 0x714
 
 #define S_SRINDX    0
 #define M_SRINDX    0x3fffff
 #define V_SRINDX(x) ((x) << S_SRINDX)
 #define G_SRINDX(x) (((x) >> S_SRINDX) & M_SRINDX)
 
+#define A_MC5_DB_SERVER_INDEX 0x714
 #define A_MC5_DB_LIP_RAM_ADDR 0x718
 
 #define S_RAMWR    8
@@ -6115,6 +6754,7 @@ $FreeBSD$
 #define V_CLIPMAPADDR(x) ((x) << S_CLIPMAPADDR)
 #define G_CLIPMAPADDR(x) (((x) >> S_CLIPMAPADDR) & M_CLIPMAPADDR)
 
+#define A_MC5_DB_SIZE 0x73c
 #define A_MC5_DB_INT_ENABLE 0x740
 
 #define S_MSGSEL    28
@@ -6630,6 +7270,14 @@ $FreeBSD$
 
 #define A_XGM_RXFIFO_CFG 0x884
 
+#define S_RXFIFO_EMPTY    31
+#define V_RXFIFO_EMPTY(x) ((x) << S_RXFIFO_EMPTY)
+#define F_RXFIFO_EMPTY    V_RXFIFO_EMPTY(1U)
+
+#define S_RXFIFO_FULL    30
+#define V_RXFIFO_FULL(x) ((x) << S_RXFIFO_FULL)
+#define F_RXFIFO_FULL    V_RXFIFO_FULL(1U)
+
 #define S_RXFIFOPAUSEHWM    17
 #define M_RXFIFOPAUSEHWM    0xfff
 #define V_RXFIFOPAUSEHWM(x) ((x) << S_RXFIFOPAUSEHWM)
@@ -6662,6 +7310,22 @@ $FreeBSD$
 
 #define A_XGM_TXFIFO_CFG 0x888
 
+#define S_TXFIFO_EMPTY    31
+#define V_TXFIFO_EMPTY(x) ((x) << S_TXFIFO_EMPTY)
+#define F_TXFIFO_EMPTY    V_TXFIFO_EMPTY(1U)
+
+#define S_TXFIFO_FULL    30
+#define V_TXFIFO_FULL(x) ((x) << S_TXFIFO_FULL)
+#define F_TXFIFO_FULL    V_TXFIFO_FULL(1U)
+
+#define S_UNDERUNFIX    22
+#define V_UNDERUNFIX(x) ((x) << S_UNDERUNFIX)
+#define F_UNDERUNFIX    V_UNDERUNFIX(1U)
+
+#define S_ENDROPPKT    21
+#define V_ENDROPPKT(x) ((x) << S_ENDROPPKT)
+#define F_ENDROPPKT    V_ENDROPPKT(1U)
+
 #define S_TXIPG    13
 #define M_TXIPG    0xff
 #define V_TXIPG(x) ((x) << S_TXIPG)
@@ -6688,10 +7352,6 @@ $FreeBSD$
 #define V_DISPREAMBLE(x) ((x) << S_DISPREAMBLE)
 #define F_DISPREAMBLE    V_DISPREAMBLE(1U)
 
-#define S_ENDROPPKT    21
-#define V_ENDROPPKT(x) ((x) << S_ENDROPPKT)
-#define F_ENDROPPKT    V_ENDROPPKT(1U)
-
 #define A_XGM_SLOW_TIMER 0x88c
 
 #define S_PAUSESLOWTIMEREN    31
@@ -6703,6 +7363,13 @@ $FreeBSD$
 #define V_PAUSESLOWTIMER(x) ((x) << S_PAUSESLOWTIMER)
 #define G_PAUSESLOWTIMER(x) (((x) >> S_PAUSESLOWTIMER) & M_PAUSESLOWTIMER)
 
+#define A_XGM_PAUSE_TIMER 0x890
+
+#define S_PAUSETIMER    0
+#define M_PAUSETIMER    0xfffff
+#define V_PAUSETIMER(x) ((x) << S_PAUSETIMER)
+#define G_PAUSETIMER(x) (((x) >> S_PAUSETIMER) & M_PAUSETIMER)
+
 #define A_XGM_SERDES_CTRL 0x890
 
 #define S_SERDESEN    25
@@ -6761,13 +7428,6 @@ $FreeBSD$
 #define V_TXENABLE(x) ((x) << S_TXENABLE)
 #define F_TXENABLE    V_TXENABLE(1U)
 
-#define A_XGM_PAUSE_TIMER 0x890
-
-#define S_PAUSETIMER    0
-#define M_PAUSETIMER    0xfffff
-#define V_PAUSETIMER(x) ((x) << S_PAUSETIMER)
-#define G_PAUSETIMER(x) (((x) >> S_PAUSETIMER) & M_PAUSETIMER)
-
 #define A_XGM_XAUI_PCS_TEST 0x894
 
 #define S_TESTPATTERN    1
@@ -6792,6 +7452,14 @@ $FreeBSD$
 
 #define A_XGM_RGMII_IMP 0x89c
 
+#define S_CALRESET    8
+#define V_CALRESET(x) ((x) << S_CALRESET)
+#define F_CALRESET    V_CALRESET(1U)
+
+#define S_CALUPDATE    7
+#define V_CALUPDATE(x) ((x) << S_CALUPDATE)
+#define F_CALUPDATE    V_CALUPDATE(1U)
+
 #define S_XGM_IMPSETUPDATE    6
 #define V_XGM_IMPSETUPDATE(x) ((x) << S_XGM_IMPSETUPDATE)
 #define F_XGM_IMPSETUPDATE    V_XGM_IMPSETUPDATE(1U)
@@ -6806,14 +7474,6 @@ $FreeBSD$
 #define V_RGMIIIMPPU(x) ((x) << S_RGMIIIMPPU)
 #define G_RGMIIIMPPU(x) (((x) >> S_RGMIIIMPPU) & M_RGMIIIMPPU)
 
-#define S_CALRESET    8
-#define V_CALRESET(x) ((x) << S_CALRESET)
-#define F_CALRESET    V_CALRESET(1U)
-
-#define S_CALUPDATE    7
-#define V_CALUPDATE(x) ((x) << S_CALUPDATE)
-#define F_CALUPDATE    V_CALUPDATE(1U)
-
 #define A_XGM_XAUI_IMP 0x8a0
 
 #define S_XGM_CALFAULT    29
@@ -6844,6 +7504,23 @@ $FreeBSD$
 
 #define A_XGM_RX_MAX_PKT_SIZE 0x8a8
 
+#define S_RXMAXFRAMERSIZE    17
+#define M_RXMAXFRAMERSIZE    0x3fff
+#define V_RXMAXFRAMERSIZE(x) ((x) << S_RXMAXFRAMERSIZE)
+#define G_RXMAXFRAMERSIZE(x) (((x) >> S_RXMAXFRAMERSIZE) & M_RXMAXFRAMERSIZE)
+
+#define S_RXENERRORGATHER    16
+#define V_RXENERRORGATHER(x) ((x) << S_RXENERRORGATHER)
+#define F_RXENERRORGATHER    V_RXENERRORGATHER(1U)
+
+#define S_RXENSINGLEFLIT    15
+#define V_RXENSINGLEFLIT(x) ((x) << S_RXENSINGLEFLIT)
+#define F_RXENSINGLEFLIT    V_RXENSINGLEFLIT(1U)
+
+#define S_RXENFRAMER    14
+#define V_RXENFRAMER(x) ((x) << S_RXENFRAMER)
+#define F_RXENFRAMER    V_RXENFRAMER(1U)
+
 #define S_RXMAXPKTSIZE    0
 #define M_RXMAXPKTSIZE    0x3fff
 #define V_RXMAXPKTSIZE(x) ((x) << S_RXMAXPKTSIZE)
@@ -6851,6 +7528,10 @@ $FreeBSD$
 
 #define A_XGM_RESET_CTRL 0x8ac
 
+#define S_XGMAC_STOP_EN    4
+#define V_XGMAC_STOP_EN(x) ((x) << S_XGMAC_STOP_EN)
+#define F_XGMAC_STOP_EN    V_XGMAC_STOP_EN(1U)
+
 #define S_XG2G_RESET_    3
 #define V_XG2G_RESET_(x) ((x) << S_XG2G_RESET_)
 #define F_XG2G_RESET_    V_XG2G_RESET_(1U)
@@ -6930,9 +7611,9 @@ $FreeBSD$
 
 #define A_XGM_INT_ENABLE 0x8d4
 
-#define S_SERDESCMULOCK_LOSS    24
-#define V_SERDESCMULOCK_LOSS(x) ((x) << S_SERDESCMULOCK_LOSS)
-#define F_SERDESCMULOCK_LOSS    V_SERDESCMULOCK_LOSS(1U)
+#define S_XAUIPCSDECERR    24
+#define V_XAUIPCSDECERR(x) ((x) << S_XAUIPCSDECERR)
+#define F_XAUIPCSDECERR    V_XAUIPCSDECERR(1U)
 
 #define S_RGMIIRXFIFOOVERFLOW    23
 #define V_RGMIIRXFIFOOVERFLOW(x) ((x) << S_RGMIIRXFIFOOVERFLOW)
@@ -6968,15 +7649,15 @@ $FreeBSD$
 #define V_RXFIFO_OVERFLOW(x) ((x) << S_RXFIFO_OVERFLOW)
 #define F_RXFIFO_OVERFLOW    V_RXFIFO_OVERFLOW(1U)
 
-#define S_SERDESBIST_ERR    8
-#define M_SERDESBIST_ERR    0xf
-#define V_SERDESBIST_ERR(x) ((x) << S_SERDESBIST_ERR)
-#define G_SERDESBIST_ERR(x) (((x) >> S_SERDESBIST_ERR) & M_SERDESBIST_ERR)
+#define S_SERDESBISTERR    8
+#define M_SERDESBISTERR    0xf
+#define V_SERDESBISTERR(x) ((x) << S_SERDESBISTERR)
+#define G_SERDESBISTERR(x) (((x) >> S_SERDESBISTERR) & M_SERDESBISTERR)
 
-#define S_SERDES_LOS    4
-#define M_SERDES_LOS    0xf
-#define V_SERDES_LOS(x) ((x) << S_SERDES_LOS)
-#define G_SERDES_LOS(x) (((x) >> S_SERDES_LOS) & M_SERDES_LOS)
+#define S_SERDESLOWSIGCHANGE    4
+#define M_SERDESLOWSIGCHANGE    0xf
+#define V_SERDESLOWSIGCHANGE(x) ((x) << S_SERDESLOWSIGCHANGE)
+#define G_SERDESLOWSIGCHANGE(x) (((x) >> S_SERDESLOWSIGCHANGE) & M_SERDESLOWSIGCHANGE)
 
 #define S_XAUIPCSCTCERR    3
 #define V_XAUIPCSCTCERR(x) ((x) << S_XAUIPCSCTCERR)
@@ -6994,15 +7675,19 @@ $FreeBSD$
 #define V_XGM_INT(x) ((x) << S_XGM_INT)
 #define F_XGM_INT    V_XGM_INT(1U)
 
-#define S_SERDESBISTERR    8
-#define M_SERDESBISTERR    0xf
-#define V_SERDESBISTERR(x) ((x) << S_SERDESBISTERR)
-#define G_SERDESBISTERR(x) (((x) >> S_SERDESBISTERR) & M_SERDESBISTERR)
+#define S_SERDESCMULOCK_LOSS    24
+#define V_SERDESCMULOCK_LOSS(x) ((x) << S_SERDESCMULOCK_LOSS)
+#define F_SERDESCMULOCK_LOSS    V_SERDESCMULOCK_LOSS(1U)
 
-#define S_SERDESLOWSIGCHANGE    4
-#define M_SERDESLOWSIGCHANGE    0xf
-#define V_SERDESLOWSIGCHANGE(x) ((x) << S_SERDESLOWSIGCHANGE)
-#define G_SERDESLOWSIGCHANGE(x) (((x) >> S_SERDESLOWSIGCHANGE) & M_SERDESLOWSIGCHANGE)
+#define S_SERDESBIST_ERR    8
+#define M_SERDESBIST_ERR    0xf
+#define V_SERDESBIST_ERR(x) ((x) << S_SERDESBIST_ERR)
+#define G_SERDESBIST_ERR(x) (((x) >> S_SERDESBIST_ERR) & M_SERDESBIST_ERR)
+
+#define S_SERDES_LOS    4
+#define M_SERDES_LOS    0xf
+#define V_SERDES_LOS(x) ((x) << S_SERDES_LOS)
+#define G_SERDES_LOS(x) (((x) >> S_SERDES_LOS) & M_SERDES_LOS)
 
 #define A_XGM_INT_CAUSE 0x8d8
 #define A_XGM_XAUI_ACT_CTRL 0x8dc
@@ -7298,6 +7983,14 @@ $FreeBSD$
 #define V_EXTBISTCHKFMD0(x) ((x) << S_EXTBISTCHKFMD0)
 #define F_EXTBISTCHKFMD0    V_EXTBISTCHKFMD0(1U)
 
+#define S_LOWSIGFORCEEN0    2
+#define V_LOWSIGFORCEEN0(x) ((x) << S_LOWSIGFORCEEN0)
+#define F_LOWSIGFORCEEN0    V_LOWSIGFORCEEN0(1U)
+
+#define S_LOWSIGFORCEVALUE0    1
+#define V_LOWSIGFORCEVALUE0(x) ((x) << S_LOWSIGFORCEVALUE0)
+#define F_LOWSIGFORCEVALUE0    V_LOWSIGFORCEVALUE0(1U)
+
 #define S_LOWSIG0    0
 #define V_LOWSIG0(x) ((x) << S_LOWSIG0)
 #define F_LOWSIG0    V_LOWSIG0(1U)
@@ -7313,6 +8006,14 @@ $FreeBSD$
 #define V_EXTBISTCHKFMD1(x) ((x) << S_EXTBISTCHKFMD1)
 #define F_EXTBISTCHKFMD1    V_EXTBISTCHKFMD1(1U)
 
+#define S_LOWSIGFORCEEN1    2
+#define V_LOWSIGFORCEEN1(x) ((x) << S_LOWSIGFORCEEN1)
+#define F_LOWSIGFORCEEN1    V_LOWSIGFORCEEN1(1U)
+
+#define S_LOWSIGFORCEVALUE1    1
+#define V_LOWSIGFORCEVALUE1(x) ((x) << S_LOWSIGFORCEVALUE1)
+#define F_LOWSIGFORCEVALUE1    V_LOWSIGFORCEVALUE1(1U)
+
 #define S_LOWSIG1    0
 #define V_LOWSIG1(x) ((x) << S_LOWSIG1)
 #define F_LOWSIG1    V_LOWSIG1(1U)
@@ -7328,6 +8029,14 @@ $FreeBSD$
 #define V_EXTBISTCHKFMD2(x) ((x) << S_EXTBISTCHKFMD2)
 #define F_EXTBISTCHKFMD2    V_EXTBISTCHKFMD2(1U)
 
+#define S_LOWSIGFORCEEN2    2
+#define V_LOWSIGFORCEEN2(x) ((x) << S_LOWSIGFORCEEN2)
+#define F_LOWSIGFORCEEN2    V_LOWSIGFORCEEN2(1U)
+
+#define S_LOWSIGFORCEVALUE2    1
+#define V_LOWSIGFORCEVALUE2(x) ((x) << S_LOWSIGFORCEVALUE2)
+#define F_LOWSIGFORCEVALUE2    V_LOWSIGFORCEVALUE2(1U)
+
 #define S_LOWSIG2    0
 #define V_LOWSIG2(x) ((x) << S_LOWSIG2)
 #define F_LOWSIG2    V_LOWSIG2(1U)
@@ -7343,6 +8052,14 @@ $FreeBSD$
 #define V_EXTBISTCHKFMD3(x) ((x) << S_EXTBISTCHKFMD3)
 #define F_EXTBISTCHKFMD3    V_EXTBISTCHKFMD3(1U)
 
+#define S_LOWSIGFORCEEN3    2
+#define V_LOWSIGFORCEEN3(x) ((x) << S_LOWSIGFORCEEN3)
+#define F_LOWSIGFORCEEN3    V_LOWSIGFORCEEN3(1U)
+
+#define S_LOWSIGFORCEVALUE3    1
+#define V_LOWSIGFORCEVALUE3(x) ((x) << S_LOWSIGFORCEVALUE3)
+#define F_LOWSIGFORCEVALUE3    V_LOWSIGFORCEVALUE3(1U)
+
 #define S_LOWSIG3    0
 #define V_LOWSIG3(x) ((x) << S_LOWSIG3)
 #define F_LOWSIG3    V_LOWSIG3(1U)
diff --git a/sys/dev/cxgb/common/cxgb_t3_cpl.h b/sys/dev/cxgb/common/cxgb_t3_cpl.h
index 1f0eb3f..dd24571 100644
--- a/sys/dev/cxgb/common/cxgb_t3_cpl.h
+++ b/sys/dev/cxgb/common/cxgb_t3_cpl.h
@@ -173,8 +173,9 @@ enum {                     /* TCP congestion control algorithms */
 
 enum {			   /* RSS hash type */
 	RSS_HASH_NONE = 0,
-	RSS_HASH_2_TUPLE = 1 << 0,
-	RSS_HASH_4_TUPLE = 1 << 1
+	RSS_HASH_2_TUPLE = 1,
+	RSS_HASH_4_TUPLE = 2,
+	RSS_HASH_TCPV6 = 3
 };
 
 union opcode_tid {
@@ -1097,6 +1098,11 @@ struct cpl_rx_data_ddp {
 #define V_DDP_OFFSET(x) ((x) << S_DDP_OFFSET)
 #define G_DDP_OFFSET(x) (((x) >> S_DDP_OFFSET) & M_DDP_OFFSET)
 
+#define S_DDP_DACK_MODE    22
+#define M_DDP_DACK_MODE    0x3
+#define V_DDP_DACK_MODE(x) ((x) << S_DDP_DACK_MODE)
+#define G_DDP_DACK_MODE(x) (((x) >> S_DDP_DACK_MODE) & M_DDP_DACK_MODE)
+
 #define S_DDP_URG    24
 #define V_DDP_URG(x) ((x) << S_DDP_URG)
 #define F_DDP_URG    V_DDP_URG(1U)
diff --git a/sys/dev/cxgb/common/cxgb_t3_hw.c b/sys/dev/cxgb/common/cxgb_t3_hw.c
index 6c8b53a..d082c74 100644
--- a/sys/dev/cxgb/common/cxgb_t3_hw.c
+++ b/sys/dev/cxgb/common/cxgb_t3_hw.c
@@ -404,6 +404,29 @@ int t3_phy_advertise(struct cphy *phy, unsigned int advert)
 }
 
 /**
+ *	t3_phy_advertise_fiber - set fiber PHY advertisement register
+ *	@phy: the PHY to operate on
+ *	@advert: bitmap of capabilities the PHY should advertise
+ *
+ *	Sets a fiber PHY's advertisement register to advertise the
+ *	requested capabilities.
+ */
+int t3_phy_advertise_fiber(struct cphy *phy, unsigned int advert)
+{
+	unsigned int val = 0;
+
+	if (advert & ADVERTISED_1000baseT_Half)
+		val |= ADVERTISE_1000XHALF;
+	if (advert & ADVERTISED_1000baseT_Full)
+		val |= ADVERTISE_1000XFULL;
+	if (advert & ADVERTISED_Pause)
+		val |= ADVERTISE_1000XPAUSE;
+	if (advert & ADVERTISED_Asym_Pause)
+		val |= ADVERTISE_1000XPSE_ASYM;
+	return mdio_write(phy, 0, MII_ADVERTISE, val);
+}
+
+/**
  *	t3_set_phy_speed_duplex - force PHY speed and duplex
  *	@phy: the PHY to operate on
  *	@speed: requested PHY speed
@@ -451,8 +474,8 @@ static struct adapter_info t3_adap_info[] = {
 	  &mi1_mdio_ops, "Chelsio T302" },
 	{ 1, 0, 0, 0, 0,
 	  F_GPIO1_OEN | F_GPIO6_OEN | F_GPIO7_OEN | F_GPIO10_OEN |
-	  F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL, 0,
-	  SUPPORTED_10000baseT_Full | SUPPORTED_AUI,
+	  F_GPIO11_OEN | F_GPIO1_OUT_VAL | F_GPIO6_OUT_VAL | F_GPIO10_OUT_VAL,
+	  0, SUPPORTED_10000baseT_Full | SUPPORTED_AUI,
 	  &mi1_mdio_ext_ops, "Chelsio T310" },
 	{ 1, 1, 0, 0, 0,
 	  F_GPIO1_OEN | F_GPIO2_OEN | F_GPIO4_OEN | F_GPIO5_OEN | F_GPIO6_OEN |
@@ -476,31 +499,20 @@ const struct adapter_info *t3_get_adapter_info(unsigned int id)
 	return id < ARRAY_SIZE(t3_adap_info) ? &t3_adap_info[id] : NULL;
 }
 
-#define CAPS_1G (SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full | \
-		 SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII)
-#define CAPS_10G (SUPPORTED_10000baseT_Full | SUPPORTED_AUI)
-
 static struct port_type_info port_types[] = {
 	{ NULL },
-	{ t3_ael1002_phy_prep, CAPS_10G | SUPPORTED_FIBRE,
-	  "10GBASE-XR" },
-	{ t3_vsc8211_phy_prep, CAPS_1G | SUPPORTED_TP | SUPPORTED_IRQ,
-	  "10/100/1000BASE-T" },
-	{ t3_mv88e1xxx_phy_prep, CAPS_1G | SUPPORTED_TP | SUPPORTED_IRQ,
-	  "10/100/1000BASE-T" },
-	{ t3_xaui_direct_phy_prep, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" },
-	{ NULL, CAPS_10G, "10GBASE-KX4" },
-	{ t3_qt2045_phy_prep, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" },
-	{ t3_ael1006_phy_prep, CAPS_10G | SUPPORTED_FIBRE,
-	  "10GBASE-SR" },
-	{ NULL, CAPS_10G | SUPPORTED_TP, "10GBASE-CX4" },
+	{ t3_ael1002_phy_prep },
+	{ t3_vsc8211_phy_prep },
+	{ t3_mv88e1xxx_phy_prep },
+	{ t3_xaui_direct_phy_prep },
+	{ NULL },
+	{ t3_qt2045_phy_prep },
+	{ t3_ael1006_phy_prep },
+	{ NULL },
 };
 
-#undef CAPS_1G
-#undef CAPS_10G
-
 #define VPD_ENTRY(name, len) \
-	u8 name##_kword[2]; u8 name##_len; char name##_data[len]
+	u8 name##_kword[2]; u8 name##_len; u8 name##_data[len]
 
 /*
  * Partial EEPROM Vital Product Data structure.  Includes only the ID and
@@ -678,6 +690,15 @@ static int get_vpd_params(adapter_t *adapter, struct vpd_params *p)
 	return 0;
 }
 
+/* BIOS boot header */
+typedef struct boot_header_s {
+	u8	signature[2];	/* signature */
+	u8	length;		/* image length (include header) */
+	u8	offset[4];	/* initialization vector */
+	u8	reserved[19];	/* reserved */
+	u8	exheader[2];	/* offset to expansion header */
+} boot_header_t;
+
 /* serial flash and firmware constants */
 enum {
 	SF_ATTEMPTS = 5,           /* max retries for SF1 operations */
@@ -694,7 +715,14 @@ enum {
 
 	FW_FLASH_BOOT_ADDR = 0x70000, /* start address of FW in flash */
 	FW_VERS_ADDR = 0x77ffc,    /* flash address holding FW version */
-	FW_MIN_SIZE = 8            /* at least version and csum */
+	FW_MIN_SIZE = 8,           /* at least version and csum */
+	FW_MAX_SIZE = FW_VERS_ADDR - FW_FLASH_BOOT_ADDR,
+
+	BOOT_FLASH_BOOT_ADDR = 0x0,/* start address of boot image in flash */
+	BOOT_SIGNATURE = 0xaa55,   /* signature of BIOS boot ROM */
+	BOOT_SIZE_INC = 512,       /* image size measured in 512B chunks */
+	BOOT_MIN_SIZE = sizeof(boot_header_t), /* at least basic header */
+	BOOT_MAX_SIZE = 0xff*BOOT_SIZE_INC /* 1 byte * length increment  */
 };
 
 /**
@@ -817,16 +845,21 @@ int t3_read_flash(adapter_t *adapter, unsigned int addr, unsigned int nwords,
  *	@addr: the start address to write
  *	@n: length of data to write
  *	@data: the data to write
+ *	@byte_oriented: whether to store data as bytes or as words
  *
  *	Writes up to a page of data (256 bytes) to the serial flash starting
  *	at the given address.
+ *	If @byte_oriented is set the write data is stored as a 32-bit
+ *	big-endian array, otherwise in the processor's native endianess.
+ *	
  */
 static int t3_write_flash(adapter_t *adapter, unsigned int addr,
-			  unsigned int n, const u8 *data)
+			  unsigned int n, const u8 *data,
+			  int byte_oriented)
 {
 	int ret;
 	u32 buf[64];
-	unsigned int i, c, left, val, offset = addr & 0xff;
+	unsigned int c, left, val, offset = addr & 0xff;
 
 	if (addr + n > SF_SIZE || offset + n > 256)
 		return -EINVAL;
@@ -839,8 +872,10 @@ static int t3_write_flash(adapter_t *adapter, unsigned int addr,
 
 	for (left = n; left; left -= c) {
 		c = min(left, 4U);
-		for (val = 0, i = 0; i < c; ++i)
-			val = (val << 8) + *data++;
+		val = *(const u32*)data;
+		data += c;
+		if (byte_oriented)
+			val = htonl(val);
 
 		ret = sf1_write(adapter, c, c != left, val);
 		if (ret)
@@ -850,7 +885,8 @@ static int t3_write_flash(adapter_t *adapter, unsigned int addr,
 		return ret;
 
 	/* Read the page to verify the write succeeded */
-	ret = t3_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf, 1);
+	ret = t3_read_flash(adapter, addr & ~0xff, ARRAY_SIZE(buf), buf,
+			    byte_oriented);
 	if (ret)
 		return ret;
 
@@ -887,16 +923,18 @@ int t3_get_tp_version(adapter_t *adapter, u32 *vers)
  *	@adapter: the adapter
  *
  */
-int t3_check_tpsram_version(adapter_t *adapter)
+int t3_check_tpsram_version(adapter_t *adapter, int *must_load)
 {
 	int ret;
 	u32 vers;
 	unsigned int major, minor;
 
-	/* Get version loaded in SRAM */
-	t3_write_reg(adapter, A_TP_EMBED_OP_FIELD0, 0);
-	ret = t3_wait_op_done(adapter, A_TP_EMBED_OP_FIELD0,
-			      1, 1, 5, 1);
+	if (adapter->params.rev == T3_REV_A)
+		return 0;
+
+	*must_load = 1;
+
+	ret = t3_get_tp_version(adapter, &vers);
 	if (ret)
 		return ret;
 	
@@ -908,9 +946,16 @@ int t3_check_tpsram_version(adapter_t *adapter)
 	if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR) 
 		return 0;
 
-	CH_WARN(adapter, "found wrong TP version (%u.%u), "
-	       "driver needs version %d.%d\n", major, minor,
-	       TP_VERSION_MAJOR, TP_VERSION_MINOR);
+	if (major != TP_VERSION_MAJOR)
+		CH_ERR(adapter, "found wrong TP version (%u.%u), "
+		       "driver needs version %d.%d\n", major, minor,
+		       TP_VERSION_MAJOR, TP_VERSION_MINOR);
+	else {
+		*must_load = 0;
+		CH_ERR(adapter, "found wrong TP version (%u.%u), "
+		       "driver compiled for version %d.%d\n", major, minor,
+		       TP_VERSION_MAJOR, TP_VERSION_MINOR);
+	}
 	return -EINVAL;
 }
 
@@ -966,12 +1011,13 @@ int t3_get_fw_version(adapter_t *adapter, u32 *vers)
  *	Checks if an adapter's FW is compatible with the driver.  Returns 0
  *	if the versions are compatible, a negative error otherwise.
  */
-int t3_check_fw_version(adapter_t *adapter)
+int t3_check_fw_version(adapter_t *adapter, int *must_load)
 {
 	int ret;
 	u32 vers;
 	unsigned int type, major, minor;
 
+	*must_load = 1;
 	ret = t3_get_fw_version(adapter, &vers);
 	if (ret)
 		return ret;
@@ -984,9 +1030,21 @@ int t3_check_fw_version(adapter_t *adapter)
 	    minor == FW_VERSION_MINOR)
 		return 0;
 
-	CH_WARN(adapter, "found wrong FW version (%u.%u), "
-	       "driver needs version %d.%d\n", major, minor,
-	       FW_VERSION_MAJOR, FW_VERSION_MINOR);
+	if (major != FW_VERSION_MAJOR)
+		CH_ERR(adapter, "found wrong FW version(%u.%u), "
+		       "driver needs version %u.%u\n", major, minor,
+		       FW_VERSION_MAJOR, FW_VERSION_MINOR);
+	else if ((int)minor < FW_VERSION_MINOR) {
+		*must_load = 0;
+		CH_WARN(adapter, "found old FW minor version(%u.%u), "
+		        "driver compiled for version %u.%u\n", major, minor,
+			FW_VERSION_MAJOR, FW_VERSION_MINOR);
+	} else {
+		CH_WARN(adapter, "found newer FW version(%u.%u), "
+		        "driver compiled for version %u.%u\n", major, minor,
+			FW_VERSION_MAJOR, FW_VERSION_MINOR);
+			return 0;
+	}
 	return -EINVAL;
 }
 
@@ -1033,7 +1091,7 @@ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size)
 
 	if ((size & 3) || size < FW_MIN_SIZE)
 		return -EINVAL;
-	if (size > FW_VERS_ADDR + 8 - FW_FLASH_BOOT_ADDR)
+	if (size - 8 > FW_MAX_SIZE)
 		return -EFBIG;
 
 	for (csum = 0, i = 0; i < size / sizeof(csum); i++)
@@ -1052,7 +1110,7 @@ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size)
 	for (addr = FW_FLASH_BOOT_ADDR; size; ) {
 		unsigned int chunk_size = min(size, 256U);
 
-		ret = t3_write_flash(adapter, addr, chunk_size, fw_data);
+		ret = t3_write_flash(adapter, addr, chunk_size, fw_data, 1);
 		if (ret)
 			goto out;
 
@@ -1061,13 +1119,71 @@ int t3_load_fw(adapter_t *adapter, const u8 *fw_data, unsigned int size)
 		size -= chunk_size;
 	}
 
-	ret = t3_write_flash(adapter, FW_VERS_ADDR, 4, fw_data);
+	ret = t3_write_flash(adapter, FW_VERS_ADDR, 4, fw_data, 1);
 out:
 	if (ret)
 		CH_ERR(adapter, "firmware download failed, error %d\n", ret);
 	return ret;
 }
 
+/*
+ *	t3_load_boot - download boot flash
+ *	@adapter: the adapter
+ *	@boot_data: the boot image to write
+ *	@size: image size
+ *
+ *	Write the supplied boot image to the card's serial flash.
+ *	The boot image has the following sections: a 28-byte header and the
+ *	boot image.
+ */
+int t3_load_boot(adapter_t *adapter, u8 *boot_data, unsigned int size)
+{
+	boot_header_t *header = (boot_header_t *)boot_data;
+	int ret;
+	unsigned int addr;
+	unsigned int boot_sector = BOOT_FLASH_BOOT_ADDR >> 16;
+	unsigned int boot_end = (BOOT_FLASH_BOOT_ADDR + size - 1) >> 16;
+
+	/*
+	 * Perform some primitive sanity testing to avoid accidentally
+	 * writing garbage over the boot sectors.  We ought to check for
+	 * more but it's not worth it for now ...
+	 */
+	if (size < BOOT_MIN_SIZE || size > BOOT_MAX_SIZE) {
+		CH_ERR(adapter, "boot image too small/large\n");
+		return -EFBIG;
+	}
+	if (le16_to_cpu(*(u16*)header->signature) != BOOT_SIGNATURE) {
+		CH_ERR(adapter, "boot image missing signature\n");
+		return -EINVAL;
+	}
+	if (header->length * BOOT_SIZE_INC != size) {
+		CH_ERR(adapter, "boot image header length != image length\n");
+		return -EINVAL;
+	}
+
+	ret = t3_flash_erase_sectors(adapter, boot_sector, boot_end);
+	if (ret)
+		goto out;
+
+	for (addr = BOOT_FLASH_BOOT_ADDR; size; ) {
+		unsigned int chunk_size = min(size, 256U);
+
+		ret = t3_write_flash(adapter, addr, chunk_size, boot_data, 0);
+		if (ret)
+			goto out;
+
+		addr += chunk_size;
+		boot_data += chunk_size;
+		size -= chunk_size;
+	}
+
+out:
+	if (ret)
+		CH_ERR(adapter, "boot image download failed, error %d\n", ret);
+	return ret;
+}
+
 #define CIM_CTL_BASE 0x2000
 
 /**
@@ -1175,7 +1291,6 @@ int t3_link_start(struct cphy *phy, struct cmac *mac, struct link_config *lc)
 						   fc);
 			/* Also disables autoneg */
 			phy->ops->set_speed_duplex(phy, lc->speed, lc->duplex);
-			phy->ops->reset(phy, 0);
 		} else
 			phy->ops->autoneg_enable(phy);
 	} else {
@@ -1248,7 +1363,13 @@ static int t3_handle_intr_status(adapter_t *adapter, unsigned int reg,
 	return fatal;
 }
 
-#define SGE_INTR_MASK (F_RSPQDISABLED)
+#define SGE_INTR_MASK (F_RSPQDISABLED | \
+		       F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR | \
+		       F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
+		       F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
+		       V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
+		       F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
+		       F_HIRCQPARITYERROR)
 #define MC5_INTR_MASK (F_PARITYERR | F_ACTRGNFULL | F_UNKNOWNCMD | \
 		       F_REQQPARERR | F_DISPQPARERR | F_DELACTEMPTY | \
 		       F_NFASRCHFAIL)
@@ -1265,16 +1386,23 @@ static int t3_handle_intr_status(adapter_t *adapter, unsigned int reg,
 #define PCIE_INTR_MASK (F_UNXSPLCPLERRR | F_UNXSPLCPLERRC | F_PCIE_PIOPARERR |\
 			F_PCIE_WFPARERR | F_PCIE_RFPARERR | F_PCIE_CFPARERR | \
 			/* V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR) | */ \
-			V_BISTERR(M_BISTERR) | F_PEXERR)
-#define ULPRX_INTR_MASK F_PARERR
-#define ULPTX_INTR_MASK 0
-#define CPLSW_INTR_MASK (F_TP_FRAMING_ERROR | \
+			F_RETRYBUFPARERR | F_RETRYLUTPARERR | F_RXPARERR | \
+			F_TXPARERR | V_BISTERR(M_BISTERR))
+#define ULPRX_INTR_MASK (F_PARERRDATA | F_PARERRPCMD | F_ARBPF1PERR | \
+			 F_ARBPF0PERR | F_ARBFPERR | F_PCMDMUXPERR | \
+			 F_DATASELFRAMEERR1 | F_DATASELFRAMEERR0)
+#define ULPTX_INTR_MASK 0xfc
+#define CPLSW_INTR_MASK (F_CIM_OP_MAP_PERR | F_TP_FRAMING_ERROR | \
 			 F_SGE_FRAMING_ERROR | F_CIM_FRAMING_ERROR | \
 			 F_ZERO_SWITCH_ERROR)
 #define CIM_INTR_MASK (F_BLKWRPLINT | F_BLKRDPLINT | F_BLKWRCTLINT | \
 		       F_BLKRDCTLINT | F_BLKWRFLASHINT | F_BLKRDFLASHINT | \
 		       F_SGLWRFLASHINT | F_WRBLKFLASHINT | F_BLKWRBOOTINT | \
-	 	       F_FLASHRANGEINT | F_SDRAMRANGEINT | F_RSVDSPACEINT)
+	 	       F_FLASHRANGEINT | F_SDRAMRANGEINT | F_RSVDSPACEINT | \
+		       F_DRAMPARERR | F_ICACHEPARERR | F_DCACHEPARERR | \
+		       F_OBQSGEPARERR | F_OBQULPHIPARERR | F_OBQULPLOPARERR | \
+		       F_IBQSGELOPARERR | F_IBQSGEHIPARERR | F_IBQULPPARERR | \
+		       F_IBQTPPARERR | F_ITAGPARERR | F_DTAGPARERR)
 #define PMTX_INTR_MASK (F_ZERO_C_CMD_ERROR | ICSPI_FRM_ERR | OESPI_FRM_ERR | \
 			V_ICSPI_PAR_ERROR(M_ICSPI_PAR_ERROR) | \
 			V_OESPI_PAR_ERROR(M_OESPI_PAR_ERROR))
@@ -1343,6 +1471,10 @@ static void pcie_intr_handler(adapter_t *adapter)
 		{ F_PCIE_CFPARERR, "PCI command FIFO parity error", -1, 1 },
 		{ V_PCIE_MSIXPARERR(M_PCIE_MSIXPARERR),
 		  "PCI MSI-X table/PBA parity error", -1, 1 },
+		{ F_RETRYBUFPARERR, "PCI retry buffer parity error", -1, 1 },
+		{ F_RETRYLUTPARERR, "PCI retry LUT parity error", -1, 1 },
+		{ F_RXPARERR, "PCI Rx parity error", -1, 1 },
+		{ F_TXPARERR, "PCI Tx parity error", -1, 1 },
 		{ V_BISTERR(M_BISTERR), "PCI BIST error", -1, 1 },
 		{ 0 }
 	};
@@ -1367,9 +1499,16 @@ static void tp_intr_handler(adapter_t *adapter)
 		{ 0x2000000, "TP out of Tx pages", -1, 1 },
 		{ 0 }
 	};
+	static struct intr_info tp_intr_info_t3c[] = {
+		{ 0x1fffffff,  "TP parity error", -1, 1 },
+		{ F_FLMRXFLSTEMPTY, "TP out of Rx pages", -1, 1 },
+		{ F_FLMTXFLSTEMPTY, "TP out of Tx pages", -1, 1 },
+		{ 0 }
+	};
 
 	if (t3_handle_intr_status(adapter, A_TP_INT_CAUSE, 0xffffffff,
-				  tp_intr_info, NULL))
+				  adapter->params.rev < T3_REV_C ?
+					tp_intr_info : tp_intr_info_t3c, NULL))
 		t3_fatal_err(adapter);
 }
 
@@ -1391,10 +1530,22 @@ static void cim_intr_handler(adapter_t *adapter)
 		{ F_BLKWRCTLINT, "CIM block write to CTL space", -1, 1 },
 		{ F_BLKRDPLINT, "CIM block read from PL space", -1, 1 },
 		{ F_BLKWRPLINT, "CIM block write to PL space", -1, 1 },
+		{ F_DRAMPARERR, "CIM DRAM parity error", -1, 1 },
+		{ F_ICACHEPARERR, "CIM icache parity error", -1, 1 },
+		{ F_DCACHEPARERR, "CIM dcache parity error", -1, 1 },
+		{ F_OBQSGEPARERR, "CIM OBQ SGE parity error", -1, 1 },
+		{ F_OBQULPHIPARERR, "CIM OBQ ULPHI parity error", -1, 1 },
+		{ F_OBQULPLOPARERR, "CIM OBQ ULPLO parity error", -1, 1 },
+		{ F_IBQSGELOPARERR, "CIM IBQ SGELO parity error", -1, 1 },
+		{ F_IBQSGEHIPARERR, "CIM IBQ SGEHI parity error", -1, 1 },
+		{ F_IBQULPPARERR, "CIM IBQ ULP parity error", -1, 1 },
+		{ F_IBQTPPARERR, "CIM IBQ TP parity error", -1, 1 },
+		{ F_ITAGPARERR, "CIM itag parity error", -1, 1 },
+		{ F_DTAGPARERR, "CIM dtag parity error", -1, 1 },
 		{ 0 }
         };
 
-	if (t3_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE, 0xffffffff,
+	if (t3_handle_intr_status(adapter, A_CIM_HOST_INT_CAUSE, CIM_INTR_MASK,
 				  cim_intr_info, NULL))
 		t3_fatal_err(adapter);
 }
@@ -1405,7 +1556,14 @@ static void cim_intr_handler(adapter_t *adapter)
 static void ulprx_intr_handler(adapter_t *adapter)
 {
 	static struct intr_info ulprx_intr_info[] = {
-		{ F_PARERR, "ULP RX parity error", -1, 1 },
+		{ F_PARERRDATA, "ULP RX data parity error", -1, 1 },
+		{ F_PARERRPCMD, "ULP RX command parity error", -1, 1 },
+		{ F_ARBPF1PERR, "ULP RX ArbPF1 parity error", -1, 1 },
+		{ F_ARBPF0PERR, "ULP RX ArbPF0 parity error", -1, 1 },
+		{ F_ARBFPERR, "ULP RX ArbF parity error", -1, 1 },
+		{ F_PCMDMUXPERR, "ULP RX PCMDMUX parity error", -1, 1 },
+		{ F_DATASELFRAMEERR1, "ULP RX frame error", -1, 1 },
+		{ F_DATASELFRAMEERR0, "ULP RX frame error", -1, 1 },
 		{ 0 }
         };
 
@@ -1424,6 +1582,7 @@ static void ulptx_intr_handler(adapter_t *adapter)
 		  STAT_ULP_CH0_PBL_OOB, 0 },
 		{ F_PBL_BOUND_ERR_CH1, "ULP TX channel 1 PBL out of bounds",
 		  STAT_ULP_CH1_PBL_OOB, 0 },
+		{ 0xfc, "ULP TX parity error", -1, 1 },
 		{ 0 }
         };
 
@@ -1498,7 +1657,8 @@ static void pmrx_intr_handler(adapter_t *adapter)
 static void cplsw_intr_handler(adapter_t *adapter)
 {
 	static struct intr_info cplsw_intr_info[] = {
-//		{ F_CIM_OVFL_ERROR, "CPL switch CIM overflow", -1, 1 },
+		{ F_CIM_OP_MAP_PERR, "CPL switch CIM parity error", -1, 1 },
+		{ F_CIM_OVFL_ERROR, "CPL switch CIM overflow", -1, 1 },
 		{ F_TP_FRAMING_ERROR, "CPL switch TP framing error", -1, 1 },
 		{ F_SGE_FRAMING_ERROR, "CPL switch SGE framing error", -1, 1 },
 		{ F_CIM_FRAMING_ERROR, "CPL switch CIM framing error", -1, 1 },
@@ -1632,7 +1792,7 @@ int t3_phy_intr_handler(adapter_t *adapter)
 		mask = gpi - (gpi & (gpi - 1));
 		gpi -= mask;
 
-		if (!(p->port_type->caps & SUPPORTED_IRQ))
+		if (!(p->phy.caps & SUPPORTED_IRQ))
 			continue;
 
 		if (cause & mask) {
@@ -1728,7 +1888,6 @@ void t3_intr_enable(adapter_t *adapter)
 			MC7_INTR_MASK },
 		{ A_MC5_DB_INT_ENABLE, MC5_INTR_MASK },
 		{ A_ULPRX_INT_ENABLE, ULPRX_INTR_MASK },
-		{ A_TP_INT_ENABLE, 0x3bfffff },
 		{ A_PM1_TX_INT_ENABLE, PMTX_INTR_MASK },
 		{ A_PM1_RX_INT_ENABLE, PMRX_INTR_MASK },
 		{ A_CIM_HOST_INT_ENABLE, CIM_INTR_MASK },
@@ -1738,6 +1897,8 @@ void t3_intr_enable(adapter_t *adapter)
 	adapter->slow_intr_mask = PL_INTR_MASK;
 
 	t3_write_regs(adapter, intr_en_avp, ARRAY_SIZE(intr_en_avp), 0);
+	t3_write_reg(adapter, A_TP_INT_ENABLE,
+		     adapter->params.rev >= T3_REV_C ? 0x2bfffff : 0x3bfffff);
 
 	if (adapter->params.rev > 0) {
 		t3_write_reg(adapter, A_CPL_INTR_ENABLE,
@@ -1889,6 +2050,15 @@ static int t3_sge_write_context(adapter_t *adapter, unsigned int id,
 			       0, SG_CONTEXT_CMD_ATTEMPTS, 1);
 }
 
+static int clear_sge_ctxt(adapter_t *adap, unsigned int id, unsigned int type)
+{
+	t3_write_reg(adap, A_SG_CONTEXT_DATA0, 0);
+	t3_write_reg(adap, A_SG_CONTEXT_DATA1, 0);
+	t3_write_reg(adap, A_SG_CONTEXT_DATA2, 0);
+	t3_write_reg(adap, A_SG_CONTEXT_DATA3, 0);
+	return t3_sge_write_context(adap, id, type);
+}
+
 /**
  *	t3_sge_init_ecntxt - initialize an SGE egress context
  *	@adapter: the adapter to configure
@@ -2390,20 +2560,6 @@ static void tp_wr_bits_indirect(adapter_t *adap, unsigned int addr,
 }
 
 /**
- *	t3_enable_filters - enable the HW filters
- *	@adap: the adapter
- *
- *	Enables the HW filters for NIC traffic.
- */
-void t3_enable_filters(adapter_t *adap)
-{
-	t3_set_reg_field(adap, A_TP_IN_CONFIG, F_NICMODE, 0);
-	t3_set_reg_field(adap, A_MC5_DB_CONFIG, 0, F_FILTEREN);
-	t3_set_reg_field(adap, A_TP_GLOBAL_CONFIG, 0, V_FIVETUPLELOOKUP(3));
-	tp_wr_bits_indirect(adap, A_TP_INGRESS_CONFIG, 0, F_LOOKUPEVERYPKT);
-}
-
-/**
  *	pm_num_pages - calculate the number of pages of the payload memory
  *	@mem_size: the size of the payload memory
  *	@pg_size: the size of each payload memory page
@@ -2508,7 +2664,7 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
 		     V_AUTOSTATE2(1) | V_AUTOSTATE1(0) |
 		     V_BYTETHRESHOLD(16384) | V_MSSTHRESHOLD(2) |
 		     F_AUTOCAREFUL | F_AUTOENABLE | V_DACK_MODE(1));
-	t3_set_reg_field(adap, A_TP_IN_CONFIG, F_IPV6ENABLE | F_NICMODE,
+	t3_set_reg_field(adap, A_TP_IN_CONFIG, F_RXFBARBPRIO | F_TXFBARBPRIO,
 			 F_IPV6ENABLE | F_NICMODE);
 	t3_write_reg(adap, A_TP_TX_RESOURCE_LIMIT, 0x18141814);
 	t3_write_reg(adap, A_TP_PARA_REG4, 0x5050105);
@@ -2519,7 +2675,9 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
 			 F_ENABLEEPCMDAFULL,
 			 F_ENABLEOCSPIFULL |F_TXDEFERENABLE | F_HEARBEATDACK |
 			 F_TXCONGESTIONMODE | F_RXCONGESTIONMODE);
-	t3_set_reg_field(adap, A_TP_PC_CONFIG2, F_CHDRAFULL, 0);
+	t3_set_reg_field(adap, A_TP_PC_CONFIG2, F_CHDRAFULL,
+			 F_ENABLEIPV6RSS | F_ENABLENONOFDTNLSYN |
+			 F_ENABLEARPMISS | F_DISBLEDAPARBIT0);
 	t3_write_reg(adap, A_TP_PROXY_FLOW_CNTL, 1080);
 	t3_write_reg(adap, A_TP_PROXY_FLOW_CNTL, 1000);
 
@@ -2534,6 +2692,11 @@ static void tp_config(adapter_t *adap, const struct tp_params *p)
 	} else
 		t3_set_reg_field(adap, A_TP_PARA_REG3, 0, F_TXPACEFIXED);
 
+	if (adap->params.rev == T3_REV_C)
+		t3_set_reg_field(adap, A_TP_PC_CONFIG,
+				 V_TABLELATENCYDELTA(M_TABLELATENCYDELTA),
+				 V_TABLELATENCYDELTA(4));
+
 	t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT1, 0);
 	t3_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0, 0);
 	t3_write_reg(adap, A_TP_MOD_CHANNEL_WEIGHT, 0);
@@ -2972,7 +3135,7 @@ int t3_config_sched(adapter_t *adap, unsigned int kbps, int sched)
 			if (bpt > 0 && bpt <= 255) {
 				v = bpt * tps;
 				delta = v >= kbps ? v - kbps : kbps - v;
-				if (delta <= mindelta) {
+				if (delta < mindelta) {
 					mindelta = delta;
 					selected_cpt = cpt;
 					selected_bpt = bpt;
@@ -3383,7 +3546,8 @@ static void config_pcie(adapter_t *adap)
 			 V_REPLAYLMT(rpllmt));
 
 	t3_write_reg(adap, A_PCIE_PEX_ERR, 0xffffffff);
-	t3_set_reg_field(adap, A_PCIE_CFG, F_PCIE_CLIDECEN, F_PCIE_CLIDECEN);
+	t3_set_reg_field(adap, A_PCIE_CFG, 0,
+			 F_PCIE_DMASTOPEN | F_PCIE_CLIDECEN);
 }
 
 /**
@@ -3401,7 +3565,7 @@ static void config_pcie(adapter_t *adap)
  */
 int t3_init_hw(adapter_t *adapter, u32 fw_params)
 {
-	int err = -EIO, attempts = 100;
+	int err = -EIO, attempts, i;
 	const struct vpd_params *vpd = &adapter->params.vpd;
 
 	if (adapter->params.rev > 0)
@@ -3422,6 +3586,10 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
 			        adapter->params.mc5.nfilters,
 			       	adapter->params.mc5.nroutes))
 			goto out_err;
+
+		for (i = 0; i < 32; i++)
+			if (clear_sge_ctxt(adapter, i, F_CQ))
+				goto out_err;
 	}
 
 	if (tp_init(adapter, &adapter->params.tp))
@@ -3438,7 +3606,12 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
 	if (is_pcie(adapter))
 		config_pcie(adapter);
 	else
-		t3_set_reg_field(adapter, A_PCIX_CFG, 0, F_CLIDECEN);
+		t3_set_reg_field(adapter, A_PCIX_CFG, 0,
+				 F_DMASTOPEN | F_CLIDECEN);
+
+	if (adapter->params.rev == T3_REV_C)
+		t3_set_reg_field(adapter, A_ULPTX_CONFIG, 0,
+				 F_CFG_CQE_SOP_MASK);
 
 	t3_write_reg(adapter, A_PM1_RX_CFG, 0xffffffff);
 	t3_write_reg(adapter, A_PM1_RX_MODE, 0);
@@ -3451,6 +3624,7 @@ int t3_init_hw(adapter_t *adapter, u32 fw_params)
 		     V_BOOTADDR(FW_FLASH_BOOT_ADDR >> 2));
 	(void) t3_read_reg(adapter, A_CIM_BOOT_CFG);    /* flush */
 
+	attempts = 100;
 	do {                          /* wait for uP to initialize */
 		msleep(20);
 	} while (t3_read_reg(adapter, A_CIM_HOST_ACC_DATA) && --attempts);
@@ -3601,6 +3775,7 @@ void early_hw_init(adapter_t *adapter, const struct adapter_info *ai)
 	t3_write_reg(adapter, A_T3DBG_GPIO_EN,
 		     ai->gpio_out | F_GPIO0_OEN | F_GPIO0_OUT_VAL);
 	t3_write_reg(adapter, A_MC5_DB_SERVER_INDEX, 0);
+	t3_write_reg(adapter, A_SG_OCO_BASE, V_BASE1(0xfff));
 
 	if (adapter->params.rev == 0 || !uses_xaui(adapter))
 		val |= F_ENRGMII;
@@ -3651,6 +3826,36 @@ static int t3_reset_adapter(adapter_t *adapter)
 	return 0;
 }
 
+static int __devinit init_parity(adapter_t *adap)
+{
+	int i, err, addr;
+
+	if (t3_read_reg(adap, A_SG_CONTEXT_CMD) & F_CONTEXT_CMD_BUSY)
+		return -EBUSY;
+
+	for (err = i = 0; !err && i < 16; i++)
+		err = clear_sge_ctxt(adap, i, F_EGRESS);
+	for (i = 0xfff0; !err && i <= 0xffff; i++)
+		err = clear_sge_ctxt(adap, i, F_EGRESS);
+	for (i = 0; !err && i < SGE_QSETS; i++)
+		err = clear_sge_ctxt(adap, i, F_RESPONSEQ);
+	if (err)
+		return err;
+
+	t3_write_reg(adap, A_CIM_IBQ_DBG_DATA, 0);
+	for (i = 0; i < 4; i++)
+		for (addr = 0; addr <= M_IBQDBGADDR; addr++) {
+			t3_write_reg(adap, A_CIM_IBQ_DBG_CFG, F_IBQDBGEN |
+				     F_IBQDBGWR | V_IBQDBGQID(i) |
+				     V_IBQDBGADDR(addr));
+			err = t3_wait_op_done(adap, A_CIM_IBQ_DBG_CFG,
+					      F_IBQDBGBUSY, 0, 2, 1);
+			if (err)
+				return err;
+		}
+	return 0;
+}
+
 /**
  *	t3_prep_adapter - prepare SW and HW for operation
  *	@adapter: the adapter
@@ -3732,6 +3937,9 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
 	}
 
 	early_hw_init(adapter, ai);
+	ret = init_parity(adapter);
+	if (ret)
+		return ret;
 
 	if (adapter->params.nports > 2 &&
 	    (ret = t3_vsc7323_init(adapter, adapter->params.nports)))
@@ -3739,14 +3947,17 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
 
 	for_each_port(adapter, i) {
 		u8 hw_addr[6];
+		const struct port_type_info *pti;
 		struct port_info *p = adap2pinfo(adapter, i);
 
 		while (!adapter->params.vpd.port_type[j])
 			++j;
 
-		p->port_type = &port_types[adapter->params.vpd.port_type[j]];
-		p->port_type->phy_prep(&p->phy, adapter, ai->phy_base_addr + j,
-				       ai->mdio_ops);
+		pti = &port_types[adapter->params.vpd.port_type[j]];
+		ret = pti->phy_prep(&p->phy, adapter, ai->phy_base_addr + j,
+				    ai->mdio_ops);
+		if (ret)
+			return ret;
 		mac_prep(&p->mac, adapter, j);
 		++j;
 
@@ -3759,9 +3970,9 @@ int __devinit t3_prep_adapter(adapter_t *adapter,
 		hw_addr[5] = adapter->params.vpd.eth_base[5] + i;
 
 		t3_os_set_hw_addr(adapter, i, hw_addr);
-		init_link_config(&p->link_config, p->port_type->caps);
+		init_link_config(&p->link_config, p->phy.caps);
 		p->phy.ops->power_down(&p->phy, 1);
-		if (!(p->port_type->caps & SUPPORTED_IRQ))
+		if (!(p->phy.caps & SUPPORTED_IRQ))
 			adapter->params.linkpoll_period = 10;
 	}
 
diff --git a/sys/dev/cxgb/common/cxgb_tcb.h b/sys/dev/cxgb/common/cxgb_tcb.h
index 5dc72f5..7785466 100644
--- a/sys/dev/cxgb/common/cxgb_tcb.h
+++ b/sys/dev/cxgb/common/cxgb_tcb.h
@@ -668,7 +668,10 @@ $FreeBSD$
 #define S_TF_DDP_BUF1_FLUSH    28
 #define V_TF_DDP_BUF1_FLUSH(x) ((x) << S_TF_DDP_BUF1_FLUSH)
 
-#define S_TF_DDP_PSH_NO_INVALIDATE    29
-#define V_TF_DDP_PSH_NO_INVALIDATE(x) ((x) << S_TF_DDP_PSH_NO_INVALIDATE)
+#define S_TF_DDP_PSH_NO_INVALIDATE0    29
+#define V_TF_DDP_PSH_NO_INVALIDATE0(x) ((x) << S_TF_DDP_PSH_NO_INVALIDATE0)
+
+#define S_TF_DDP_PSH_NO_INVALIDATE1    30
+#define V_TF_DDP_PSH_NO_INVALIDATE1(x) ((x) << S_TF_DDP_PSH_NO_INVALIDATE1)
 
 #endif /* _TCB_DEFS_H */
diff --git a/sys/dev/cxgb/common/cxgb_version.h b/sys/dev/cxgb/common/cxgb_version.h
index 88296af..5867beb 100644
--- a/sys/dev/cxgb/common/cxgb_version.h
+++ b/sys/dev/cxgb/common/cxgb_version.h
@@ -37,5 +37,5 @@ $FreeBSD$
 #define __CHELSIO_VERSION_H
 #define DRV_DESC "Chelsio T3 Network Driver"
 #define DRV_NAME "cxgb"
-#define DRV_VERSION "1.0.086"
+#define DRV_VERSION "1.0.129a"
 #endif
diff --git a/sys/dev/cxgb/common/cxgb_vsc8211.c b/sys/dev/cxgb/common/cxgb_vsc8211.c
index 382ecc7..61bdc9c 100644
--- a/sys/dev/cxgb/common/cxgb_vsc8211.c
+++ b/sys/dev/cxgb/common/cxgb_vsc8211.c
@@ -36,11 +36,17 @@ __FBSDID("$FreeBSD$");
 #include <dev/cxgb/cxgb_include.h>
 #endif
 
+#undef msleep
+#define msleep t3_os_sleep
+
 /* VSC8211 PHY specific registers. */
 enum {
+	VSC8211_SIGDET_CTRL   = 19,
+	VSC8211_EXT_CTRL      = 23,
 	VSC8211_INTR_ENABLE   = 25,
 	VSC8211_INTR_STATUS   = 26,
 	VSC8211_AUX_CTRL_STAT = 28,
+	VSC8211_EXT_PAGE_AXS  = 31,
 };
 
 enum {
@@ -55,11 +61,19 @@ enum {
 	VSC_INTR_SYMBOL_ERR = 1 << 8,  /* symbol error */
 	VSC_INTR_NEG_DONE   = 1 << 10, /* autoneg done */
 	VSC_INTR_NEG_ERR    = 1 << 11, /* autoneg error */
+	VSC_INTR_DPLX_CHG   = 1 << 12, /* duplex change */
 	VSC_INTR_LINK_CHG   = 1 << 13, /* link change */
+	VSC_INTR_SPD_CHG    = 1 << 14, /* speed change */
 	VSC_INTR_ENABLE     = 1 << 15, /* interrupt enable */
 };
 
+enum {
+	VSC_CTRL_CLAUSE37_VIEW = 1 << 4,   /* Switch to Clause 37 view */
+	VSC_CTRL_MEDIA_MODE_HI = 0xf000    /* High part of media mode select */
+};
+
 #define CFG_CHG_INTR_MASK (VSC_INTR_LINK_CHG | VSC_INTR_NEG_ERR | \
+			   VSC_INTR_DPLX_CHG | VSC_INTR_SPD_CHG | \
 	 		   VSC_INTR_NEG_DONE)
 #define INTR_MASK (CFG_CHG_INTR_MASK | VSC_INTR_TX_FIFO | VSC_INTR_RX_FIFO | \
 		   VSC_INTR_ENABLE)
@@ -189,6 +203,98 @@ static int vsc8211_get_link_status(struct cphy *cphy, int *link_ok,
 	return 0;
 }
 
+static int vsc8211_get_link_status_fiber(struct cphy *cphy, int *link_ok,
+					 int *speed, int *duplex, int *fc)
+{
+	unsigned int bmcr, status, lpa, adv;
+	int err, sp = -1, dplx = -1, pause = 0;
+
+	err = mdio_read(cphy, 0, MII_BMCR, &bmcr);
+	if (!err)
+		err = mdio_read(cphy, 0, MII_BMSR, &status);
+	if (err)
+		return err;
+
+	if (link_ok) {
+		/*
+		 * BMSR_LSTATUS is latch-low, so if it is 0 we need to read it
+		 * once more to get the current link state.
+		 */
+		if (!(status & BMSR_LSTATUS))
+			err = mdio_read(cphy, 0, MII_BMSR, &status);
+		if (err)
+			return err;
+		*link_ok = (status & BMSR_LSTATUS) != 0;
+	}
+	if (!(bmcr & BMCR_ANENABLE)) {
+		dplx = (bmcr & BMCR_FULLDPLX) ? DUPLEX_FULL : DUPLEX_HALF;
+		if (bmcr & BMCR_SPEED1000)
+			sp = SPEED_1000;
+		else if (bmcr & BMCR_SPEED100)
+			sp = SPEED_100;
+		else
+			sp = SPEED_10;
+	} else if (status & BMSR_ANEGCOMPLETE) {
+		err = mdio_read(cphy, 0, MII_LPA, &lpa);
+		if (!err)
+			err = mdio_read(cphy, 0, MII_ADVERTISE, &adv);
+		if (err)
+			return err;
+
+		if (adv & lpa & ADVERTISE_1000XFULL) {
+			dplx = DUPLEX_FULL;
+			sp = SPEED_1000;
+		} else if (adv & lpa & ADVERTISE_1000XHALF) {
+			dplx = DUPLEX_HALF;
+			sp = SPEED_1000;
+		}
+
+		if (fc && dplx == DUPLEX_FULL) {
+			if (lpa & adv & ADVERTISE_1000XPAUSE)
+				pause = PAUSE_RX | PAUSE_TX;
+			else if ((lpa & ADVERTISE_1000XPAUSE) &&
+				 (adv & lpa & ADVERTISE_1000XPSE_ASYM))
+				pause = PAUSE_TX;
+			else if ((lpa & ADVERTISE_1000XPSE_ASYM) &&
+				 (adv & ADVERTISE_1000XPAUSE))
+				pause = PAUSE_RX;
+		}
+	}
+	if (speed)
+		*speed = sp;
+	if (duplex)
+		*duplex = dplx;
+	if (fc)
+		*fc = pause;
+	return 0;
+}
+
+/*
+ * Enable/disable auto MDI/MDI-X in forced link speed mode.
+ */
+static int vsc8211_set_automdi(struct cphy *phy, int enable)
+{
+	int err;
+
+	if ((err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 0x52b5)) != 0 ||
+	    (err = mdio_write(phy, 0, 18, 0x12)) != 0 ||
+	    (err = mdio_write(phy, 0, 17, enable ? 0x2803 : 0x3003)) != 0 ||
+	    (err = mdio_write(phy, 0, 16, 0x87fa)) != 0 ||
+	    (err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 0)) != 0)
+		return err;
+	return 0;
+}
+
+static int vsc8211_set_speed_duplex(struct cphy *phy, int speed, int duplex)
+{
+	int err;
+
+	err = t3_set_phy_speed_duplex(phy, speed, duplex);
+	if (!err)
+		err = vsc8211_set_automdi(phy, 1);
+	return err;
+}
+
 static int vsc8211_power_down(struct cphy *cphy, int enable)
 {
 	return t3_mdio_change_bits(cphy, 0, MII_BMCR, BMCR_PDOWN,
@@ -214,7 +320,6 @@ static int vsc8211_intr_handler(struct cphy *cphy)
 
 #ifdef C99_NOT_SUPPORTED
 static struct cphy_ops vsc8211_ops = {
-	NULL,
 	vsc8211_reset,
 	vsc8211_intr_enable,
 	vsc8211_intr_disable,
@@ -224,10 +329,25 @@ static struct cphy_ops vsc8211_ops = {
 	vsc8211_autoneg_restart,
 	t3_phy_advertise,
 	NULL,
-	t3_set_phy_speed_duplex,
+	vsc8211_set_speed_duplex,
 	vsc8211_get_link_status,
 	vsc8211_power_down,
 };
+
+static struct cphy_ops vsc8211_fiber_ops = {
+	vsc8211_reset,
+	vsc8211_intr_enable,
+	vsc8211_intr_disable,
+	vsc8211_intr_clear,
+	vsc8211_intr_handler,
+	vsc8211_autoneg_enable,
+	vsc8211_autoneg_restart,
+	t3_phy_advertise_fiber,
+	NULL,
+	t3_set_phy_speed_duplex,
+	vsc8211_get_link_status_fiber,
+	vsc8211_power_down,
+};
 #else
 static struct cphy_ops vsc8211_ops = {
 	.reset             = vsc8211_reset,
@@ -238,15 +358,57 @@ static struct cphy_ops vsc8211_ops = {
 	.autoneg_enable    = vsc8211_autoneg_enable,
 	.autoneg_restart   = vsc8211_autoneg_restart,
 	.advertise         = t3_phy_advertise,
-	.set_speed_duplex  = t3_set_phy_speed_duplex,
+	.set_speed_duplex  = vsc8211_set_speed_duplex,
 	.get_link_status   = vsc8211_get_link_status,
 	.power_down        = vsc8211_power_down,
 };
+
+static struct cphy_ops vsc8211_fiber_ops = {
+	.reset             = vsc8211_reset,
+	.intr_enable       = vsc8211_intr_enable,
+	.intr_disable      = vsc8211_intr_disable,
+	.intr_clear        = vsc8211_intr_clear,
+	.intr_handler      = vsc8211_intr_handler,
+	.autoneg_enable    = vsc8211_autoneg_enable,
+	.autoneg_restart   = vsc8211_autoneg_restart,
+	.advertise         = t3_phy_advertise_fiber,
+	.set_speed_duplex  = t3_set_phy_speed_duplex,
+	.get_link_status   = vsc8211_get_link_status_fiber,
+	.power_down        = vsc8211_power_down,
+};
 #endif
 
-void t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
-			 const struct mdio_ops *mdio_ops)
+int t3_vsc8211_phy_prep(struct cphy *phy, adapter_t *adapter, int phy_addr,
+			const struct mdio_ops *mdio_ops)
 {
-	cphy_init(phy, adapter, phy_addr, &vsc8211_ops, mdio_ops);
-	t3_os_sleep(20);       /* PHY needs ~10ms to start responding to MDIO */
+	int err;
+	unsigned int val;
+
+	cphy_init(phy, adapter, phy_addr, &vsc8211_ops, mdio_ops,
+		  SUPPORTED_10baseT_Full | SUPPORTED_100baseT_Full |
+		  SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg | SUPPORTED_MII |
+		  SUPPORTED_TP | SUPPORTED_IRQ, "10/100/1000BASE-T");
+	msleep(20);       /* PHY needs ~10ms to start responding to MDIO */
+
+	err = mdio_read(phy, 0, VSC8211_EXT_CTRL, &val);
+	if (err)
+		return err;
+	if (val & VSC_CTRL_MEDIA_MODE_HI)
+		return 0;   /* copper interface, done */
+
+	phy->caps = SUPPORTED_1000baseT_Full | SUPPORTED_Autoneg |
+		    SUPPORTED_MII | SUPPORTED_FIBRE | SUPPORTED_IRQ;
+	phy->desc = "1000BASE-X";
+	phy->ops = &vsc8211_fiber_ops;
+
+	if ((err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 1)) != 0 ||
+	    (err = mdio_write(phy, 0, VSC8211_SIGDET_CTRL, 1)) != 0 ||
+	    (err = mdio_write(phy, 0, VSC8211_EXT_PAGE_AXS, 0)) != 0 ||
+	    (err = mdio_write(phy, 0, VSC8211_EXT_CTRL,
+			      val | VSC_CTRL_CLAUSE37_VIEW)) != 0 ||
+	    (err = vsc8211_reset(phy, 0)) != 0)
+		return err;
+
+	udelay(5); /* delay after reset before next SMI */
+	return 0;
 }
diff --git a/sys/dev/cxgb/common/cxgb_xgmac.c b/sys/dev/cxgb/common/cxgb_xgmac.c
index ca8801f..745cc4b 100644
--- a/sys/dev/cxgb/common/cxgb_xgmac.c
+++ b/sys/dev/cxgb/common/cxgb_xgmac.c
@@ -75,6 +75,12 @@ static void xaui_serdes_reset(struct cmac *mac)
 	}
 }
 
+/**
+ *	t3b_pcs_reset - reset the PCS on T3B+ adapters
+ *	@mac: the XGMAC handle
+ *
+ *	Reset the XGMAC PCS block on T3B+ adapters.
+ */
 void t3b_pcs_reset(struct cmac *mac)
 {
 	t3_set_reg_field(mac->adapter, A_XGM_RESET_CTRL + mac->offset,
@@ -84,6 +90,12 @@ void t3b_pcs_reset(struct cmac *mac)
 			 F_PCS_RESET_);
 }
 
+/**
+ *	t3_mac_reset - reset a MAC
+ *	@mac: the MAC to reset
+ *
+ *	Reset the given MAC.
+ */
 int t3_mac_reset(struct cmac *mac)
 {
 	static struct addr_val_pair mac_reset_avp[] = {
@@ -114,6 +126,7 @@ int t3_mac_reset(struct cmac *mac)
 	t3_set_reg_field(adap, A_XGM_RXFIFO_CFG + oft,
 			 F_RXSTRFRWRD | F_DISERRFRAMES,
 			 uses_xaui(adap) ? 0 : F_RXSTRFRWRD);
+	t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + oft, 0, F_UNDERUNFIX);
 
 	if (uses_xaui(adap)) {
 		if (adap->params.rev == 0) {
@@ -146,8 +159,10 @@ int t3_mac_reset(struct cmac *mac)
 		t3_write_reg(adap, A_XGM_TX_CTRL + oft, F_TXEN);
 		t3_write_reg(adap, A_XGM_RX_CTRL + oft, F_RXEN);
 	}
-
-	val = F_MAC_RESET_;
+	t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + oft,
+			 V_RXMAXFRAMERSIZE(M_RXMAXFRAMERSIZE),
+			 V_RXMAXFRAMERSIZE(MAX_FRAME_SIZE) | F_RXENFRAMER);
+	val = F_MAC_RESET_ | F_XGMAC_STOP_EN;
 	if (is_10G(adap) || mac->multiport)
 		val |= F_PCS_RESET_;
 	else if (uses_xaui(adap))
@@ -236,7 +251,14 @@ static void set_addr_filter(struct cmac *mac, int idx, const u8 *addr)
 	t3_write_reg(mac->adapter, A_XGM_RX_EXACT_MATCH_HIGH_1 + oft, addr_hi);
 }
 
-/* Set one of the station's unicast MAC addresses. */
+/**
+ *	t3_mac_set_address - set one of the station's unicast MAC addresses
+ *	@mac: the MAC handle
+ *	@idx: index of the exact address match filter to use
+ *	@addr: the Ethernet address
+ *
+ *	Set one of the station's unicast MAC addresses.
+ */
 int t3_mac_set_address(struct cmac *mac, unsigned int idx, u8 addr[6])
 {
 	if (mac->multiport)
@@ -249,10 +271,14 @@ int t3_mac_set_address(struct cmac *mac, unsigned int idx, u8 addr[6])
 	return 0;
 }
 
-/*
- * Specify the number of exact address filters that should be reserved for
- * unicast addresses.  Caller should reload the unicast and multicast addresses
- * after calling this.
+/**
+ *	t3_mac_set_num_ucast - set the number of unicast addresses needed
+ *	@mac: the MAC handle
+ *	@n: number of unicast addresses needed
+ *
+ *	Specify the number of exact address filters that should be reserved for
+ *	unicast addresses.  Caller should reload the unicast and multicast
+ *	addresses after calling this.
  */
 int t3_mac_set_num_ucast(struct cmac *mac, unsigned char n)
 {
@@ -298,6 +324,14 @@ static int hash_hw_addr(const u8 *addr)
 	return hash;
 }
 
+/**
+ *	t3_mac_set_rx_mode - set the Rx mode and address filters
+ *	@mac: the MAC to configure
+ *	@rm: structure containing the Rx mode and MAC addresses needed
+ *
+ *	Configures the MAC Rx mode (promiscuity, etc) and exact and hash
+ *	address filters.
+ */
 int t3_mac_set_rx_mode(struct cmac *mac, struct t3_rx_mode *rm)
 {
 	u32 hash_lo, hash_hi;
@@ -344,10 +378,18 @@ static int rx_fifo_hwm(int mtu)
 	return min(hwm, MAC_RXFIFO_SIZE - 8192);
 }
 
+/**
+ *	t3_mac_set_mtu - set the MAC MTU
+ *	@mac: the MAC to configure
+ *	@mtu: the MTU
+ *
+ *	Sets the MAC MTU and adjusts the FIFO PAUSE watermarks accordingly.
+ */
 int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu) 
 {
-	int hwm, lwm;
-	unsigned int thres, v;
+	int hwm, lwm, divisor;
+	int ipg;
+	unsigned int thres, v, reg;
 	adapter_t *adap = mac->adapter;
 
 	/*
@@ -362,27 +404,33 @@ int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu)
 	if (mac->multiport)
 		return t3_vsc7323_set_mtu(adap, mtu - 4, mac->ext_port);
 
-	if (adap->params.rev == T3_REV_B2 &&
+	if (adap->params.rev >= T3_REV_B2 &&
 	    (t3_read_reg(adap, A_XGM_RX_CTRL + mac->offset) & F_RXEN)) {
 		disable_exact_filters(mac);
 		v = t3_read_reg(adap, A_XGM_RX_CFG + mac->offset);
 		t3_set_reg_field(adap, A_XGM_RX_CFG + mac->offset,
 				 F_ENHASHMCAST | F_COPYALLFRAMES, F_DISBCAST);
 
-		/* drain rx FIFO */
-		if (t3_wait_op_done(adap,
-				    A_XGM_RX_MAX_PKT_SIZE_ERR_CNT + mac->offset,
-				    1 << 31, 1, 20, 5)) {
+		reg = adap->params.rev == T3_REV_B2 ?
+			A_XGM_RX_MAX_PKT_SIZE_ERR_CNT : A_XGM_RXFIFO_CFG;
+	
+		/* drain RX FIFO */
+		if (t3_wait_op_done(adap, reg + mac->offset,
+				    F_RXFIFO_EMPTY, 1, 20, 5)) {
 			t3_write_reg(adap, A_XGM_RX_CFG + mac->offset, v);
 			enable_exact_filters(mac);
 			return -EIO;
 		}
-		t3_write_reg(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, mtu);
+		t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset,
+				 V_RXMAXPKTSIZE(M_RXMAXPKTSIZE),
+				 V_RXMAXPKTSIZE(mtu));
 		t3_write_reg(adap, A_XGM_RX_CFG + mac->offset, v);
 		enable_exact_filters(mac);
 	} else
-		t3_write_reg(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset, mtu);
-
+		t3_set_reg_field(adap, A_XGM_RX_MAX_PKT_SIZE + mac->offset,
+		                 V_RXMAXPKTSIZE(M_RXMAXPKTSIZE),
+		                 V_RXMAXPKTSIZE(mtu));
+	
 	/*
 	 * Adjust the PAUSE frame watermarks.  We always set the LWM, and the
 	 * HWM only if flow-control is enabled.
@@ -405,20 +453,34 @@ int t3_mac_set_mtu(struct cmac *mac, unsigned int mtu)
 		thres /= 10;
 	thres = mtu > thres ? (mtu - thres + 7) / 8 : 0;
 	thres = max(thres, 8U);                          /* need at least 8 */
+	ipg = (adap->params.rev == T3_REV_C) ? 0 : 1;
 	t3_set_reg_field(adap, A_XGM_TXFIFO_CFG + mac->offset,
 			 V_TXFIFOTHRESH(M_TXFIFOTHRESH) | V_TXIPG(M_TXIPG),
-			 V_TXFIFOTHRESH(thres) | V_TXIPG(1));
+			 V_TXFIFOTHRESH(thres) | V_TXIPG(ipg));
 
 	/* Assuming a minimum drain rate of 2.5Gbps...
 	 */
-	if (adap->params.rev > 0)
+	if (adap->params.rev > 0) {
+		divisor = (adap->params.rev == T3_REV_C) ? 64 : 8;
 		t3_write_reg(adap, A_XGM_PAUSE_TIMER + mac->offset, 
-			     (hwm - lwm) * 4 / 8);
+		    (hwm - lwm) * 4 / divisor);
+	}
 	t3_write_reg(adap, A_XGM_TX_PAUSE_QUANTA + mac->offset, 
 		     MAC_RXFIFO_SIZE * 4 * 8 / 512);
 	return 0;
 }
 
+/**
+ *	t3_mac_set_speed_duplex_fc - set MAC speed, duplex and flow control
+ *	@mac: the MAC to configure
+ *	@speed: the desired speed (10/100/1000/10000)
+ *	@duplex: the desired duplex
+ *	@fc: desired Tx/Rx PAUSE configuration
+ *
+ *	Set the MAC speed, duplex (actually only full-duplex is supported), and
+ *	flow control.  If a parameter value is negative the corresponding
+ *	MAC setting is left at its current value.
+ */
 int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc)
 {
 	u32 val;
@@ -466,6 +528,15 @@ int t3_mac_set_speed_duplex_fc(struct cmac *mac, int speed, int duplex, int fc)
 	return 0;
 }
 
+/**
+ *	t3_mac_enable - enable the MAC in the given directions
+ *	@mac: the MAC to configure
+ *	@which: bitmap indicating which directions to enable
+ *
+ *	Enables the MAC for operation in the given directions.
+ *	%MAC_DIRECTION_TX enables the Tx direction, and %MAC_DIRECTION_RX
+ *	enables the Rx one.
+ */
 int t3_mac_enable(struct cmac *mac, int which)
 {
 	int idx = macidx(mac);
@@ -478,9 +549,13 @@ int t3_mac_enable(struct cmac *mac, int which)
 
 	if (which & MAC_DIRECTION_TX) {
 		t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_CFG_CH0 + idx);
-		t3_write_reg(adap, A_TP_PIO_DATA, 0xc0ede401);
+		t3_write_reg(adap, A_TP_PIO_DATA,
+			     adap->params.rev == T3_REV_C ?
+			     0xc4ffff01 : 0xc0ede401);
 		t3_write_reg(adap, A_TP_PIO_ADDR, A_TP_TX_DROP_MODE);
-		t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx, 1 << idx);
+		t3_set_reg_field(adap, A_TP_PIO_DATA, 1 << idx,
+				 adap->params.rev == T3_REV_C ?
+				 0 : 1 << idx);
 
 		t3_write_reg(adap, A_XGM_TX_CTRL + oft, F_TXEN);
 
@@ -505,6 +580,15 @@ int t3_mac_enable(struct cmac *mac, int which)
 	return 0;
 }
 
+/**
+ *	t3_mac_disable - disable the MAC in the given directions
+ *	@mac: the MAC to configure
+ *	@which: bitmap indicating which directions to disable
+ *
+ *	Disables the MAC in the given directions.
+ *	%MAC_DIRECTION_TX disables the Tx direction, and %MAC_DIRECTION_RX
+ *	disables the Rx one.
+ */
 int t3_mac_disable(struct cmac *mac, int which)
 {
 	adapter_t *adap = mac->adapter;
@@ -621,12 +705,15 @@ out:
 	return status;
 }
 
-/*
- * This function is called periodically to accumulate the current values of the
- * RMON counters into the port statistics.  Since the packet counters are only
- * 32 bits they can overflow in ~286 secs at 10G, so the function should be
- * called more frequently than that.  The byte counters are 45-bit wide, they
- * would overflow in ~7.8 hours.
+/**
+ *	t3_mac_update_stats - accumulate MAC statistics
+ *	@mac: the MAC handle
+ *
+ *	This function is called periodically to accumulate the current values
+ *	of the RMON counters into the port statistics.  Since the packet
+ *	counters are only 32 bits they can overflow in ~286 secs at 10G, so the
+ *	function should be called more frequently than that.  The byte counters
+ *	are 45-bit wide, they would overflow in ~7.8 hours.
  */
 const struct mac_stats *t3_mac_update_stats(struct cmac *mac)
 {