cxgbe(4): Overhaul the shared code that deals with the chip's TP block,

which is responsible for filtering and RSS.

Add the ability to use filters that match on PF/VF (aka "VNIC id") while
here.  This is mutually exclusive with filtering on outer VLAN tag with
Q-in-Q.

Sponsored by:	Chelsio Communications

6 files changed, 441 insertions, 178 deletions

diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 88807d0..3d54c65 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -1026,6 +1026,17 @@ tx_resume_threshold(struct sge_eq *eq)
 	return (eq->sidx / 4);
 }
 
+static inline int
+t4_use_ldst(struct adapter *sc)
+{
+
+#ifdef notyet
+	return (sc->flags & FW_OK || !sc->use_bd);
+#else
+	return (0);
+#endif
+}
+
 /* t4_main.c */
 int t4_os_find_pci_capability(struct adapter *, int);
 int t4_os_pci_save_state(struct adapter *);
diff --git a/sys/dev/cxgbe/common/common.h b/sys/dev/cxgbe/common/common.h
index 142ab73..d0c3b39 100644
--- a/sys/dev/cxgbe/common/common.h
+++ b/sys/dev/cxgbe/common/common.h
@@ -234,17 +234,25 @@ struct sge_params {
 };
 
 struct tp_params {
-	unsigned int ntxchan;        /* # of Tx channels */
 	unsigned int tre;            /* log2 of core clocks per TP tick */
 	unsigned int dack_re;        /* DACK timer resolution */
 	unsigned int la_mask;        /* what events are recorded by TP LA */
 	unsigned short tx_modq[MAX_NCHAN];  /* channel to modulation queue map */
+
 	uint32_t vlan_pri_map;
 	uint32_t ingress_config;
-	int8_t vlan_shift;
-	int8_t vnic_shift;
+	uint32_t rx_pkt_encap;
+
+	int8_t fcoe_shift;
 	int8_t port_shift;
+	int8_t vnic_shift;
+	int8_t vlan_shift;
+	int8_t tos_shift;
 	int8_t protocol_shift;
+	int8_t ethertype_shift;
+	int8_t macmatch_shift;
+	int8_t matchtype_shift;
+	int8_t frag_shift;
 };
 
 struct vpd_params {
@@ -492,7 +500,6 @@ int t4_init_sge_params(struct adapter *adapter);
 int t4_init_tp_params(struct adapter *adap);
 int t4_filter_field_shift(const struct adapter *adap, int filter_sel);
 int t4_port_init(struct port_info *p, int mbox, int pf, int vf);
-int t4_reinit_adapter(struct adapter *adap);
 void t4_fatal_err(struct adapter *adapter);
 int t4_set_trace_filter(struct adapter *adapter, const struct trace_params *tp,
 			int filter_index, int enable);
@@ -505,8 +512,10 @@ int t4_config_glbl_rss(struct adapter *adapter, int mbox, unsigned int mode,
 int t4_config_vi_rss(struct adapter *adapter, int mbox, unsigned int viid,
 		     unsigned int flags, unsigned int defq);
 int t4_read_rss(struct adapter *adapter, u16 *entries);
+void t4_fw_tp_pio_rw(struct adapter *adap, u32 *vals, unsigned int nregs,
+		  unsigned int start_index, unsigned int rw);
 void t4_read_rss_key(struct adapter *adapter, u32 *key);
-void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx);
+void t4_write_rss_key(struct adapter *adap, u32 *key, int idx);
 void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, u32 *valp);
 void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index, u32 val);
 void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index,
diff --git a/sys/dev/cxgbe/common/t4_hw.c b/sys/dev/cxgbe/common/t4_hw.c
index 32d0f53..86a9180 100644
--- a/sys/dev/cxgbe/common/t4_hw.c
+++ b/sys/dev/cxgbe/common/t4_hw.c
@@ -4867,6 +4867,42 @@ int t4_read_rss(struct adapter *adapter, u16 *map)
 }
 
 /**
+ *	t4_fw_tp_pio_rw - Access TP PIO through LDST
+ *	@adap: the adapter
+ *	@vals: where the indirect register values are stored/written
+ *	@nregs: how many indirect registers to read/write
+ *	@start_idx: index of first indirect register to read/write
+ *	@rw: Read (1) or Write (0)
+ *
+ *	Access TP PIO registers through LDST
+ */
+void t4_fw_tp_pio_rw(struct adapter *adap, u32 *vals, unsigned int nregs,
+		     unsigned int start_index, unsigned int rw)
+{
+	int ret, i;
+	int cmd = FW_LDST_ADDRSPC_TP_PIO;
+	struct fw_ldst_cmd c;
+
+	for (i = 0 ; i < nregs; i++) {
+		memset(&c, 0, sizeof(c));
+		c.op_to_addrspace = cpu_to_be32(V_FW_CMD_OP(FW_LDST_CMD) |
+						F_FW_CMD_REQUEST |
+						(rw ? F_FW_CMD_READ :
+						     F_FW_CMD_WRITE) |
+						V_FW_LDST_CMD_ADDRSPACE(cmd));
+		c.cycles_to_len16 = cpu_to_be32(FW_LEN16(c));
+
+		c.u.addrval.addr = cpu_to_be32(start_index + i);
+		c.u.addrval.val  = rw ? 0 : cpu_to_be32(vals[i]);
+		ret = t4_wr_mbox(adap, adap->mbox, &c, sizeof(c), &c);
+		if (ret == 0) {
+			if (rw)
+				vals[i] = be32_to_cpu(c.u.addrval.val);
+		}
+	}
+}
+
+/**
  *	t4_read_rss_key - read the global RSS key
  *	@adap: the adapter
  *	@key: 10-entry array holding the 320-bit RSS key
@@ -4875,8 +4911,11 @@ int t4_read_rss(struct adapter *adapter, u16 *map)
  */
 void t4_read_rss_key(struct adapter *adap, u32 *key)
 {
-	t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, key, 10,
-			 A_TP_RSS_SECRET_KEY0);
+	if (t4_use_ldst(adap))
+		t4_fw_tp_pio_rw(adap, key, 10, A_TP_RSS_SECRET_KEY0, 1);
+	else
+		t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, key, 10,
+				 A_TP_RSS_SECRET_KEY0);
 }
 
 /**
@@ -4889,13 +4928,35 @@ void t4_read_rss_key(struct adapter *adap, u32 *key)
  *	0..15 the corresponding entry in the RSS key table is written,
  *	otherwise the global RSS key is written.
  */
-void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx)
+void t4_write_rss_key(struct adapter *adap, u32 *key, int idx)
 {
-	t4_write_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, key, 10,
-			  A_TP_RSS_SECRET_KEY0);
-	if (idx >= 0 && idx < 16)
-		t4_write_reg(adap, A_TP_RSS_CONFIG_VRT,
-			     V_KEYWRADDR(idx) | F_KEYWREN);
+	u8 rss_key_addr_cnt = 16;
+	u32 vrt = t4_read_reg(adap, A_TP_RSS_CONFIG_VRT);
+
+	/*
+	 * T6 and later: for KeyMode 3 (per-vf and per-vf scramble),
+	 * allows access to key addresses 16-63 by using KeyWrAddrX
+	 * as index[5:4](upper 2) into key table
+	 */
+	if ((chip_id(adap) > CHELSIO_T5) &&
+	    (vrt & F_KEYEXTEND) && (G_KEYMODE(vrt) == 3))
+		rss_key_addr_cnt = 32;
+
+	if (t4_use_ldst(adap))
+		t4_fw_tp_pio_rw(adap, key, 10, A_TP_RSS_SECRET_KEY0, 0);
+	else
+		t4_write_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, key, 10,
+				  A_TP_RSS_SECRET_KEY0);
+
+	if (idx >= 0 && idx < rss_key_addr_cnt) {
+		if (rss_key_addr_cnt > 16)
+			t4_write_reg(adap, A_TP_RSS_CONFIG_VRT,
+				     V_KEYWRADDRX(idx >> 4) |
+				     V_T6_VFWRADDR(idx) | F_KEYWREN);
+		else
+			t4_write_reg(adap, A_TP_RSS_CONFIG_VRT,
+				     V_KEYWRADDR(idx) | F_KEYWREN);
+	}
 }
 
 /**
@@ -4907,10 +4968,15 @@ void t4_write_rss_key(struct adapter *adap, const u32 *key, int idx)
  *	Reads the PF RSS Configuration Table at the specified index and returns
  *	the value found there.
  */
-void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, u32 *valp)
+void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index,
+			   u32 *valp)
 {
-	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			 valp, 1, A_TP_RSS_PF0_CONFIG + index);
+	if (t4_use_ldst(adapter))
+		t4_fw_tp_pio_rw(adapter, valp, 1,
+				A_TP_RSS_PF0_CONFIG + index, 1);
+	else
+		t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				 valp, 1, A_TP_RSS_PF0_CONFIG + index);
 }
 
 /**
@@ -4922,10 +4988,15 @@ void t4_read_rss_pf_config(struct adapter *adapter, unsigned int index, u32 *val
  *	Writes the PF RSS Configuration Table at the specified index with the
  *	specified value.
  */
-void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index, u32 val)
+void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index,
+			    u32 val)
 {
-	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			  &val, 1, A_TP_RSS_PF0_CONFIG + index);
+	if (t4_use_ldst(adapter))
+		t4_fw_tp_pio_rw(adapter, &val, 1,
+				A_TP_RSS_PF0_CONFIG + index, 0);
+	else
+		t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				  &val, 1, A_TP_RSS_PF0_CONFIG + index);
 }
 
 /**
@@ -4941,28 +5012,40 @@ void t4_write_rss_pf_config(struct adapter *adapter, unsigned int index, u32 val
 void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index,
 			   u32 *vfl, u32 *vfh)
 {
-	u32 vrt;
+	u32 vrt, mask, data;
 
+	if (chip_id(adapter) <= CHELSIO_T5) {
+		mask = V_VFWRADDR(M_VFWRADDR);
+		data = V_VFWRADDR(index);
+	} else {
+		 mask =  V_T6_VFWRADDR(M_T6_VFWRADDR);
+		 data = V_T6_VFWRADDR(index);
+	}
 	/*
 	 * Request that the index'th VF Table values be read into VFL/VFH.
 	 */
 	vrt = t4_read_reg(adapter, A_TP_RSS_CONFIG_VRT);
-	vrt &= ~(F_VFRDRG | V_VFWRADDR(M_VFWRADDR) | F_VFWREN | F_KEYWREN);
-	vrt |= V_VFWRADDR(index) | F_VFRDEN;
+	vrt &= ~(F_VFRDRG | F_VFWREN | F_KEYWREN | mask);
+	vrt |= data | F_VFRDEN;
 	t4_write_reg(adapter, A_TP_RSS_CONFIG_VRT, vrt);
 
 	/*
 	 * Grab the VFL/VFH values ...
 	 */
-	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			 vfl, 1, A_TP_RSS_VFL_CONFIG);
-	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			 vfh, 1, A_TP_RSS_VFH_CONFIG);
+	if (t4_use_ldst(adapter)) {
+		t4_fw_tp_pio_rw(adapter, vfl, 1, A_TP_RSS_VFL_CONFIG, 1);
+		t4_fw_tp_pio_rw(adapter, vfh, 1, A_TP_RSS_VFH_CONFIG, 1);
+	} else {
+		t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				 vfl, 1, A_TP_RSS_VFL_CONFIG);
+		t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				 vfh, 1, A_TP_RSS_VFH_CONFIG);
+	}
 }
 
 /**
  *	t4_write_rss_vf_config - write VF RSS Configuration Table
- *	
+ *
  *	@adapter: the adapter
  *	@index: the entry in the VF RSS table to write
  *	@vfl: the VFL to store
@@ -4974,22 +5057,35 @@ void t4_read_rss_vf_config(struct adapter *adapter, unsigned int index,
 void t4_write_rss_vf_config(struct adapter *adapter, unsigned int index,
 			    u32 vfl, u32 vfh)
 {
-	u32 vrt;
+	u32 vrt, mask, data;
+
+	if (chip_id(adapter) <= CHELSIO_T5) {
+		mask = V_VFWRADDR(M_VFWRADDR);
+		data = V_VFWRADDR(index);
+	} else {
+		mask =  V_T6_VFWRADDR(M_T6_VFWRADDR);
+		data = V_T6_VFWRADDR(index);
+	}
 
 	/*
 	 * Load up VFL/VFH with the values to be written ...
 	 */
-	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			  &vfl, 1, A_TP_RSS_VFL_CONFIG);
-	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			  &vfh, 1, A_TP_RSS_VFH_CONFIG);
+	if (t4_use_ldst(adapter)) {
+		t4_fw_tp_pio_rw(adapter, &vfl, 1, A_TP_RSS_VFL_CONFIG, 0);
+		t4_fw_tp_pio_rw(adapter, &vfh, 1, A_TP_RSS_VFH_CONFIG, 0);
+	} else {
+		t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				  &vfl, 1, A_TP_RSS_VFL_CONFIG);
+		t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				  &vfh, 1, A_TP_RSS_VFH_CONFIG);
+	}
 
 	/*
 	 * Write the VFL/VFH into the VF Table at index'th location.
 	 */
 	vrt = t4_read_reg(adapter, A_TP_RSS_CONFIG_VRT);
-	vrt &= ~(F_VFRDRG | F_VFRDEN | V_VFWRADDR(M_VFWRADDR) | F_KEYWREN);
-	vrt |= V_VFWRADDR(index) | F_VFWREN;
+	vrt &= ~(F_VFRDRG | F_VFWREN | F_KEYWREN | mask);
+	vrt |= data | F_VFRDEN;
 	t4_write_reg(adapter, A_TP_RSS_CONFIG_VRT, vrt);
 }
 
@@ -5003,8 +5099,11 @@ u32 t4_read_rss_pf_map(struct adapter *adapter)
 {
 	u32 pfmap;
 
-	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			 &pfmap, 1, A_TP_RSS_PF_MAP);
+	if (t4_use_ldst(adapter))
+		t4_fw_tp_pio_rw(adapter, &pfmap, 1, A_TP_RSS_PF_MAP, 1);
+	else
+		t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				 &pfmap, 1, A_TP_RSS_PF_MAP);
 	return pfmap;
 }
 
@@ -5017,8 +5116,11 @@ u32 t4_read_rss_pf_map(struct adapter *adapter)
  */
 void t4_write_rss_pf_map(struct adapter *adapter, u32 pfmap)
 {
-	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			  &pfmap, 1, A_TP_RSS_PF_MAP);
+	if (t4_use_ldst(adapter))
+		t4_fw_tp_pio_rw(adapter, &pfmap, 1, A_TP_RSS_PF_MAP, 0);
+	else
+		t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				  &pfmap, 1, A_TP_RSS_PF_MAP);
 }
 
 /**
@@ -5031,8 +5133,11 @@ u32 t4_read_rss_pf_mask(struct adapter *adapter)
 {
 	u32 pfmask;
 
-	t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			 &pfmask, 1, A_TP_RSS_PF_MSK);
+	if (t4_use_ldst(adapter))
+		t4_fw_tp_pio_rw(adapter, &pfmask, 1, A_TP_RSS_PF_MSK, 1);
+	else
+		t4_read_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				 &pfmask, 1, A_TP_RSS_PF_MSK);
 	return pfmask;
 }
 
@@ -5045,61 +5150,11 @@ u32 t4_read_rss_pf_mask(struct adapter *adapter)
  */
 void t4_write_rss_pf_mask(struct adapter *adapter, u32 pfmask)
 {
-	t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			  &pfmask, 1, A_TP_RSS_PF_MSK);
-}
-
-static void refresh_vlan_pri_map(struct adapter *adap)
-{
-
-        t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-                         &adap->params.tp.vlan_pri_map, 1,
-                         A_TP_VLAN_PRI_MAP);
-
-	/*
-	 * Now that we have TP_VLAN_PRI_MAP cached, we can calculate the field
-	 * shift positions of several elements of the Compressed Filter Tuple
-	 * for this adapter which we need frequently ...
-	 */
-	adap->params.tp.vlan_shift = t4_filter_field_shift(adap, F_VLAN);
-	adap->params.tp.vnic_shift = t4_filter_field_shift(adap, F_VNIC_ID);
-	adap->params.tp.port_shift = t4_filter_field_shift(adap, F_PORT);
-	adap->params.tp.protocol_shift = t4_filter_field_shift(adap, F_PROTOCOL);
-
-	/*
-	 * If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID
-	 * represents the presense of an Outer VLAN instead of a VNIC ID.
-	 */
-	if ((adap->params.tp.ingress_config & F_VNIC) == 0)
-		adap->params.tp.vnic_shift = -1;
-}
-
-/**
- *	t4_set_filter_mode - configure the optional components of filter tuples
- *	@adap: the adapter
- *	@mode_map: a bitmap selcting which optional filter components to enable
- *
- *	Sets the filter mode by selecting the optional components to enable
- *	in filter tuples.  Returns 0 on success and a negative error if the
- *	requested mode needs more bits than are available for optional
- *	components.
- */
-int t4_set_filter_mode(struct adapter *adap, unsigned int mode_map)
-{
-	static u8 width[] = { 1, 3, 17, 17, 8, 8, 16, 9, 3, 1 };
-
-	int i, nbits = 0;
-
-	for (i = S_FCOE; i <= S_FRAGMENTATION; i++)
-		if (mode_map & (1 << i))
-			nbits += width[i];
-	if (nbits > FILTER_OPT_LEN)
-		return -EINVAL;
-	t4_write_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, &mode_map, 1,
-			  A_TP_VLAN_PRI_MAP);
-	refresh_vlan_pri_map(adap);
-
-	return 0;
+	if (t4_use_ldst(adapter))
+		t4_fw_tp_pio_rw(adapter, &pfmask, 1, A_TP_RSS_PF_MSK, 0);
+	else
+		t4_write_indirect(adapter, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				  &pfmask, 1, A_TP_RSS_PF_MSK);
 }
 
 /**
@@ -7700,41 +7755,91 @@ int t4_init_sge_params(struct adapter *adapter)
 	return 0;
 }
 
+/*
+ * Read and cache the adapter's compressed filter mode and ingress config.
+ */
+static void read_filter_mode_and_ingress_config(struct adapter *adap)
+{
+	struct tp_params *tpp = &adap->params.tp;
+
+	if (t4_use_ldst(adap)) {
+		t4_fw_tp_pio_rw(adap, &tpp->vlan_pri_map, 1,
+				A_TP_VLAN_PRI_MAP, 1);
+		t4_fw_tp_pio_rw(adap, &tpp->ingress_config, 1,
+				A_TP_INGRESS_CONFIG, 1);
+	} else {
+		t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				 &tpp->vlan_pri_map, 1, A_TP_VLAN_PRI_MAP);
+		t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA,
+				 &tpp->ingress_config, 1, A_TP_INGRESS_CONFIG);
+	}
+
+	/*
+	 * Now that we have TP_VLAN_PRI_MAP cached, we can calculate the field
+	 * shift positions of several elements of the Compressed Filter Tuple
+	 * for this adapter which we need frequently ...
+	 */
+	tpp->fcoe_shift = t4_filter_field_shift(adap, F_FCOE);
+	tpp->port_shift = t4_filter_field_shift(adap, F_PORT);
+	tpp->vnic_shift = t4_filter_field_shift(adap, F_VNIC_ID);
+	tpp->vlan_shift = t4_filter_field_shift(adap, F_VLAN);
+	tpp->tos_shift = t4_filter_field_shift(adap, F_TOS);
+	tpp->protocol_shift = t4_filter_field_shift(adap, F_PROTOCOL);
+	tpp->ethertype_shift = t4_filter_field_shift(adap, F_ETHERTYPE);
+	tpp->macmatch_shift = t4_filter_field_shift(adap, F_MACMATCH);
+	tpp->matchtype_shift = t4_filter_field_shift(adap, F_MPSHITTYPE);
+	tpp->frag_shift = t4_filter_field_shift(adap, F_FRAGMENTATION);
+
+	/*
+	 * If TP_INGRESS_CONFIG.VNID == 0, then TP_VLAN_PRI_MAP.VNIC_ID
+	 * represents the presense of an Outer VLAN instead of a VNIC ID.
+	 */
+	if ((tpp->ingress_config & F_VNIC) == 0)
+		tpp->vnic_shift = -1;
+}
+
 /**
- *	t4_init_tp_params - initialize adap->params.tp
- *	@adap: the adapter
+ *      t4_init_tp_params - initialize adap->params.tp
+ *      @adap: the adapter
  *
- *	Initialize various fields of the adapter's TP Parameters structure.
+ *      Initialize various fields of the adapter's TP Parameters structure.
  */
-int __devinit t4_init_tp_params(struct adapter *adap)
+int t4_init_tp_params(struct adapter *adap)
 {
 	int chan;
 	u32 v;
+	struct tp_params *tpp = &adap->params.tp;
 
 	v = t4_read_reg(adap, A_TP_TIMER_RESOLUTION);
-	adap->params.tp.tre = G_TIMERRESOLUTION(v);
-	adap->params.tp.dack_re = G_DELAYEDACKRESOLUTION(v);
+	tpp->tre = G_TIMERRESOLUTION(v);
+	tpp->dack_re = G_DELAYEDACKRESOLUTION(v);
 
 	/* MODQ_REQ_MAP defaults to setting queues 0-3 to chan 0-3 */
 	for (chan = 0; chan < MAX_NCHAN; chan++)
-		adap->params.tp.tx_modq[chan] = chan;
+		tpp->tx_modq[chan] = chan;
 
-	t4_read_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA,
-			 &adap->params.tp.ingress_config, 1,
-			 A_TP_INGRESS_CONFIG);
-	refresh_vlan_pri_map(adap);
+	read_filter_mode_and_ingress_config(adap);
+
+	/*
+	 * For T6, cache the adapter's compressed error vector
+	 * and passing outer header info for encapsulated packets.
+	 */
+	if (chip_id(adap) > CHELSIO_T5) {
+		v = t4_read_reg(adap, A_TP_OUT_CONFIG);
+		tpp->rx_pkt_encap = (v & F_CRXPKTENC) ? 1 : 0;
+	}
 
 	return 0;
 }
 
 /**
- *	t4_filter_field_shift - calculate filter field shift
- *	@adap: the adapter
- *	@filter_sel: the desired field (from TP_VLAN_PRI_MAP bits)
+ *      t4_filter_field_shift - calculate filter field shift
+ *      @adap: the adapter
+ *      @filter_sel: the desired field (from TP_VLAN_PRI_MAP bits)
  *
- *	Return the shift position of a filter field within the Compressed
- *	Filter Tuple.  The filter field is specified via its selection bit
- *	within TP_VLAN_PRI_MAL (filter mode).  E.g. F_VLAN.
+ *      Return the shift position of a filter field within the Compressed
+ *      Filter Tuple.  The filter field is specified via its selection bit
+ *      within TP_VLAN_PRI_MAL (filter mode).  E.g. F_VLAN.
  */
 int t4_filter_field_shift(const struct adapter *adap, int filter_sel)
 {
@@ -7746,18 +7851,38 @@ int t4_filter_field_shift(const struct adapter *adap, int filter_sel)
 		return -1;
 
 	for (sel = 1, field_shift = 0; sel < filter_sel; sel <<= 1) {
-	    switch (filter_mode & sel) {
-		case F_FCOE:          field_shift += W_FT_FCOE;          break;
-		case F_PORT:          field_shift += W_FT_PORT;          break;
-		case F_VNIC_ID:       field_shift += W_FT_VNIC_ID;       break;
-		case F_VLAN:          field_shift += W_FT_VLAN;          break;
-		case F_TOS:           field_shift += W_FT_TOS;           break;
-		case F_PROTOCOL:      field_shift += W_FT_PROTOCOL;      break;
-		case F_ETHERTYPE:     field_shift += W_FT_ETHERTYPE;     break;
-		case F_MACMATCH:      field_shift += W_FT_MACMATCH;      break;
-		case F_MPSHITTYPE:    field_shift += W_FT_MPSHITTYPE;    break;
-		case F_FRAGMENTATION: field_shift += W_FT_FRAGMENTATION; break;
-	    }
+		switch (filter_mode & sel) {
+		case F_FCOE:
+			field_shift += W_FT_FCOE;
+			break;
+		case F_PORT:
+			field_shift += W_FT_PORT;
+			break;
+		case F_VNIC_ID:
+			field_shift += W_FT_VNIC_ID;
+			break;
+		case F_VLAN:
+			field_shift += W_FT_VLAN;
+			break;
+		case F_TOS:
+			field_shift += W_FT_TOS;
+			break;
+		case F_PROTOCOL:
+			field_shift += W_FT_PROTOCOL;
+			break;
+		case F_ETHERTYPE:
+			field_shift += W_FT_ETHERTYPE;
+			break;
+		case F_MACMATCH:
+			field_shift += W_FT_MACMATCH;
+			break;
+		case F_MPSHITTYPE:
+			field_shift += W_FT_MPSHITTYPE;
+			break;
+		case F_FRAGMENTATION:
+			field_shift += W_FT_FRAGMENTATION;
+			break;
+		}
 	}
 	return field_shift;
 }
@@ -7822,6 +7947,37 @@ int __devinit t4_port_init(struct port_info *p, int mbox, int pf, int vf)
 	return 0;
 }
 
+/**
+ *	t4_set_filter_mode - configure the optional components of filter tuples
+ *	@adap: the adapter
+ *	@mode_map: a bitmap selcting which optional filter components to enable
+ *
+ *	Sets the filter mode by selecting the optional components to enable
+ *	in filter tuples.  Returns 0 on success and a negative error if the
+ *	requested mode needs more bits than are available for optional
+ *	components.
+ */
+int t4_set_filter_mode(struct adapter *adap, unsigned int mode_map)
+{
+	static u8 width[] = { 1, 3, 17, 17, 8, 8, 16, 9, 3, 1 };
+
+	int i, nbits = 0;
+
+	for (i = S_FCOE; i <= S_FRAGMENTATION; i++)
+		if (mode_map & (1 << i))
+			nbits += width[i];
+	if (nbits > FILTER_OPT_LEN)
+		return -EINVAL;
+	if (t4_use_ldst(adap))
+		t4_fw_tp_pio_rw(adap, &mode_map, 1, A_TP_VLAN_PRI_MAP, 0);
+	else
+		t4_write_indirect(adap, A_TP_PIO_ADDR, A_TP_PIO_DATA, &mode_map,
+				  1, A_TP_VLAN_PRI_MAP);
+	read_filter_mode_and_ingress_config(adap);
+
+	return 0;
+}
+
 int t4_sched_config(struct adapter *adapter, int type, int minmaxen,
     		    int sleep_ok)
 {
diff --git a/sys/dev/cxgbe/t4_ioctl.h b/sys/dev/cxgbe/t4_ioctl.h
index 0d6dec5..473cf89 100644
--- a/sys/dev/cxgbe/t4_ioctl.h
+++ b/sys/dev/cxgbe/t4_ioctl.h
@@ -105,6 +105,12 @@ struct t4_i2c_data {
 #define T4_FILTER_MPS_HIT_TYPE	0x4000	/* MPS match type */
 #define T4_FILTER_IP_FRAGMENT	0x8000	/* IP fragment */
 
+#define T4_FILTER_IC_VNIC	0x80000000	/* TP Ingress Config's F_VNIC
+						   bit.  It indicates whether
+						   T4_FILTER_VNIC bit means VNIC
+						   id (PF/VF) or outer VLAN.
+						   0 = oVLAN, 1 = VNIC */
+
 /* Filter action */
 enum {
 	FILTER_PASS = 0,	/* default */
@@ -154,7 +160,7 @@ struct t4_filter_tuple {
 	 * is used to select the global mode and all filters are limited to the
 	 * set of fields allowed by the global mode.
 	 */
-	uint16_t vnic;		/* VNIC id or outer VLAN tag */
+	uint16_t vnic;		/* VNIC id (PF/VF) or outer VLAN tag */
 	uint16_t vlan;		/* VLAN tag */
 	uint16_t ethtype;	/* Ethernet type */
 	uint8_t  tos;		/* TOS/Traffic Type */
@@ -165,7 +171,8 @@ struct t4_filter_tuple {
 	uint32_t frag:1;	/* fragmentation extension header */
 	uint32_t macidx:9;	/* exact match MAC index */
 	uint32_t vlan_vld:1;	/* VLAN valid */
-	uint32_t vnic_vld:1;	/* VNIC id/outer VLAN tag valid */
+	uint32_t ovlan_vld:1;	/* outer VLAN tag valid, value in "vnic" */
+	uint32_t pfvf_vld:1;	/* VNIC id (PF/VF) valid, value in "vnic" */
 };
 
 struct t4_filter_specification {
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index da81228..310dc5e 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -476,9 +476,11 @@ static int sysctl_tx_rate(SYSCTL_HANDLER_ARGS);
 static int sysctl_ulprx_la(SYSCTL_HANDLER_ARGS);
 static int sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS);
 #endif
-static uint32_t fconf_to_mode(uint32_t);
+static uint32_t fconf_iconf_to_mode(uint32_t, uint32_t);
 static uint32_t mode_to_fconf(uint32_t);
-static uint32_t fspec_to_fconf(struct t4_filter_specification *);
+static uint32_t mode_to_iconf(uint32_t);
+static int check_fspec_against_fconf_iconf(struct adapter *,
+    struct t4_filter_specification *);
 static int get_filter_mode(struct adapter *, uint32_t *);
 static int set_filter_mode(struct adapter *, uint32_t);
 static inline uint64_t get_filter_hits(struct adapter *, uint32_t);
@@ -3917,7 +3919,7 @@ vi_full_init(struct vi_info *vi)
 	for (i = 0; i < nitems(rss_key); i++) {
 		rss_key[i] = htobe32(raw_rss_key[nitems(rss_key) - 1 - i]);
 	}
-	t4_write_rss_key(sc, (void *)&rss_key[0], -1);
+	t4_write_rss_key(sc, &rss_key[0], -1);
 #endif
 	rss = malloc(vi->rss_size * sizeof (*rss), M_CXGBE, M_ZERO | M_WAITOK);
 	for (i = 0; i < vi->rss_size;) {
@@ -7210,7 +7212,7 @@ sysctl_wcwr_stats(SYSCTL_HANDLER_ARGS)
 #endif
 
 static uint32_t
-fconf_to_mode(uint32_t fconf)
+fconf_iconf_to_mode(uint32_t fconf, uint32_t iconf)
 {
 	uint32_t mode;
 
@@ -7238,8 +7240,11 @@ fconf_to_mode(uint32_t fconf)
 	if (fconf & F_VLAN)
 		mode |= T4_FILTER_VLAN;
 
-	if (fconf & F_VNIC_ID)
+	if (fconf & F_VNIC_ID) {
 		mode |= T4_FILTER_VNIC;
+		if (iconf & F_VNIC)
+			mode |= T4_FILTER_IC_VNIC;
+	}
 
 	if (fconf & F_PORT)
 		mode |= T4_FILTER_PORT;
@@ -7289,8 +7294,18 @@ mode_to_fconf(uint32_t mode)
 }
 
 static uint32_t
-fspec_to_fconf(struct t4_filter_specification *fs)
+mode_to_iconf(uint32_t mode)
+{
+
+	if (mode & T4_FILTER_IC_VNIC)
+		return (F_VNIC);
+	return (0);
+}
+
+static int check_fspec_against_fconf_iconf(struct adapter *sc,
+    struct t4_filter_specification *fs)
 {
+	struct tp_params *tpp = &sc->params.tp;
 	uint32_t fconf = 0;
 
 	if (fs->val.frag || fs->mask.frag)
@@ -7314,8 +7329,17 @@ fspec_to_fconf(struct t4_filter_specification *fs)
 	if (fs->val.vlan_vld || fs->mask.vlan_vld)
 		fconf |= F_VLAN;
 
-	if (fs->val.vnic_vld || fs->mask.vnic_vld)
+	if (fs->val.ovlan_vld || fs->mask.ovlan_vld) {
 		fconf |= F_VNIC_ID;
+		if (tpp->ingress_config & F_VNIC)
+			return (EINVAL);
+	}
+
+	if (fs->val.pfvf_vld || fs->mask.pfvf_vld) {
+		fconf |= F_VNIC_ID;
+		if ((tpp->ingress_config & F_VNIC) == 0)
+			return (EINVAL);
+	}
 
 	if (fs->val.iport || fs->mask.iport)
 		fconf |= F_PORT;
@@ -7323,41 +7347,45 @@ fspec_to_fconf(struct t4_filter_specification *fs)
 	if (fs->val.fcoe || fs->mask.fcoe)
 		fconf |= F_FCOE;
 
-	return (fconf);
+	if ((tpp->vlan_pri_map | fconf) != tpp->vlan_pri_map)
+		return (E2BIG);
+
+	return (0);
 }
 
 static int
 get_filter_mode(struct adapter *sc, uint32_t *mode)
 {
-	int rc;
-	uint32_t fconf;
+	struct tp_params *tpp = &sc->params.tp;
 
-	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
-	    "t4getfm");
-	if (rc)
-		return (rc);
-
-	t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &fconf, 1,
-	    A_TP_VLAN_PRI_MAP);
-
-	if (sc->params.tp.vlan_pri_map != fconf) {
-		log(LOG_WARNING, "%s: cached filter mode out of sync %x %x.\n",
-		    device_get_nameunit(sc->dev), sc->params.tp.vlan_pri_map,
-		    fconf);
-	}
-
-	*mode = fconf_to_mode(fconf);
+	/*
+	 * We trust the cached values of the relevant TP registers.  This means
+	 * things work reliably only if writes to those registers are always via
+	 * t4_set_filter_mode.
+	 */
+	*mode = fconf_iconf_to_mode(tpp->vlan_pri_map, tpp->ingress_config);
 
-	end_synchronized_op(sc, LOCK_HELD);
 	return (0);
 }
 
 static int
 set_filter_mode(struct adapter *sc, uint32_t mode)
 {
-	uint32_t fconf;
+	struct tp_params *tpp = &sc->params.tp;
+	uint32_t fconf, iconf;
 	int rc;
 
+	iconf = mode_to_iconf(mode);
+	if ((iconf ^ tpp->ingress_config) & F_VNIC) {
+		/*
+		 * For now we just complain if A_TP_INGRESS_CONFIG is not
+		 * already set to the correct value for the requested filter
+		 * mode.  It's not clear if it's safe to write to this register
+		 * on the fly.  (And we trust the cached value of the register).
+		 */
+		return (EBUSY);
+	}
+
 	fconf = mode_to_fconf(mode);
 
 	rc = begin_synchronized_op(sc, NULL, HOLD_LOCK | SLEEP_OK | INTR_OK,
@@ -7390,6 +7418,7 @@ get_filter_hits(struct adapter *sc, uint32_t fid)
 	uint64_t hits;
 
 	memwin_info(sc, 0, &mw_base, NULL);
+
 	off = position_memwin(sc, 0,
 	    tcb_base + (fid + sc->tids.ftid_base) * TCB_SIZE);
 	if (is_t4(sc)) {
@@ -7471,12 +7500,10 @@ set_filter(struct adapter *sc, struct t4_filter *t)
 		goto done;
 	}
 
-	/* Validate against the global filter mode */
-	if ((sc->params.tp.vlan_pri_map | fspec_to_fconf(&t->fs)) !=
-	    sc->params.tp.vlan_pri_map) {
-		rc = E2BIG;
+	/* Validate against the global filter mode and ingress config */
+	rc = check_fspec_against_fconf_iconf(sc, &t->fs);
+	if (rc != 0)
 		goto done;
-	}
 
 	if (t->fs.action == FILTER_SWITCH && t->fs.eport >= nports) {
 		rc = EINVAL;
@@ -7639,7 +7666,7 @@ set_filter_wr(struct adapter *sc, int fidx)
 {
 	struct filter_entry *f = &sc->tids.ftid_tab[fidx];
 	struct fw_filter_wr *fwr;
-	unsigned int ftid;
+	unsigned int ftid, vnic_vld, vnic_vld_mask;
 	struct wrq_cookie cookie;
 
 	ASSERT_SYNCHRONIZED_OP(sc);
@@ -7657,6 +7684,18 @@ set_filter_wr(struct adapter *sc, int fidx)
 		}
 	}
 
+	/* Already validated against fconf, iconf */
+	MPASS((f->fs.val.pfvf_vld & f->fs.val.ovlan_vld) == 0);
+	MPASS((f->fs.mask.pfvf_vld & f->fs.mask.ovlan_vld) == 0);
+	if (f->fs.val.pfvf_vld || f->fs.val.ovlan_vld)
+		vnic_vld = 1;
+	else
+		vnic_vld = 0;
+	if (f->fs.mask.pfvf_vld || f->fs.mask.ovlan_vld)
+		vnic_vld_mask = 1;
+	else
+		vnic_vld_mask = 0;
+
 	ftid = sc->tids.ftid_base + fidx;
 
 	fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
@@ -7694,9 +7733,9 @@ set_filter_wr(struct adapter *sc, int fidx)
 	    (V_FW_FILTER_WR_FRAG(f->fs.val.frag) |
 		V_FW_FILTER_WR_FRAGM(f->fs.mask.frag) |
 		V_FW_FILTER_WR_IVLAN_VLD(f->fs.val.vlan_vld) |
-		V_FW_FILTER_WR_OVLAN_VLD(f->fs.val.vnic_vld) |
+		V_FW_FILTER_WR_OVLAN_VLD(vnic_vld) |
 		V_FW_FILTER_WR_IVLAN_VLDM(f->fs.mask.vlan_vld) |
-		V_FW_FILTER_WR_OVLAN_VLDM(f->fs.mask.vnic_vld));
+		V_FW_FILTER_WR_OVLAN_VLDM(vnic_vld_mask));
 	fwr->smac_sel = 0;
 	fwr->rx_chan_rx_rpl_iq = htobe16(V_FW_FILTER_WR_RX_CHAN(0) |
 	    V_FW_FILTER_WR_RX_RPL_IQ(sc->sge.fwq.abs_id));
diff --git a/tools/tools/cxgbetool/cxgbetool.c b/tools/tools/cxgbetool/cxgbetool.c
index c4d996e..4f78802 100644
--- a/tools/tools/cxgbetool/cxgbetool.c
+++ b/tools/tools/cxgbetool/cxgbetool.c
@@ -532,7 +532,10 @@ do_show_info_header(uint32_t mode)
 			break;
 
 		case T4_FILTER_VNIC:
-			printf("      vld:VNIC");
+			if (mode & T4_FILTER_IC_VNIC)
+				printf("   VFvld:PF:VF");
+			else
+				printf("     vld:oVLAN");
 			break;
 
 		case T4_FILTER_VLAN:
@@ -789,11 +792,19 @@ do_show_one_filter_info(struct t4_filter *t, uint32_t mode)
 			break;
 
 		case T4_FILTER_VNIC:
-			printf(" %1d:%1x:%02x/%1d:%1x:%02x",
-			    t->fs.val.vnic_vld, (t->fs.val.vnic >> 7) & 0x7,
-			    t->fs.val.vnic & 0x7f, t->fs.mask.vnic_vld,
-			    (t->fs.mask.vnic >> 7) & 0x7,
-			    t->fs.mask.vnic & 0x7f);
+			if (mode & T4_FILTER_IC_VNIC) {
+				printf(" %1d:%1x:%02x/%1d:%1x:%02x",
+				    t->fs.val.pfvf_vld,
+				    (t->fs.val.vnic >> 13) & 0x7,
+				    t->fs.val.vnic & 0x1fff,
+				    t->fs.mask.pfvf_vld,
+				    (t->fs.mask.vnic >> 13) & 0x7,
+				    t->fs.mask.vnic & 0x1fff);
+			} else {
+				printf(" %1d:%04x/%1d:%04x",
+				    t->fs.val.ovlan_vld, t->fs.val.vnic,
+				    t->fs.mask.ovlan_vld, t->fs.mask.vnic);
+			}
 			break;
 
 		case T4_FILTER_VLAN:
@@ -971,8 +982,12 @@ get_filter_mode(void)
 	if (mode & T4_FILTER_VLAN)
 		printf("vlan ");
 
-	if (mode & T4_FILTER_VNIC)
-		printf("vnic/ovlan ");
+	if (mode & T4_FILTER_VNIC) {
+		if (mode & T4_FILTER_IC_VNIC)
+			printf("vnic_id ");
+		else
+			printf("ovlan ");
+	}
 
 	if (mode & T4_FILTER_PORT)
 		printf("iport ");
@@ -989,6 +1004,7 @@ static int
 set_filter_mode(int argc, const char *argv[])
 {
 	uint32_t mode = 0;
+	int vnic = 0, ovlan = 0;
 
 	for (; argc; argc--, argv++) {
 		if (!strcmp(argv[0], "frag"))
@@ -1012,9 +1028,16 @@ set_filter_mode(int argc, const char *argv[])
 		if (!strcmp(argv[0], "vlan"))
 			mode |= T4_FILTER_VLAN;
 
-		if (!strcmp(argv[0], "ovlan") ||
-		    !strcmp(argv[0], "vnic"))
+		if (!strcmp(argv[0], "ovlan")) {
+			mode |= T4_FILTER_VNIC;
+			ovlan++;
+		}
+
+		if (!strcmp(argv[0], "vnic_id")) {
 			mode |= T4_FILTER_VNIC;
+			mode |= T4_FILTER_IC_VNIC;
+			vnic++;
+		}
 
 		if (!strcmp(argv[0], "iport"))
 			mode |= T4_FILTER_PORT;
@@ -1023,6 +1046,11 @@ set_filter_mode(int argc, const char *argv[])
 			mode |= T4_FILTER_FCoE;
 	}
 
+	if (vnic > 0 && ovlan > 0) {
+		warnx("\"vnic_id\" and \"ovlan\" are mutually exclusive.");
+		return (EINVAL);
+	}
+
 	return doit(CHELSIO_T4_SET_FILTER_MODE, &mode);
 }
 
@@ -1081,18 +1109,27 @@ set_filter(uint32_t idx, int argc, const char *argv[])
 		} else if (!parse_val_mask("ovlan", args, &val, &mask)) {
 			t.fs.val.vnic = val;
 			t.fs.mask.vnic = mask;
-			t.fs.val.vnic_vld = 1;
-			t.fs.mask.vnic_vld = 1;
-		} else if (!parse_val_mask("vnic", args, &val, &mask)) {
-			t.fs.val.vnic = val;
-			t.fs.mask.vnic = mask;
-			t.fs.val.vnic_vld = 1;
-			t.fs.mask.vnic_vld = 1;
+			t.fs.val.ovlan_vld = 1;
+			t.fs.mask.ovlan_vld = 1;
 		} else if (!parse_val_mask("ivlan", args, &val, &mask)) {
 			t.fs.val.vlan = val;
 			t.fs.mask.vlan = mask;
 			t.fs.val.vlan_vld = 1;
 			t.fs.mask.vlan_vld = 1;
+		} else if (!parse_val_mask("pf", args, &val, &mask)) {
+			t.fs.val.vnic &= 0x1fff;
+			t.fs.val.vnic |= (val & 0x7) << 13;
+			t.fs.mask.vnic &= 0x1fff;
+			t.fs.mask.vnic |= (mask & 0x7) << 13;
+			t.fs.val.pfvf_vld = 1;
+			t.fs.mask.pfvf_vld = 1;
+		} else if (!parse_val_mask("vf", args, &val, &mask)) {
+			t.fs.val.vnic &= 0xe000;
+			t.fs.val.vnic |= val & 0x1fff;
+			t.fs.mask.vnic &= 0xe000;
+			t.fs.mask.vnic |= mask & 0x1fff;
+			t.fs.val.pfvf_vld = 1;
+			t.fs.mask.pfvf_vld = 1;
 		} else if (!parse_val_mask("tos", args, &val, &mask)) {
 			t.fs.val.tos = val;
 			t.fs.mask.tos = mask;
@@ -1228,6 +1265,10 @@ set_filter(uint32_t idx, int argc, const char *argv[])
 		     " action \"drop\" or \"switch\"");
 		return (EINVAL);
 	}
+	if (t.fs.val.ovlan_vld && t.fs.val.pfvf_vld) {
+		warnx("ovlan and vnic_id (pf/vf) are mutually exclusive");
+		return (EINVAL);
+	}
 
 	t.fs.type = (af == AF_INET6 ? 1 : 0); /* default IPv4 */
 	return doit(CHELSIO_T4_SET_FILTER, &t);