Patches from Naresh Raju Gottumukkala

- Feature: UMC - Universal Multi Channel support
- Bugfix: BE3 Firmware Flashing bug.
- Code improvements:
  - Removed duplicate switch cases in the oce_ioctl routine.
  - Made changes to mcc_async notifications routine(oce_mq_handler)

MFC after:	1 week

8 files changed, 260 insertions, 147 deletions

diff --git a/sys/dev/oce/oce_hw.c b/sys/dev/oce/oce_hw.c
index 05486d2..d279e5e 100644
--- a/sys/dev/oce/oce_hw.c
+++ b/sys/dev/oce/oce_hw.c
@@ -340,8 +340,10 @@ oce_hw_shutdown(POCE_SOFTC sc)
 	oce_stats_free(sc);
 	/* disable hardware interrupts */
 	oce_hw_intr_disable(sc);
+#if defined(INET6) || defined(INET)
 	/* Free LRO resources */
 	oce_free_lro(sc);
+#endif
 	/* Release queue*/
 	oce_queue_release_all(sc);
 	/*Delete Network Interface*/
@@ -494,12 +496,17 @@ oce_hw_start(POCE_SOFTC sc)
 
 	rc = oce_start_mq(sc->mq);
 	
-	/* we need to get MCC aync events.
-	   So enable intrs and also arm first EQ
+	/* we need to get MCC aync events. So enable intrs and arm
+	   first EQ, Other EQs will be armed after interface is UP 
 	*/
 	oce_hw_intr_enable(sc);
 	oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
 
+	/* Send first mcc cmd and after that we get gracious
+	   MCC notifications from FW
+	*/
+	oce_first_mcc_cmd(sc);
+
 	return rc;
 }
 
@@ -537,8 +544,8 @@ oce_hw_intr_disable(POCE_SOFTC sc)
 
 
 /**
- * @brief               Function for hardware update multicast filter
- * @param sc            software handle to the device
+ * @brief		Function for hardware update multicast filter
+ * @param sc		software handle to the device
  */
 int
 oce_hw_update_multicast(POCE_SOFTC sc)
diff --git a/sys/dev/oce/oce_hw.h b/sys/dev/oce/oce_hw.h
index 6aee9fa..0e79218 100644
--- a/sys/dev/oce/oce_hw.h
+++ b/sys/dev/oce/oce_hw.h
@@ -154,6 +154,9 @@
 #define	ASYNC_EVENT_CODE_LINK_STATE	0x1
 #define	ASYNC_EVENT_LINK_UP		0x1
 #define	ASYNC_EVENT_LINK_DOWN		0x0
+#define ASYNC_EVENT_GRP5		0x5
+#define ASYNC_EVENT_PVID_STATE		0x3
+#define VLAN_VID_MASK			0x0FFF
 
 /* port link_status */
 #define	ASYNC_EVENT_LOGICAL		0x02
@@ -610,7 +613,10 @@ struct oce_mq_cqe {
 			uint32_t hpi_buffer_cmpl:1;
 			uint32_t completed:1;
 			uint32_t consumed:1;
-			uint32_t rsvd0:27;
+			uint32_t rsvd0:3;
+			uint32_t async_type:8;
+			uint32_t event_type:8;
+			uint32_t rsvd1:8;
 #else
 			/* dw0 */
 			uint32_t completion_status:16;
@@ -618,7 +624,10 @@ struct oce_mq_cqe {
 			/* dw1 dw2 */
 			uint32_t mq_tag[2];
 			/* dw3 */
-			uint32_t rsvd0:27;
+			uint32_t rsvd1:8;
+			uint32_t event_type:8;
+			uint32_t async_type:8;
+			uint32_t rsvd0:3;
 			uint32_t consumed:1;
 			uint32_t completed:1;
 			uint32_t hpi_buffer_cmpl:1;
@@ -687,6 +696,63 @@ struct oce_async_cqe_link_state {
 	} u0;
 };
 
+
+/* PVID aync event */
+struct oce_async_event_grp5_pvid_state {
+	uint8_t enabled;
+	uint8_t rsvd0;
+	uint16_t tag;
+	uint32_t event_tag;
+	uint32_t rsvd1;
+	uint32_t code;
+};
+
+typedef union oce_mq_ext_ctx_u {
+	uint32_t dw[6];
+	struct {
+		#ifdef _BIG_ENDIAN
+		/* dw0 */
+		uint32_t dw4rsvd1:16;
+		uint32_t num_pages:16;
+		/* dw1 */
+		uint32_t async_evt_bitmap;
+		/* dw2 */
+		uint32_t cq_id:10;
+		uint32_t dw5rsvd2:2;
+		uint32_t ring_size:4;
+		uint32_t dw5rsvd1:16;
+		/* dw3 */
+		uint32_t valid:1;
+		uint32_t dw6rsvd1:31;
+		/* dw4 */
+		uint32_t dw7rsvd1:21;
+		uint32_t async_cq_id:10;
+		uint32_t async_cq_valid:1;
+	#else
+		/* dw0 */
+		uint32_t num_pages:16;
+		uint32_t dw4rsvd1:16;
+		/* dw1 */
+		uint32_t async_evt_bitmap;
+		/* dw2 */
+		uint32_t dw5rsvd1:16;
+		uint32_t ring_size:4;
+		uint32_t dw5rsvd2:2;
+		uint32_t cq_id:10;
+		/* dw3 */
+		uint32_t dw6rsvd1:31;
+		uint32_t valid:1;
+		/* dw4 */
+		uint32_t async_cq_valid:1;
+		uint32_t async_cq_id:10;
+		uint32_t dw7rsvd1:21;
+	#endif
+		/* dw5 */
+		uint32_t dw8rsvd1;
+	} v0;
+} oce_mq_ext_ctx_t;
+
+
 /* MQ mailbox structure */
 struct oce_bmbx {
 	struct oce_mbx mbx;
@@ -1342,6 +1408,23 @@ struct mbx_create_common_mq {
 	} params;
 };
 
+struct mbx_create_common_mq_ex {
+	struct mbx_hdr hdr;
+	union {
+		struct {
+			oce_mq_ext_ctx_t context;
+			struct phys_addr pages[8];
+		} req;
+
+		struct {
+			uint32_t mq_id:16;
+			uint32_t rsvd0:16;
+		} rsp;
+	} params;
+};
+
+
+
 /* [53] OPCODE_COMMON_DESTROY_MQ */
 struct mbx_destroy_common_mq {
 	struct mbx_hdr hdr;
@@ -1584,7 +1667,7 @@ enum CQFW_FUNCTION_MODES_SUPPORTED {
 	FNM_BE3_COMPAT_MODE = 0x10000,	/* BE3 features */
 	FNM_VNIC_MODE = 0x20000,	/* Set when IBM vNIC mode is set */
 	FNM_VNTAG_MODE = 0x40000, 	/* Set when VNTAG mode is set */
-	FNM_UMC_MODE = 0x80000,		/* Set when UMC mode is set */
+	FNM_UMC_MODE = 0x1000000,	/* Set when UMC mode is set */
 	FNM_UMC_DEF_EN = 0x100000,	/* Set when UMC Default is set */
 	FNM_ONE_GB_EN = 0x200000,	/* Set when 1GB Default is set */
 	FNM_VNIC_DEF_VALID = 0x400000,	/* Set when VNIC_DEF_EN is valid */
diff --git a/sys/dev/oce/oce_if.c b/sys/dev/oce/oce_if.c
index 601b1b7..fdeb339 100644
--- a/sys/dev/oce/oce_if.c
+++ b/sys/dev/oce/oce_if.c
@@ -36,7 +36,6 @@
  * Costa Mesa, CA 92626
  */
 
-
 /* $FreeBSD$ */
 
 #include "opt_inet6.h"
@@ -71,10 +70,6 @@ static int  oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
 static void oce_tx_complete(struct oce_wq *wq, uint32_t wqe_idx,
 					uint32_t status);
-#if defined(INET6) || defined(INET)
-static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp,
-					uint16_t *mss);
-#endif
 static int  oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m,
 				 struct oce_wq *wq);
 
@@ -82,9 +77,6 @@ static int  oce_multiq_transmit(struct ifnet *ifp, struct mbuf *m,
 static void oce_discard_rx_comp(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
 static int  oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
 static int  oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
-#if defined(INET6) || defined(INET)
-static void oce_rx_flush_lro(struct oce_rq *rq);
-#endif
 static void oce_rx(struct oce_rq *rq, uint32_t rqe_idx,
 						struct oce_nic_rx_cqe *cqe);
 
@@ -96,13 +88,20 @@ static int  oce_vid_config(POCE_SOFTC sc);
 static void oce_mac_addr_set(POCE_SOFTC sc);
 static int  oce_handle_passthrough(struct ifnet *ifp, caddr_t data);
 static void oce_local_timer(void *arg);
-#if defined(INET6) || defined(INET)
-static int  oce_init_lro(POCE_SOFTC sc);
-#endif
 static void oce_if_deactivate(POCE_SOFTC sc);
 static void oce_if_activate(POCE_SOFTC sc);
 static void setup_max_queues_want(POCE_SOFTC sc);
 static void update_queues_got(POCE_SOFTC sc);
+static void process_link_state(POCE_SOFTC sc,
+		 struct oce_async_cqe_link_state *acqe);
+
+
+/* IP specific */
+#if defined(INET6) || defined(INET)
+static int  oce_init_lro(POCE_SOFTC sc);
+static void oce_rx_flush_lro(struct oce_rq *rq);
+static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
+#endif
 
 static device_method_t oce_dispatch[] = {
 	DEVMETHOD(device_probe, oce_probe),
@@ -157,10 +156,10 @@ static uint32_t supportedDevices[] =  {
 static int
 oce_probe(device_t dev)
 {
-	uint16_t vendor;
-	uint16_t device;
-	int i;
-	char str[80];
+	uint16_t vendor = 0;
+	uint16_t device = 0;
+	int i = 0;
+	char str[256] = {0};
 	POCE_SOFTC sc;
 
 	sc = device_get_softc(dev);
@@ -170,11 +169,10 @@ oce_probe(device_t dev)
 	vendor = pci_get_vendor(dev);
 	device = pci_get_device(dev);
 
-	for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint16_t)); i++) {
+	for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
 		if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
 			if (device == (supportedDevices[i] & 0xffff)) {
-				sprintf(str, "%s:%s",
-					"Emulex CNA NIC function",
+				sprintf(str, "%s:%s", "Emulex CNA NIC function",
 					component_revision);
 				device_set_desc_copy(dev, str);
 
@@ -228,37 +226,30 @@ oce_attach(device_t dev)
 	if (rc)
 		goto pci_res_free;
 
-
 	setup_max_queues_want(sc);	
 
-
 	rc = oce_setup_intr(sc);
 	if (rc)
 		goto mbox_free;
 
-
 	rc = oce_queue_init_all(sc);
 	if (rc)
 		goto intr_free;
 
-
 	rc = oce_attach_ifp(sc);
 	if (rc)
 		goto queues_free;
 
-
 #if defined(INET6) || defined(INET)
 	rc = oce_init_lro(sc);
 	if (rc)
-		goto ifp_free;	
+		goto ifp_free;
 #endif
 
-
 	rc = oce_hw_start(sc);
 	if (rc)
 		goto lro_free;;
 
-
 	sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
 				oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
 	sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
@@ -270,11 +261,12 @@ oce_attach(device_t dev)
 
 	oce_add_sysctls(sc);
 
-
 	callout_init(&sc->timer, CALLOUT_MPSAFE);
 	rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
 	if (rc)
 		goto stats_free;
+#ifdef DEV_NETMAP
+#endif /* DEV_NETMAP */
 
 	return 0;
 
@@ -315,9 +307,7 @@ oce_detach(device_t dev)
 	POCE_SOFTC sc = device_get_softc(dev);
 
 	LOCK(&sc->dev_lock);
-	
 	oce_if_deactivate(sc);
-
 	UNLOCK(&sc->dev_lock);
 
 	callout_drain(&sc->timer);
@@ -359,34 +349,11 @@ oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 	uint32_t u;
 
 	switch (command) {
-	case SIOCGIFPSRCADDR_IN6:
-		rc = ether_ioctl(ifp, command, data);
-		break;
-
-	case SIOCGIFPSRCADDR:
-		rc = ether_ioctl(ifp, command, data);
-		break;
-
-	case SIOCGIFSTATUS:
-		rc = ether_ioctl(ifp, command, data);
-		break;
 
 	case SIOCGIFMEDIA:
 		rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
 		break;
 
-	case SIOCSIFMEDIA:
-		rc = ether_ioctl(ifp, command, data);
-		break;
-
-	case SIOCGIFGENERIC:
-		rc = ether_ioctl(ifp, command, data);
-		break;
-
-	case SIOCGETMIFCNT_IN6:
-		rc = ether_ioctl(ifp, command, data);
-		break;
-
 	case SIOCSIFMTU:
 		if (ifr->ifr_mtu > OCE_MAX_MTU)
 			rc = EINVAL;
@@ -474,7 +441,6 @@ oce_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
 			oce_vid_config(sc);
 		}
-
 #if defined(INET6) || defined(INET)
 		if (u & IFCAP_LRO)
 			ifp->if_capenable ^= IFCAP_LRO;
@@ -813,9 +779,6 @@ oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
 	struct oce_nic_frag_wqe *nicfrag;
 	int num_wqes;
 	uint32_t reg_value;
-#if defined(INET6) || defined(INET)
-	uint16_t mss = 0;
-#endif
 
 	m = *mpp;
 	if (!m)
@@ -827,9 +790,9 @@ oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
 	}
 
 	if (m->m_pkthdr.csum_flags & CSUM_TSO) {
-#if defined(INET6) || defined(INET)
 		/* consolidate packet buffers for TSO/LSO segment offload */
-		m = oce_tso_setup(sc, mpp, &mss);
+#if defined(INET6) || defined(INET)
+		m = oce_tso_setup(sc, mpp);
 #else
 		m = NULL;
 #endif
@@ -1012,9 +975,10 @@ oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
 
 }
 
+
 #if defined(INET6) || defined(INET)
 static struct mbuf *
-oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp, uint16_t *mss)
+oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
 {
 	struct mbuf *m;
 #ifdef INET
@@ -1025,12 +989,10 @@ oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp, uint16_t *mss)
 #endif
 	struct ether_vlan_header *eh;
 	struct tcphdr *th;
-	int total_len = 0;
 	uint16_t etype;
-	int ehdrlen = 0;
+	int total_len = 0, ehdrlen = 0;
 	
 	m = *mpp;
-	*mss = m->m_pkthdr.tso_segsz;
 
 	if (M_WRITABLE(m) == 0) {
 		m = m_dup(*mpp, M_DONTWAIT);
@@ -1049,7 +1011,6 @@ oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp, uint16_t *mss)
 		ehdrlen = ETHER_HDR_LEN;
 	}
 
-	
 	switch (etype) {
 #ifdef INET
 	case ETHERTYPE_IP:
@@ -1084,7 +1045,6 @@ oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp, uint16_t *mss)
 }
 #endif /* INET6 || INET */
 
-
 void
 oce_tx_task(void *arg, int npending)
 {
@@ -1115,6 +1075,7 @@ oce_start(struct ifnet *ifp)
 	POCE_SOFTC sc = ifp->if_softc;
 	struct mbuf *m;
 	int rc = 0;
+	int def_q = 0; /* Defualt tx queue is 0*/
 
 	if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
 			IFF_DRV_RUNNING)
@@ -1124,13 +1085,13 @@ oce_start(struct ifnet *ifp)
 		IF_DEQUEUE(&sc->ifp->if_snd, m);
 		if (m == NULL)
 			break;
-		/* oce_start always uses default TX queue 0 */
-		LOCK(&sc->wq[0]->tx_lock);
-		rc = oce_tx(sc, &m, 0);
-		UNLOCK(&sc->wq[0]->tx_lock);
+
+		LOCK(&sc->wq[def_q]->tx_lock);
+		rc = oce_tx(sc, &m, def_q);
+		UNLOCK(&sc->wq[def_q]->tx_lock);
 		if (rc) {
 			if (m != NULL) {
-				sc->wq[0]->tx_stats.tx_stops ++;
+				sc->wq[def_q]->tx_stats.tx_stops ++;
 				ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 				IFQ_DRV_PREPEND(&ifp->if_snd, m);
 				m = NULL;
@@ -1140,7 +1101,7 @@ oce_start(struct ifnet *ifp)
 		if (m != NULL)
 			ETHER_BPF_MTAP(ifp, m);
 
-	} while (1);
+	} while (TRUE);
 
 	return;
 }
@@ -1248,13 +1209,19 @@ oce_rx(struct oce_rq *rq, uint32_t rqe_idx, struct oce_nic_rx_cqe *cqe)
 	uint16_t vtag;
 
 	len = cqe->u0.s.pkt_size;
-	vtag = cqe->u0.s.vlan_tag;
 	if (!len) {
 		/*partial DMA workaround for Lancer*/
 		oce_discard_rx_comp(rq, cqe);
 		goto exit;
 	}
 
+	 /* Get vlan_tag value */
+	if(IS_BE(sc))
+		vtag = BSWAP_16(cqe->u0.s.vlan_tag);
+	else
+		vtag = cqe->u0.s.vlan_tag;
+
+
 	for (i = 0; i < cqe->u0.s.num_fragments; i++) {
 
 		if (rq->packets_out == rq->packets_in) {
@@ -1290,7 +1257,7 @@ oce_rx(struct oce_rq *rq, uint32_t rqe_idx, struct oce_nic_rx_cqe *cqe)
 					pd->mbuf->m_pkthdr.csum_data = 0xffff;
 				}
 				if (cqe->u0.s.ip_cksum_pass) {
-					if (!cqe->u0.s.ip_ver) { //IPV4 
+					if (!cqe->u0.s.ip_ver) { /* IPV4 */
 						pd->mbuf->m_pkthdr.csum_flags |=
 						(CSUM_IP_CHECKED|CSUM_IP_VALID);
 					}
@@ -1313,24 +1280,20 @@ oce_rx(struct oce_rq *rq, uint32_t rqe_idx, struct oce_nic_rx_cqe *cqe)
 		m->m_pkthdr.flowid = rq->queue_index;
 		m->m_flags |= M_FLOWID;
 #endif
-		//This deternies if vlan tag is present
+		/* This deternies if vlan tag is Valid */
 		if (oce_cqe_vtp_valid(sc, cqe)) { 
 			if (sc->function_mode & FNM_FLEX10_MODE) {
-				/* FLEX10 */
+				/* FLEX10. If QnQ is not set, neglect VLAN */
 				if (cqe->u0.s.qnq) {
-					/* If QnQ is not set, neglect VLAN */
-					if (IS_BE(sc))	
-						m->m_pkthdr.ether_vtag = 
-								BSWAP_16(vtag);
-					else
-						m->m_pkthdr.ether_vtag = vtag;
+					m->m_pkthdr.ether_vtag = vtag;
 					m->m_flags |= M_VLANTAG;
 				}
-			} else {
-				if (IS_BE(sc))	
-					m->m_pkthdr.ether_vtag = BSWAP_16(vtag);
-				else
-					m->m_pkthdr.ether_vtag = vtag;
+			} else if (sc->pvid != (vtag & VLAN_VID_MASK))  {
+				/* In UMC mode generally pvid will be striped by
+				   hw. But in some cases we have seen it comes
+				   with pvid. So if pvid == vlan, neglect vlan.
+				*/
+				m->m_pkthdr.ether_vtag = vtag;
 				m->m_flags |= M_VLANTAG;
 			}
 		}
@@ -1415,9 +1378,8 @@ oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
 	if (sc->be3_native) {
 		cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
 		vtp =  cqe_v1->u0.s.vlan_tag_present; 
-	} else {
+	} else
 		vtp = cqe->u0.s.vlan_tag_present;
-	}
 	
 	return vtp;
 
@@ -1442,7 +1404,6 @@ oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
 
 }
 
-
 #if defined(INET6) || defined(INET)
 static void
 oce_rx_flush_lro(struct oce_rq *rq)
@@ -1482,12 +1443,11 @@ oce_init_lro(POCE_SOFTC sc)
 
 	return rc;		
 }
-#endif /* INET6 || INET */
+
 
 void
 oce_free_lro(POCE_SOFTC sc)
 {
-#if defined(INET6) || defined(INET)
 	struct lro_ctrl *lro = NULL;
 	int i = 0;
 
@@ -1496,9 +1456,8 @@ oce_free_lro(POCE_SOFTC sc)
 		if (lro)
 			tcp_lro_free(lro);
 	}
-#endif
 }
-
+#endif /* INET6 || INET */
 
 int
 oce_alloc_rx_bufs(struct oce_rq *rq, int count)
@@ -1620,6 +1579,7 @@ oce_rq_handler(void *arg)
 		if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
 			break;
 	}
+
 #if defined(INET6) || defined(INET)
 	if (IF_LRO_ENABLED(sc))
 		oce_rx_flush_lro(rq);
@@ -1682,9 +1642,11 @@ oce_attach_ifp(POCE_SOFTC sc)
 	sc->ifp->if_capabilities = OCE_IF_CAPABILITIES;
 	sc->ifp->if_capabilities |= IFCAP_HWCSUM;
 	sc->ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
+
 #if defined(INET6) || defined(INET)
 	sc->ifp->if_capabilities |= IFCAP_TSO;
 	sc->ifp->if_capabilities |= IFCAP_LRO;
+	sc->ifp->if_capabilities |= IFCAP_VLAN_HWTSO;
 #endif
 	
 	sc->ifp->if_capenable = sc->ifp->if_capabilities;
@@ -1944,6 +1906,26 @@ oce_if_activate(POCE_SOFTC sc)
 
 }
 
+static void
+process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
+{
+	/* Update Link status */
+	if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
+	     ASYNC_EVENT_LINK_UP) {
+		sc->link_status = ASYNC_EVENT_LINK_UP;
+		if_link_state_change(sc->ifp, LINK_STATE_UP);
+	} else {
+		sc->link_status = ASYNC_EVENT_LINK_DOWN;
+		if_link_state_change(sc->ifp, LINK_STATE_DOWN);
+	}
+
+	/* Update speed */
+	sc->link_speed = acqe->u0.s.speed;
+	sc->qos_link_speed = (uint32_t) acqe->u0.s.qos_link_speed * 10;
+
+}
+
+
 /* Handle the Completion Queue for the Mailbox/Async notifications */
 uint16_t
 oce_mq_handler(void *arg)
@@ -1951,36 +1933,39 @@ oce_mq_handler(void *arg)
 	struct oce_mq *mq = (struct oce_mq *)arg;
 	POCE_SOFTC sc = mq->parent;
 	struct oce_cq *cq = mq->cq;
-	int num_cqes = 0;
+	int num_cqes = 0, evt_type = 0, optype = 0;
 	struct oce_mq_cqe *cqe;
 	struct oce_async_cqe_link_state *acqe;
+	struct oce_async_event_grp5_pvid_state *gcqe;
+
 
 	bus_dmamap_sync(cq->ring->dma.tag,
 			cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
 	cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
+
 	while (cqe->u0.dw[3]) {
 		DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
 		if (cqe->u0.s.async_event) {
-			acqe = (struct oce_async_cqe_link_state *)cqe;
-			if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
-			    ASYNC_EVENT_LINK_UP) {
-				sc->link_status = ASYNC_EVENT_LINK_UP;
-				if_link_state_change(sc->ifp, LINK_STATE_UP);
-			} else {
-				sc->link_status = ASYNC_EVENT_LINK_DOWN;
-				if_link_state_change(sc->ifp, LINK_STATE_DOWN);
-			}
-
-			if (acqe->u0.s.event_code ==
-				ASYNC_EVENT_CODE_LINK_STATE) {
-				sc->link_speed = acqe->u0.s.speed;
-				sc->qos_link_speed =
-				(uint32_t )acqe->u0.s.qos_link_speed * 10;
+			evt_type = cqe->u0.s.event_type;
+			optype = cqe->u0.s.async_type;
+			if (evt_type  == ASYNC_EVENT_CODE_LINK_STATE) {
+				/* Link status evt */
+				acqe = (struct oce_async_cqe_link_state *)cqe;
+				process_link_state(sc, acqe);
+			} else if ((evt_type == ASYNC_EVENT_GRP5) &&
+				   (optype == ASYNC_EVENT_PVID_STATE)) {
+				/* GRP5 PVID */
+				gcqe = 
+				(struct oce_async_event_grp5_pvid_state *)cqe;
+				if (gcqe->enabled)
+					sc->pvid = gcqe->tag & VLAN_VID_MASK;
+				else
+					sc->pvid = 0;
+				
 			}
 		}
 		cqe->u0.dw[3] = 0;
 		RING_GET(cq->ring, 1);
-		RING_GET(mq->ring, 1);
 		bus_dmamap_sync(cq->ring->dma.tag,
 				cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
 		cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
@@ -2001,8 +1986,10 @@ setup_max_queues_want(POCE_SOFTC sc)
 
 	/* Check if it is FLEX machine. Is so dont use RSS */	
 	if ((sc->function_mode & FNM_FLEX10_MODE) ||
-		(!sc->rss_enable) ||
-		(sc->flags & OCE_FLAGS_BE2)) {
+	    (sc->function_mode & FNM_UMC_MODE)    ||
+	    (sc->function_mode & FNM_VNIC_MODE)	  ||
+	    (!sc->rss_enable)			  ||
+	    (sc->flags & OCE_FLAGS_BE2)) {
 		sc->nrqs = 1;
 		sc->nwqs = 1;
 		sc->rss_enable = 0;
@@ -2018,11 +2005,6 @@ setup_max_queues_want(POCE_SOFTC sc)
 
 		sc->nrqs = MIN(OCE_NCPUS, max_rss) + 1; /* 1 for def RX */
 		sc->nwqs = MIN(OCE_NCPUS, max_rss);
-	
-		/*Hardware issue. Turn off multi TX for be2 */	
-		if (IS_BE(sc) && (sc->flags & OCE_FLAGS_BE2))
-			sc->nwqs = 1;
-
 	}
 
 }
@@ -2034,8 +2016,6 @@ update_queues_got(POCE_SOFTC sc)
 	if (sc->rss_enable) {
 		sc->nrqs = sc->intr_count + 1;
 		sc->nwqs = sc->intr_count;
-		if (IS_BE(sc) && (sc->flags & OCE_FLAGS_BE2))
-			sc->nwqs = 1;
 	} else {
 		sc->nrqs = 1;
 		sc->nwqs = 1;
diff --git a/sys/dev/oce/oce_if.h b/sys/dev/oce/oce_if.h
index b08865d..cd35531 100644
--- a/sys/dev/oce/oce_if.h
+++ b/sys/dev/oce/oce_if.h
@@ -36,7 +36,6 @@
  * Costa Mesa, CA 92626
  */
 
-
 /* $FreeBSD$ */
 
 #include <sys/param.h>
@@ -88,7 +87,9 @@
 
 #include "oce_hw.h"
 
-#define COMPONENT_REVISION "4.2.116.0"
+/* OCE device driver module component revision informaiton */
+#define COMPONENT_REVISION "4.2.127.0"
+
 
 /* OCE devices supported by this driver */
 #define PCI_VENDOR_EMULEX		0x10df	/* Emulex */
@@ -111,7 +112,9 @@
 
 extern int mp_ncpus;			/* system's total active cpu cores */
 #define OCE_NCPUS			mp_ncpus
-#define OCE_MAX_RSS			8 /* This should be powers of 2. Like 2,4,8 & 16 */ 
+
+/* This should be powers of 2. Like 2,4,8 & 16 */
+#define OCE_MAX_RSS			4 /* TODO: 8*/ 
 #define OCE_LEGACY_MODE_RSS		4 /* For BE3 Legacy mode*/
 
 #define OCE_MIN_RQ			1
@@ -171,8 +174,7 @@ extern int mp_ncpus;			/* system's total active cpu cores */
 #define OCE_IF_HWASSIST			(CSUM_IP | CSUM_TCP | CSUM_UDP)
 #define OCE_IF_CAPABILITIES		(IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING | \
 					IFCAP_HWCSUM | IFCAP_VLAN_HWCSUM | \
-					IFCAP_VLAN_HWTSO | IFCAP_JUMBO_MTU | \
-					IFCAP_VLAN_MTU)
+					IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU)
 #define OCE_IF_HWASSIST_NONE		0
 #define OCE_IF_CAPABILITIES_NONE 	0
 
@@ -835,6 +837,7 @@ typedef struct oce_softc {
 	struct oce_drv_stats oce_stats_info;
 	struct callout  timer;
 	int8_t be3_native;
+	uint32_t pvid;
 
 } OCE_SOFTC, *POCE_SOFTC;
 
@@ -926,10 +929,12 @@ uint32_t oce_page_list(oce_ring_buffer_t *ring, struct phys_addr *pa_list);
 /***********************************************************
  * cleanup  functions
  ***********************************************************/
-void oce_free_lro(POCE_SOFTC sc);
 void oce_stop_rx(POCE_SOFTC sc);
 void oce_intr_free(POCE_SOFTC sc);
 void oce_free_posted_rxbuf(struct oce_rq *rq);
+#if defined(INET6) || defined(INET)
+void oce_free_lro(POCE_SOFTC sc);
+#endif
 
 
 /************************************************************
@@ -940,6 +945,8 @@ int oce_reset_fun(POCE_SOFTC sc);
 int oce_mbox_init(POCE_SOFTC sc);
 int oce_mbox_dispatch(POCE_SOFTC sc, uint32_t tmo_sec);
 int oce_get_fw_version(POCE_SOFTC sc);
+int oce_first_mcc_cmd(POCE_SOFTC sc);
+
 int oce_read_mac_addr(POCE_SOFTC sc, uint32_t if_id, uint8_t perm,
 			uint8_t type, struct mac_address_format *mac);
 int oce_get_fw_config(POCE_SOFTC sc);
diff --git a/sys/dev/oce/oce_mbox.c b/sys/dev/oce/oce_mbox.c
index 7ca3d41..07107a2 100644
--- a/sys/dev/oce/oce_mbox.c
+++ b/sys/dev/oce/oce_mbox.c
@@ -37,10 +37,8 @@
  */
 
 
-
 /* $FreeBSD$ */
 
-
 #include "oce_if.h"
 
 
@@ -145,7 +143,6 @@ oce_mbox_wait(POCE_SOFTC sc, uint32_t tmo_sec)
 }
 
 
-
 /**
  * @brief Mailbox dispatch
  * @param sc		software handle to the device
@@ -289,6 +286,42 @@ oce_get_fw_version(POCE_SOFTC sc)
 
 
 /**
+ * @brief	Firmware will send gracious notifications during
+ *		attach only after sending first mcc commnad. We  
+ *		use MCC queue only for getting async and mailbox
+ *		for sending cmds. So to get gracious notifications
+ *		atleast send one dummy command on mcc.
+ */
+int 
+oce_first_mcc_cmd(POCE_SOFTC sc)
+{
+	struct oce_mbx *mbx;
+	struct oce_mq *mq = sc->mq;
+	struct mbx_get_common_fw_version *fwcmd;
+	uint32_t reg_value;
+
+	mbx = RING_GET_PRODUCER_ITEM_VA(mq->ring, struct oce_mbx);
+	bzero(mbx, sizeof(struct oce_mbx));
+	
+	fwcmd = (struct mbx_get_common_fw_version *)&mbx->payload;
+	mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0,
+				MBX_SUBSYSTEM_COMMON,
+				OPCODE_COMMON_GET_FW_VERSION,
+				MBX_TIMEOUT_SEC,
+				sizeof(struct mbx_get_common_fw_version),
+				OCE_MBX_VER_V0);
+	mbx->u0.s.embedded = 1;
+	mbx->payload_length = sizeof(struct mbx_get_common_fw_version);
+	bus_dmamap_sync(mq->ring->dma.tag, mq->ring->dma.map,
+				BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
+	RING_PUT(mq->ring, 1);
+	reg_value = (1 << 16) | mq->mq_id;
+	OCE_WRITE_REG32(sc, db, PD_MQ_DB, reg_value);
+
+	return 0;
+}
+
+/**
  * @brief		Function to post a MBX to the mbox
  * @param sc		software handle to the device
  * @param mbx 		pointer to the MBX to send
@@ -1328,6 +1361,7 @@ oce_mbox_write_flashrom(POCE_SOFTC sc, uint32_t optype,uint32_t opcode,
 	sgl->length = payload_len;
 
 	/* post the command */
+	rc = oce_mbox_post(sc, &mbx, NULL);
 	if (rc) {
 		device_printf(sc->dev, "Write FlashROM mbox post failed\n");
 	} else {
diff --git a/sys/dev/oce/oce_queue.c b/sys/dev/oce/oce_queue.c
index c59dc50..580853e 100644
--- a/sys/dev/oce/oce_queue.c
+++ b/sys/dev/oce/oce_queue.c
@@ -36,11 +36,8 @@
  * Costa Mesa, CA 92626
  */
 
-
-
 /* $FreeBSD$ */
 
-
 #include "oce_if.h"
 
 /*****************************************************
@@ -653,11 +650,11 @@ static struct oce_mq *
 oce_mq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len)
 {
 	struct oce_mbx mbx;
-	struct mbx_create_common_mq *fwcmd = NULL;
+	struct mbx_create_common_mq_ex *fwcmd = NULL;
 	struct oce_mq *mq = NULL;
 	int rc = 0;
 	struct oce_cq *cq;
-	oce_mq_ctx_t *ctx;
+	oce_mq_ext_ctx_t *ctx;
 	uint32_t num_pages;
 	uint32_t page_size;
 	uint32_t version;
@@ -683,13 +680,13 @@ oce_mq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len)
 
 	bzero(&mbx, sizeof(struct oce_mbx));
 
-	fwcmd = (struct mbx_create_common_mq *)&mbx.payload;
+	fwcmd = (struct mbx_create_common_mq_ex *)&mbx.payload;
 	version = OCE_MBX_VER_V0;
 	mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0,
 				MBX_SUBSYSTEM_COMMON,
-				OPCODE_COMMON_CREATE_MQ,
+				OPCODE_COMMON_CREATE_MQ_EXT,
 				MBX_TIMEOUT_SEC,
-				sizeof(struct mbx_create_common_mq),
+				sizeof(struct mbx_create_common_mq_ex),
 				version);
 
 	num_pages = oce_page_list(mq->ring, &fwcmd->params.req.pages[0]);
@@ -700,9 +697,11 @@ oce_mq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len)
 	ctx->v0.cq_id = cq->cq_id;
 	ctx->v0.ring_size = OCE_LOG2(q_len) + 1;
 	ctx->v0.valid = 1;
+	/* Subscribe to Link State and Group 5 Events(bits 1 and 5 set) */
+	ctx->v0.async_evt_bitmap = 0xffffffff;
 
 	mbx.u0.s.embedded = 1;
-	mbx.payload_length = sizeof(struct mbx_create_common_mq);
+	mbx.payload_length = sizeof(struct mbx_create_common_mq_ex);
 	DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ);
 
 	rc = oce_mbox_post(sc, &mbx, NULL);
diff --git a/sys/dev/oce/oce_sysctl.c b/sys/dev/oce/oce_sysctl.c
index e0c50dd..93117f6 100644
--- a/sys/dev/oce/oce_sysctl.c
+++ b/sys/dev/oce/oce_sysctl.c
@@ -36,10 +36,8 @@
  * Costa Mesa, CA 92626
  */
 
-
 /* $FreeBSD$ */
 
-
 #include "oce_if.h"
 
 static void copy_stats_to_sc_xe201(POCE_SOFTC sc);
@@ -108,6 +106,13 @@ oce_add_sysctls(POCE_SOFTC sc)
 				&sc->speed,
 				0,"Link Speed");
 
+	if (sc->function_mode & FNM_UMC_MODE)
+		SYSCTL_ADD_UINT(ctx, child,
+				OID_AUTO, "pvid",
+				CTLFLAG_RD,
+				&sc->pvid,
+				0,"PVID");
+
 	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "loop_back",
 		CTLTYPE_INT | CTLFLAG_RW, (void *)sc, 0,
 		oce_sysctl_loopback, "I", "Loop Back Tests");
diff --git a/sys/dev/oce/oce_util.c b/sys/dev/oce/oce_util.c
index 93b13bd..7b227ac 100644
--- a/sys/dev/oce/oce_util.c
+++ b/sys/dev/oce/oce_util.c
@@ -36,10 +36,8 @@
  * Costa Mesa, CA 92626
  */
 
-
 /* $FreeBSD$ */
 
-
 #include "oce_if.h"
 
 static void oce_dma_map_ring(void *arg,