MFC of the Intel Base driver for the Intel XL710 Ethernet Controller Family

	- It was decided to change the driver name to if_ixl for FreeBSD
	- This release adds the VF Driver to the tree, it can be built into
	  the kernel or as the if_ixlv module
	- The VF driver is independent for the first time, this will be
	  desireable when full SRIOV capability is added to the OS.

Submitted by:	jack.vogel@intel.com and eric.joyner@intel.com

1 files changed, 2742 insertions, 0 deletions

diff --git a/sys/dev/ixl/if_ixlv.c b/sys/dev/ixl/if_ixlv.c
new file mode 100644
index 0000000..0e6e572
--- /dev/null
+++ b/sys/dev/ixl/if_ixlv.c
@@ -0,0 +1,2742 @@
+/******************************************************************************
+
+  Copyright (c) 2013-2014, Intel Corporation 
+  All rights reserved.
+  
+  Redistribution and use in source and binary forms, with or without 
+  modification, are permitted provided that the following conditions are met:
+  
+   1. Redistributions of source code must retain the above copyright notice, 
+      this list of conditions and the following disclaimer.
+  
+   2. Redistributions in binary form must reproduce the above copyright 
+      notice, this list of conditions and the following disclaimer in the 
+      documentation and/or other materials provided with the distribution.
+  
+   3. Neither the name of the Intel Corporation nor the names of its 
+      contributors may be used to endorse or promote products derived from 
+      this software without specific prior written permission.
+  
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 
+  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 
+  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 
+  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
+  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+  POSSIBILITY OF SUCH DAMAGE.
+
+******************************************************************************/
+/*$FreeBSD$*/
+
+#include "opt_inet.h"
+#include "opt_inet6.h"
+#include "ixl.h"
+#include "ixlv.h"
+
+/*********************************************************************
+ *  Driver version
+ *********************************************************************/
+char ixlv_driver_version[] = "1.1.4";
+
+/*********************************************************************
+ *  PCI Device ID Table
+ *
+ *  Used by probe to select devices to load on
+ *  Last field stores an index into ixlv_strings
+ *  Last entry must be all 0s
+ *
+ *  { Vendor ID, Device ID, SubVendor ID, SubDevice ID, String Index }
+ *********************************************************************/
+
+static ixl_vendor_info_t ixlv_vendor_info_array[] =
+{
+	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF, 0, 0, 0},
+	{I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV, 0, 0, 0},
+	/* required last entry */
+	{0, 0, 0, 0, 0}
+};
+
+/*********************************************************************
+ *  Table of branding strings
+ *********************************************************************/
+
+static char    *ixlv_strings[] = {
+	"Intel(R) Ethernet Connection XL710 VF Driver"
+};
+
+
+/*********************************************************************
+ *  Function prototypes
+ *********************************************************************/
+static int      ixlv_probe(device_t);
+static int      ixlv_attach(device_t);
+static int      ixlv_detach(device_t);
+static int      ixlv_shutdown(device_t);
+static void	ixlv_init_locked(struct ixlv_sc *);
+static int	ixlv_allocate_pci_resources(struct ixlv_sc *);
+static void	ixlv_free_pci_resources(struct ixlv_sc *);
+static int	ixlv_assign_msix(struct ixlv_sc *);
+static int	ixlv_init_msix(struct ixlv_sc *);
+static int	ixlv_init_taskqueue(struct ixlv_sc *);
+static int	ixlv_setup_queues(struct ixlv_sc *);
+static void	ixlv_config_rss(struct ixlv_sc *);
+static void	ixlv_stop(struct ixlv_sc *);
+static void	ixlv_add_multi(struct ixl_vsi *);
+static void	ixlv_del_multi(struct ixl_vsi *);
+static void	ixlv_update_link_status(struct ixlv_sc *);
+static void	ixlv_free_queues(struct ixl_vsi *);
+static int	ixlv_setup_interface(device_t, struct ixlv_sc *);
+
+static int	ixlv_media_change(struct ifnet *);
+static void	ixlv_media_status(struct ifnet *, struct ifmediareq *);
+
+static void	ixlv_local_timer(void *);
+
+static int	ixlv_add_mac_filter(struct ixlv_sc *, u8 *, u16);
+static void	ixlv_init_filters(struct ixlv_sc *);
+static void	ixlv_free_filters(struct ixlv_sc *);
+
+static void	ixlv_msix_que(void *);
+static void	ixlv_msix_adminq(void *);
+static void	ixlv_do_adminq(void *, int);
+static void	ixlv_sched_aq(void *);
+static void	ixlv_handle_que(void *, int);
+static int	ixlv_reset(struct ixlv_sc *);
+static int	ixlv_reset_complete(struct i40e_hw *);
+static void	ixlv_set_queue_rx_itr(struct ixl_queue *);
+static void	ixlv_set_queue_tx_itr(struct ixl_queue *);
+
+static void	ixlv_enable_adminq_irq(struct i40e_hw *);
+static void	ixlv_disable_adminq_irq(struct i40e_hw *);
+static void	ixlv_enable_queue_irq(struct i40e_hw *, int);
+static void	ixlv_disable_queue_irq(struct i40e_hw *, int);
+
+static void	ixlv_setup_vlan_filters(struct ixlv_sc *);
+static void	ixlv_register_vlan(void *, struct ifnet *, u16);
+static void	ixlv_unregister_vlan(void *, struct ifnet *, u16);
+
+static void	ixlv_cap_txcsum_tso(struct ixl_vsi *,
+		    struct ifnet *, int);
+
+static void	ixlv_add_stats_sysctls(struct ixlv_sc *);
+
+/*********************************************************************
+ *  FreeBSD Device Interface Entry Points
+ *********************************************************************/
+
+static device_method_t ixlv_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe, ixlv_probe),
+	DEVMETHOD(device_attach, ixlv_attach),
+	DEVMETHOD(device_detach, ixlv_detach),
+	DEVMETHOD(device_shutdown, ixlv_shutdown),
+	{0, 0}
+};
+
+static driver_t ixlv_driver = {
+	"ixlv", ixlv_methods, sizeof(struct ixlv_sc),
+};
+
+devclass_t ixlv_devclass;
+DRIVER_MODULE(ixlv, pci, ixlv_driver, ixlv_devclass, 0, 0);
+
+MODULE_DEPEND(ixlv, pci, 1, 1, 1);
+MODULE_DEPEND(ixlv, ether, 1, 1, 1);
+
+/*
+** TUNEABLE PARAMETERS:
+*/
+
+static SYSCTL_NODE(_hw, OID_AUTO, ixlv, CTLFLAG_RD, 0,
+                   "IXLV driver parameters");
+
+/*
+** Number of descriptors per ring:
+**   - TX and RX are the same size
+*/
+static int ixlv_ringsz = DEFAULT_RING;
+TUNABLE_INT("hw.ixlv.ringsz", &ixlv_ringsz);
+SYSCTL_INT(_hw_ixlv, OID_AUTO, ring_size, CTLFLAG_RDTUN,
+    &ixlv_ringsz, 0, "Descriptor Ring Size");
+
+/* Set to zero to auto calculate  */
+int ixlv_max_queues = 0;
+TUNABLE_INT("hw.ixlv.max_queues", &ixlv_max_queues);
+SYSCTL_INT(_hw_ixlv, OID_AUTO, max_queues, CTLFLAG_RDTUN,
+    &ixlv_max_queues, 0, "Number of Queues");
+
+/*
+** Number of entries in Tx queue buf_ring.
+** Increasing this will reduce the number of
+** errors when transmitting fragmented UDP
+** packets.
+*/
+static int ixlv_txbrsz = DEFAULT_TXBRSZ;
+TUNABLE_INT("hw.ixlv.txbrsz", &ixlv_txbrsz);
+SYSCTL_INT(_hw_ixlv, OID_AUTO, txbr_size, CTLFLAG_RDTUN,
+    &ixlv_txbrsz, 0, "TX Buf Ring Size");
+
+/*
+** Controls for Interrupt Throttling
+**      - true/false for dynamic adjustment
+**      - default values for static ITR
+*/
+int ixlv_dynamic_rx_itr = 0;
+TUNABLE_INT("hw.ixlv.dynamic_rx_itr", &ixlv_dynamic_rx_itr);
+SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_rx_itr, CTLFLAG_RDTUN,
+    &ixlv_dynamic_rx_itr, 0, "Dynamic RX Interrupt Rate");
+
+int ixlv_dynamic_tx_itr = 0;
+TUNABLE_INT("hw.ixlv.dynamic_tx_itr", &ixlv_dynamic_tx_itr);
+SYSCTL_INT(_hw_ixlv, OID_AUTO, dynamic_tx_itr, CTLFLAG_RDTUN,
+    &ixlv_dynamic_tx_itr, 0, "Dynamic TX Interrupt Rate");
+
+int ixlv_rx_itr = IXL_ITR_8K;
+TUNABLE_INT("hw.ixlv.rx_itr", &ixlv_rx_itr);
+SYSCTL_INT(_hw_ixlv, OID_AUTO, rx_itr, CTLFLAG_RDTUN,
+    &ixlv_rx_itr, 0, "RX Interrupt Rate");
+
+int ixlv_tx_itr = IXL_ITR_4K;
+TUNABLE_INT("hw.ixlv.tx_itr", &ixlv_tx_itr);
+SYSCTL_INT(_hw_ixlv, OID_AUTO, tx_itr, CTLFLAG_RDTUN,
+    &ixlv_tx_itr, 0, "TX Interrupt Rate");
+
+        
+/*********************************************************************
+ *  Device identification routine
+ *
+ *  ixlv_probe determines if the driver should be loaded on
+ *  the hardware based on PCI vendor/device id of the device.
+ *
+ *  return BUS_PROBE_DEFAULT on success, positive on failure
+ *********************************************************************/
+
+static int
+ixlv_probe(device_t dev)
+{
+	ixl_vendor_info_t *ent;
+
+	u16	pci_vendor_id, pci_device_id;
+	u16	pci_subvendor_id, pci_subdevice_id;
+	char	device_name[256];
+
+	INIT_DEBUGOUT("ixlv_probe: begin");
+
+	pci_vendor_id = pci_get_vendor(dev);
+	if (pci_vendor_id != I40E_INTEL_VENDOR_ID)
+		return (ENXIO);
+
+	pci_device_id = pci_get_device(dev);
+	pci_subvendor_id = pci_get_subvendor(dev);
+	pci_subdevice_id = pci_get_subdevice(dev);
+
+	ent = ixlv_vendor_info_array;
+	while (ent->vendor_id != 0) {
+		if ((pci_vendor_id == ent->vendor_id) &&
+		    (pci_device_id == ent->device_id) &&
+
+		    ((pci_subvendor_id == ent->subvendor_id) ||
+		     (ent->subvendor_id == 0)) &&
+
+		    ((pci_subdevice_id == ent->subdevice_id) ||
+		     (ent->subdevice_id == 0))) {
+			sprintf(device_name, "%s, Version - %s",
+				ixlv_strings[ent->index],
+				ixlv_driver_version);
+			device_set_desc_copy(dev, device_name);
+			return (BUS_PROBE_DEFAULT);
+		}
+		ent++;
+	}
+	return (ENXIO);
+}
+
+/*********************************************************************
+ *  Device initialization routine
+ *
+ *  The attach entry point is called when the driver is being loaded.
+ *  This routine identifies the type of hardware, allocates all resources
+ *  and initializes the hardware.
+ *
+ *  return 0 on success, positive on failure
+ *********************************************************************/
+
+static int
+ixlv_attach(device_t dev)
+{
+	struct ixlv_sc	*sc;
+	struct i40e_hw	*hw;
+	struct ixl_vsi 	*vsi;
+	int            	bufsz, error = 0, retries = 0;
+
+	INIT_DBG_DEV(dev, "begin");
+
+	/* Allocate, clear, and link in our primary soft structure */
+	sc = device_get_softc(dev);
+	sc->dev = sc->osdep.dev = dev;
+	hw = &sc->hw;
+	vsi = &sc->vsi;
+	vsi->dev = dev;
+
+	/* Allocate filter lists */
+	ixlv_init_filters(sc);
+
+	/* Core Lock Init*/
+	mtx_init(&sc->mtx, device_get_nameunit(dev),
+	    "IXL SC Lock", MTX_DEF);
+	mtx_init(&sc->aq_task_mtx, device_get_nameunit(dev),
+	    "IXL AQ Task Lock", MTX_DEF);
+
+	/* Set up the timer & aq watchdog callouts */
+	callout_init_mtx(&sc->timer, &sc->mtx, 0);
+	callout_init_mtx(&sc->aq_task, &sc->aq_task_mtx, 0);
+
+	/* Save off the information about this board */
+	hw->vendor_id = pci_get_vendor(dev);
+	hw->device_id = pci_get_device(dev);
+	hw->revision_id = pci_read_config(dev, PCIR_REVID, 1);
+	hw->subsystem_vendor_id =
+	    pci_read_config(dev, PCIR_SUBVEND_0, 2);
+	hw->subsystem_device_id =
+	    pci_read_config(dev, PCIR_SUBDEV_0, 2);
+
+	hw->bus.device = pci_get_slot(dev);
+	hw->bus.func = pci_get_function(dev);
+
+	/* Do PCI setup - map BAR0, etc */
+	if (ixlv_allocate_pci_resources(sc)) {
+		device_printf(dev, "%s: Allocation of PCI resources failed\n",
+		    __func__);
+		error = ENXIO;
+		goto err_early;
+	}
+
+	INIT_DBG_DEV(dev, "Allocated PCI resources and MSIX vectors");
+
+	error = i40e_set_mac_type(hw);
+	if (error) {
+		device_printf(dev, "%s: set_mac_type failed: %d\n",
+		    __func__, error);
+		goto err_pci_res;
+	}
+
+	error = ixlv_reset_complete(hw);
+	if (error) {
+		device_printf(dev, "%s: Device is still being reset\n",
+		    __func__);
+		goto err_pci_res;
+	}
+
+	INIT_DBG_DEV(dev, "VF Device is ready for configuration");
+
+	hw->aq.num_arq_entries = IXL_AQ_LEN;
+	hw->aq.num_asq_entries = IXL_AQ_LEN;
+	hw->aq.arq_buf_size = IXL_AQ_BUFSZ;
+	hw->aq.asq_buf_size = IXL_AQ_BUFSZ;
+
+	error = i40e_init_adminq(hw);
+	if (error) {
+		device_printf(dev, "%s: init_adminq failed: %d\n",
+		    __func__, error);
+		goto err_pci_res;
+	}
+
+	INIT_DBG_DEV(dev, "Initialized Admin Queue");
+
+	error = ixlv_send_api_ver(sc);
+	if (error) {
+		device_printf(dev, "%s: unable to send to PF (%d)\n",
+		     __func__, error);
+		goto err_aq;
+	}
+
+	while (!i40e_asq_done(hw)) {
+		if (++retries > IXLV_AQ_MAX_ERR) {
+			device_printf(dev, "%s: Admin Queue timeout "
+			    "(waiting for send_api_ver)\n", __func__);
+			error = ENXIO;
+			goto err_aq;
+		}
+		i40e_msec_delay(10);
+	}
+
+	INIT_DBG_DEV(dev, "Sent API version message to PF");
+
+	/* Wait for API version msg to arrive */
+	error = ixlv_verify_api_ver(sc);
+	if (error) {
+		device_printf(dev,
+		    "%s: Unable to verify API version, error %d\n",
+			    __func__, error);
+		goto err_aq;
+	}
+
+	INIT_DBG_DEV(dev, "PF API version verified");
+
+	/* Need API version before sending reset message */
+	error = ixlv_reset(sc);
+	if (error) {
+		device_printf(dev, "VF reset failed; reload the driver\n");
+		goto err_aq;
+	}
+
+	INIT_DBG_DEV(dev, "VF reset complete");
+
+	/* Ask for VF config from PF */
+	error = ixlv_send_vf_config_msg(sc);
+	if (error) {
+		device_printf(dev,
+		    "%s: Unable to send VF config request, error %d\n",
+		    __func__, error);
+		goto err_aq;
+	}
+
+	retries = 0;
+	while (!i40e_asq_done(hw)) {
+		if (++retries > IXLV_AQ_MAX_ERR) {
+			device_printf(dev, "%s: Admin Queue timeout "
+			    "(waiting for send_vf_config_msg)\n", __func__);
+			error = ENXIO;
+			goto err_aq;
+		}
+		i40e_msec_delay(10);
+	}
+
+	INIT_DBG_DEV(dev, "Sent VF config message to PF");
+
+	bufsz = sizeof(struct i40e_virtchnl_vf_resource) +
+	    (I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource));
+	sc->vf_res = malloc(bufsz, M_DEVBUF, M_NOWAIT);
+	if (!sc->vf_res) {
+		device_printf(dev,
+		    "%s: Unable to allocate memory for VF configuration"
+		    " message from PF\n", __func__);
+		error = ENOMEM;
+		goto err_aq;
+	}
+
+	/* Check for VF config response */
+	error = ixlv_get_vf_config(sc);
+	if (error) {
+		device_printf(dev,
+		    "%s: Unable to get VF configuration from PF\n",
+		    __func__);
+		error = EBUSY;
+		goto err_res_buf;
+	}
+
+	INIT_DBG_DEV(dev, "Received valid VF config from PF");
+	INIT_DBG_DEV(dev, "VSIs %d, Queues %d, Max Vectors %d, Max MTU %d",
+	    sc->vf_res->num_vsis,
+	    sc->vf_res->num_queue_pairs,
+	    sc->vf_res->max_vectors,
+	    sc->vf_res->max_mtu);
+	INIT_DBG_DEV(dev, "Offload flags: %#010x",
+	    sc->vf_res->vf_offload_flags);
+
+	/* got VF config message back from PF, now we can parse it */
+	for (int i = 0; i < sc->vf_res->num_vsis; i++) {
+		if (sc->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV)
+			sc->vsi_res = &sc->vf_res->vsi_res[i];
+	}
+	if (!sc->vsi_res) {
+		device_printf(dev, "%s: no LAN VSI found\n", __func__);
+		goto err_res_buf;
+	}
+
+	INIT_DBG_DEV(dev, "Resource Acquisition complete");
+
+	/* If no mac address was assigned just make a random one */
+	if (!ixlv_check_ether_addr(hw->mac.addr)) {
+		u8 addr[ETHER_ADDR_LEN];
+		arc4rand(&addr, sizeof(addr), 0);
+		addr[0] &= 0xFE;
+		addr[0] |= 0x02;
+		bcopy(addr, hw->mac.addr, sizeof(addr));
+	}
+
+	vsi->id = sc->vsi_res->vsi_id;
+	vsi->back = (void *)sc;
+
+	/* Link in this virtual environment is always 'up' */
+	vsi->link_up = TRUE;
+
+	/* This allocates the memory and early settings */
+	if (ixlv_setup_queues(sc) != 0) {
+		device_printf(dev, "%s: setup queues failed!\n",
+		    __func__);
+		goto out;
+	}
+
+	/* Setup the stack interface */
+	if (ixlv_setup_interface(dev, sc) != 0) {
+		device_printf(dev, "%s: setup interface failed!\n",
+		    __func__);
+		goto out;
+	}
+
+	INIT_DBG_DEV(dev, "Queue memory and interface setup");
+
+	/* Do queue interrupt setup */
+	ixlv_assign_msix(sc);
+
+	/* Start AdminQ taskqueue */
+	ixlv_init_taskqueue(sc);
+
+	/* Start the admin queue scheduler timer */
+	callout_reset(&sc->aq_task, 2 * hz, ixlv_sched_aq, sc);
+
+	/* Initialize stats */
+	bzero(&sc->vsi.eth_stats, sizeof(struct i40e_eth_stats));
+	ixlv_add_stats_sysctls(sc);
+
+	/* Register for VLAN events */
+	vsi->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
+	    ixlv_register_vlan, vsi, EVENTHANDLER_PRI_FIRST);
+	vsi->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
+	    ixlv_unregister_vlan, vsi, EVENTHANDLER_PRI_FIRST);
+
+	/* We want AQ enabled early */
+	ixlv_enable_adminq_irq(hw);
+
+	/* Set things up to run init */
+	sc->aq_pending = 0;
+	sc->aq_required = 0;
+	sc->init_state = IXLV_INIT_READY;
+
+	INIT_DBG_DEV(dev, "end");
+	return (error);
+
+out:
+	ixlv_free_queues(vsi);
+err_res_buf:
+	free(sc->vf_res, M_DEVBUF);
+err_aq:
+	i40e_shutdown_adminq(hw);
+err_pci_res:
+	ixlv_free_pci_resources(sc);
+err_early:
+	mtx_destroy(&sc->mtx);
+	mtx_destroy(&sc->aq_task_mtx);
+	ixlv_free_filters(sc);
+	INIT_DBG_DEV(dev, "end: error %d", error);
+	return (error);
+}
+
+/*********************************************************************
+ *  Device removal routine
+ *
+ *  The detach entry point is called when the driver is being removed.
+ *  This routine stops the adapter and deallocates all the resources
+ *  that were allocated for driver operation.
+ *
+ *  return 0 on success, positive on failure
+ *********************************************************************/
+
+static int
+ixlv_detach(device_t dev)
+{
+	struct ixlv_sc	*sc = device_get_softc(dev);
+	struct ixl_vsi 	*vsi = &sc->vsi;
+	int 			 retries = 0;
+
+	INIT_DBG_DEV(dev, "begin");
+
+	/* Make sure VLANS are not using driver */
+	if (vsi->ifp->if_vlantrunk != NULL) {
+		device_printf(dev, "Vlan in use, detach first\n");
+		INIT_DBG_DEV(dev, "end");
+		return (EBUSY);
+	}
+
+	/* Stop driver */
+	if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		mtx_lock(&sc->mtx);	
+		ixlv_stop(sc);
+		mtx_unlock(&sc->mtx);	
+
+		/*
+		** Ensure queues are disabled before examining
+		** admin queue state later in detach.
+		*/
+		while (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING
+		    && ++retries < IXLV_AQ_MAX_ERR) {
+			i40e_msec_delay(10);
+		}
+#ifdef IXL_DEBUG
+		if (retries >= IXLV_AQ_MAX_ERR)
+			device_printf(dev, "Issue disabling queues for detach\n");
+#endif
+	}
+
+	/* Unregister VLAN events */
+	if (vsi->vlan_attach != NULL)
+		EVENTHANDLER_DEREGISTER(vlan_config, vsi->vlan_attach);
+	if (vsi->vlan_detach != NULL)
+		EVENTHANDLER_DEREGISTER(vlan_unconfig, vsi->vlan_detach);
+
+	/* Stop AQ callout */
+	callout_drain(&sc->aq_task);
+	callout_stop(&sc->aq_task);
+
+#ifdef IXL_DEBUG
+	/* Report on possible AQ failures */
+	if (sc->aq_required || sc->aq_pending) {
+		device_printf(dev, "AQ status on detach:\n");
+		device_printf(dev, "required  : 0x%4b\n", sc->aq_required,
+		    IXLV_FLAGS);
+		device_printf(dev, "pending   : 0x%4b\n", sc->aq_pending,
+		    IXLV_FLAGS);
+		device_printf(dev, "current_op: %d\n", sc->current_op);
+	}
+#endif
+
+	i40e_shutdown_adminq(&sc->hw);
+	while (taskqueue_cancel(sc->tq, &sc->aq_irq, NULL) != 0)
+		taskqueue_drain(sc->tq, &sc->aq_irq);
+	taskqueue_free(sc->tq);
+
+	/* force the state down */
+	vsi->ifp->if_flags &= ~IFF_UP;
+	ether_ifdetach(vsi->ifp);
+	if_free(vsi->ifp);
+
+	free(sc->vf_res, M_DEVBUF);
+	ixlv_free_pci_resources(sc);
+	ixlv_free_queues(vsi);
+	mtx_destroy(&sc->mtx);
+	mtx_destroy(&sc->aq_task_mtx);
+	ixlv_free_filters(sc);
+
+	bus_generic_detach(dev);
+	INIT_DBG_DEV(dev, "end");
+	return (0);
+}
+
+/*********************************************************************
+ *
+ *  Shutdown entry point
+ *
+ **********************************************************************/
+
+static int
+ixlv_shutdown(device_t dev)
+{
+	struct ixlv_sc	*sc = device_get_softc(dev);
+
+	INIT_DBG_DEV(dev, "begin");
+
+	mtx_lock(&sc->mtx);	
+	ixlv_stop(sc);
+	mtx_unlock(&sc->mtx);	
+
+	INIT_DBG_DEV(dev, "end");
+	return (0);
+}
+
+/*
+ * Configure TXCSUM(IPV6) and TSO(4/6)
+ *	- the hardware handles these together so we
+ *	  need to tweak them 
+ */
+static void
+ixlv_cap_txcsum_tso(struct ixl_vsi *vsi, struct ifnet *ifp, int mask)
+{
+	/* Enable/disable TXCSUM/TSO4 */
+	if (!(ifp->if_capenable & IFCAP_TXCSUM)
+	    && !(ifp->if_capenable & IFCAP_TSO4)) {
+		if (mask & IFCAP_TXCSUM) {
+			ifp->if_capenable |= IFCAP_TXCSUM;
+			/* enable TXCSUM, restore TSO if previously enabled */
+			if (vsi->flags & IXL_FLAGS_KEEP_TSO4) {
+				vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
+				ifp->if_capenable |= IFCAP_TSO4;
+			}
+		}
+		else if (mask & IFCAP_TSO4) {
+			ifp->if_capenable |= (IFCAP_TXCSUM | IFCAP_TSO4);
+			vsi->flags &= ~IXL_FLAGS_KEEP_TSO4;
+			if_printf(ifp,
+			    "TSO4 requires txcsum, enabling both...\n");
+		}
+	} else if((ifp->if_capenable & IFCAP_TXCSUM)
+	    && !(ifp->if_capenable & IFCAP_TSO4)) {
+		if (mask & IFCAP_TXCSUM)
+			ifp->if_capenable &= ~IFCAP_TXCSUM;
+		else if (mask & IFCAP_TSO4)
+			ifp->if_capenable |= IFCAP_TSO4;
+	} else if((ifp->if_capenable & IFCAP_TXCSUM)
+	    && (ifp->if_capenable & IFCAP_TSO4)) {
+		if (mask & IFCAP_TXCSUM) {
+			vsi->flags |= IXL_FLAGS_KEEP_TSO4;
+			ifp->if_capenable &= ~(IFCAP_TXCSUM | IFCAP_TSO4);
+			if_printf(ifp, 
+			    "TSO4 requires txcsum, disabling both...\n");
+		} else if (mask & IFCAP_TSO4)
+			ifp->if_capenable &= ~IFCAP_TSO4;
+	}
+
+	/* Enable/disable TXCSUM_IPV6/TSO6 */
+	if (!(ifp->if_capenable & IFCAP_TXCSUM_IPV6)
+	    && !(ifp->if_capenable & IFCAP_TSO6)) {
+		if (mask & IFCAP_TXCSUM_IPV6) {
+			ifp->if_capenable |= IFCAP_TXCSUM_IPV6;
+			if (vsi->flags & IXL_FLAGS_KEEP_TSO6) {
+				vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
+				ifp->if_capenable |= IFCAP_TSO6;
+			}
+		} else if (mask & IFCAP_TSO6) {
+			ifp->if_capenable |= (IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
+			vsi->flags &= ~IXL_FLAGS_KEEP_TSO6;
+			if_printf(ifp,
+			    "TSO6 requires txcsum6, enabling both...\n");
+		}
+	} else if((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
+	    && !(ifp->if_capenable & IFCAP_TSO6)) {
+		if (mask & IFCAP_TXCSUM_IPV6)
+			ifp->if_capenable &= ~IFCAP_TXCSUM_IPV6;
+		else if (mask & IFCAP_TSO6)
+			ifp->if_capenable |= IFCAP_TSO6;
+	} else if ((ifp->if_capenable & IFCAP_TXCSUM_IPV6)
+	    && (ifp->if_capenable & IFCAP_TSO6)) {
+		if (mask & IFCAP_TXCSUM_IPV6) {
+			vsi->flags |= IXL_FLAGS_KEEP_TSO6;
+			ifp->if_capenable &= ~(IFCAP_TXCSUM_IPV6 | IFCAP_TSO6);
+			if_printf(ifp,
+			    "TSO6 requires txcsum6, disabling both...\n");
+		} else if (mask & IFCAP_TSO6)
+			ifp->if_capenable &= ~IFCAP_TSO6;
+	}
+}
+
+/*********************************************************************
+ *  Ioctl entry point
+ *
+ *  ixlv_ioctl is called when the user wants to configure the
+ *  interface.
+ *
+ *  return 0 on success, positive on failure
+ **********************************************************************/
+
+static int
+ixlv_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
+{
+	struct ixl_vsi		*vsi = ifp->if_softc;
+	struct ixlv_sc	*sc = vsi->back;
+	struct ifreq		*ifr = (struct ifreq *)data;
+#if defined(INET) || defined(INET6)
+	struct ifaddr 		*ifa = (struct ifaddr *)data;
+	bool			avoid_reset = FALSE;
+#endif
+	int             	error = 0;
+
+
+	switch (command) {
+
+        case SIOCSIFADDR:
+#ifdef INET
+		if (ifa->ifa_addr->sa_family == AF_INET)
+			avoid_reset = TRUE;
+#endif
+#ifdef INET6
+		if (ifa->ifa_addr->sa_family == AF_INET6)
+			avoid_reset = TRUE;
+#endif
+#if defined(INET) || defined(INET6)
+		/*
+		** Calling init results in link renegotiation,
+		** so we avoid doing it when possible.
+		*/
+		if (avoid_reset) {
+			ifp->if_flags |= IFF_UP;
+			if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
+				ixlv_init(sc);
+			if (!(ifp->if_flags & IFF_NOARP))
+				arp_ifinit(ifp, ifa);
+		} else
+			error = ether_ioctl(ifp, command, data);
+		break;
+#endif
+	case SIOCSIFMTU:
+		IOCTL_DBG_IF2(ifp, "SIOCSIFMTU (Set Interface MTU)");
+		mtx_lock(&sc->mtx);
+		if (ifr->ifr_mtu > IXL_MAX_FRAME -
+		    ETHER_HDR_LEN - ETHER_CRC_LEN - ETHER_VLAN_ENCAP_LEN) {
+			error = EINVAL;
+			IOCTL_DBG_IF(ifp, "mtu too large");
+		} else {
+			IOCTL_DBG_IF2(ifp, "mtu: %lu -> %d", ifp->if_mtu, ifr->ifr_mtu);
+			// ERJ: Interestingly enough, these types don't match
+			ifp->if_mtu = ifr->ifr_mtu;
+			vsi->max_frame_size =
+			    ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
+			    + ETHER_VLAN_ENCAP_LEN;
+
+			ixlv_init_locked(sc);
+		}
+		mtx_unlock(&sc->mtx);
+		break;
+	case SIOCSIFFLAGS:
+		IOCTL_DBG_IF2(ifp, "SIOCSIFFLAGS (Set Interface Flags)");
+		mtx_lock(&sc->mtx);
+		if (ifp->if_flags & IFF_UP) {
+			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
+				ixlv_init_locked(sc);
+		} else
+			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+				ixlv_stop(sc);
+		sc->if_flags = ifp->if_flags;
+		mtx_unlock(&sc->mtx);
+		break;
+	case SIOCADDMULTI:
+		IOCTL_DBG_IF2(ifp, "SIOCADDMULTI");
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+			mtx_lock(&sc->mtx);
+			ixlv_disable_intr(vsi);
+			ixlv_add_multi(vsi);
+			ixlv_enable_intr(vsi);
+			mtx_unlock(&sc->mtx);
+		}
+		break;
+	case SIOCDELMULTI:
+		IOCTL_DBG_IF2(ifp, "SIOCDELMULTI");
+		if (sc->init_state == IXLV_RUNNING) {
+			mtx_lock(&sc->mtx);
+			ixlv_disable_intr(vsi);
+			ixlv_del_multi(vsi);
+			ixlv_enable_intr(vsi);
+			mtx_unlock(&sc->mtx);
+		}
+		break;
+	case SIOCSIFMEDIA:
+	case SIOCGIFMEDIA:
+		IOCTL_DBG_IF2(ifp, "SIOCxIFMEDIA (Get/Set Interface Media)");
+		error = ifmedia_ioctl(ifp, ifr, &sc->media, command);
+		break;
+	case SIOCSIFCAP:
+	{
+		int mask = ifr->ifr_reqcap ^ ifp->if_capenable;
+		IOCTL_DBG_IF2(ifp, "SIOCSIFCAP (Set Capabilities)");
+
+		ixlv_cap_txcsum_tso(vsi, ifp, mask);
+
+		if (mask & IFCAP_RXCSUM)
+			ifp->if_capenable ^= IFCAP_RXCSUM;
+		if (mask & IFCAP_RXCSUM_IPV6)
+			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
+		if (mask & IFCAP_LRO)
+			ifp->if_capenable ^= IFCAP_LRO;
+		if (mask & IFCAP_VLAN_HWTAGGING)
+			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
+		if (mask & IFCAP_VLAN_HWFILTER)
+			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
+		if (mask & IFCAP_VLAN_HWTSO)
+			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
+		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+			ixlv_init(sc);
+		}
+		VLAN_CAPABILITIES(ifp);
+
+		break;
+	}
+
+	default:
+		IOCTL_DBG_IF2(ifp, "UNKNOWN (0x%X)", (int)command);
+		error = ether_ioctl(ifp, command, data);
+		break;
+	}
+
+	return (error);
+}
+
+/*
+** To do a reinit on the VF is unfortunately more complicated
+** than a physical device, we must have the PF more or less
+** completely recreate our memory, so many things that were
+** done only once at attach in traditional drivers now must be
+** redone at each reinitialization. This function does that
+** 'prelude' so we can then call the normal locked init code.
+*/
+int
+ixlv_reinit_locked(struct ixlv_sc *sc)
+{
+	struct i40e_hw		*hw = &sc->hw;
+	struct ixl_vsi		*vsi = &sc->vsi;
+	struct ifnet		*ifp = vsi->ifp;
+	struct ixlv_vlan_filter	*vf;
+	int			error = 0;
+
+	INIT_DBG_IF(ifp, "begin");
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
+		ixlv_stop(sc);
+
+	if ((sc->init_state == IXLV_RESET_REQUIRED) ||
+	    (sc->init_state == IXLV_RESET_PENDING))
+		error = ixlv_reset(sc);
+
+	/* set the state in case we went thru RESET */
+	sc->init_state = IXLV_RUNNING;
+
+	if (vsi->num_vlans != 0)
+		SLIST_FOREACH(vf, sc->vlan_filters, next)
+			vf->flags = IXL_FILTER_ADD;
+	else { /* clean any stale filters */
+		while (!SLIST_EMPTY(sc->vlan_filters)) {
+			vf = SLIST_FIRST(sc->vlan_filters);
+			SLIST_REMOVE_HEAD(sc->vlan_filters, next);
+			free(vf, M_DEVBUF);
+		}
+	}
+
+	ixlv_enable_adminq_irq(hw);
+	sc->aq_pending = 0;
+	sc->aq_required = 0;
+
+	INIT_DBG_IF(ifp, "end");
+	return (error);
+}
+
+
+static void
+ixlv_init_locked(struct ixlv_sc *sc)
+{
+	struct i40e_hw		*hw = &sc->hw;
+	struct ixl_vsi		*vsi = &sc->vsi;
+	struct ixl_queue	*que = vsi->queues;
+	struct ifnet		*ifp = vsi->ifp;
+	int			 error = 0;
+
+	INIT_DBG_IF(ifp, "begin");
+
+	/* Verify we have the core lock */
+	if (!mtx_owned(&sc->mtx)) {
+		if_printf(ifp, "%s: sc mutex not owned; acquire"
+		    "before calling this function!\n", __func__);
+		goto init_done;
+	}
+
+	/* Do a reinit first if an init has already been done */
+	if ((sc->init_state == IXLV_RUNNING) ||
+	    (sc->init_state == IXLV_RESET_REQUIRED) ||
+	    (sc->init_state == IXLV_RESET_PENDING))
+		error = ixlv_reinit_locked(sc);
+	/* Don't bother with init if we failed reinit */
+	if (error)
+		goto init_done;
+
+	/* Check for an LAA mac address... */
+	bcopy(IF_LLADDR(ifp), hw->mac.addr, ETHER_ADDR_LEN);
+
+	ifp->if_hwassist = 0;
+	if (ifp->if_capenable & IFCAP_TSO)
+		ifp->if_hwassist |= CSUM_TSO;
+	if (ifp->if_capenable & IFCAP_TXCSUM)
+		ifp->if_hwassist |= (CSUM_OFFLOAD_IPV4 & ~CSUM_IP);
+	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
+		ifp->if_hwassist |= CSUM_OFFLOAD_IPV6;
+
+	/* Add mac filter for this VF to PF */
+	error = ixlv_add_mac_filter(sc, hw->mac.addr, 0);
+
+	// send message, then enqueue another task
+	if (!error || error == EEXIST) {
+		sc->aq_required |= IXLV_FLAG_AQ_ADD_MAC_FILTER;
+		callout_reset(&sc->aq_task, IXLV_CALLOUT_TIMO,
+		    ixlv_sched_aq, sc);
+	}
+
+	/* Setup vlan's if needed */
+	ixlv_setup_vlan_filters(sc);
+
+	/*
+	** Prepare the queues for operation
+	*/
+	for (int i = 0; i < vsi->num_queues; i++, que++) {
+		struct  rx_ring	*rxr = &que->rxr;
+
+		ixl_init_tx_ring(que);
+
+		/* Need to set mbuf size now */
+		if (vsi->max_frame_size <= 2048)
+			rxr->mbuf_sz = MCLBYTES;
+		else
+			rxr->mbuf_sz = MJUMPAGESIZE;
+		ixl_init_rx_ring(que);
+	}
+
+	/* Configure queues */
+	sc->aq_required |= IXLV_FLAG_AQ_CONFIGURE_QUEUES;
+	callout_reset(&sc->aq_task, IXLV_CALLOUT_TIMO,
+	    ixlv_sched_aq, sc);
+
+	/* Set up RSS */
+	ixlv_config_rss(sc);
+
+	/* Map vectors */
+	sc->aq_required |= IXLV_FLAG_AQ_MAP_VECTORS;
+	callout_reset(&sc->aq_task, IXLV_CALLOUT_TIMO,
+	    ixlv_sched_aq, sc);
+
+	/* Enable queues */
+	sc->aq_required |= IXLV_FLAG_AQ_ENABLE_QUEUES;
+	callout_reset(&sc->aq_task, IXLV_CALLOUT_TIMO,
+	    ixlv_sched_aq, sc);
+
+	/* Start the local timer */
+	callout_reset(&sc->timer, hz, ixlv_local_timer, sc);
+
+	sc->init_state = IXLV_RUNNING;
+
+init_done:
+	INIT_DBG_IF(ifp, "end");
+	return;
+}
+
+/*
+**  Init entry point for the stack
+*/
+void
+ixlv_init(void *arg)
+{
+	struct ixlv_sc *sc = arg;
+
+	mtx_lock(&sc->mtx);
+	ixlv_init_locked(sc);
+	mtx_unlock(&sc->mtx);
+	return;
+}
+
+/*
+ * Allocate MSI/X vectors, setup the AQ vector early
+ */
+static int
+ixlv_init_msix(struct ixlv_sc *sc)
+{
+	device_t dev = sc->dev;
+	int rid, want, vectors, queues, available;
+
+	rid = PCIR_BAR(IXL_BAR);
+	sc->msix_mem = bus_alloc_resource_any(dev,
+	    SYS_RES_MEMORY, &rid, RF_ACTIVE);
+       	if (!sc->msix_mem) {
+		/* May not be enabled */
+		device_printf(sc->dev,
+		    "Unable to map MSIX table \n");
+		goto fail;
+	}
+
+	available = pci_msix_count(dev); 
+	if (available == 0) { /* system has msix disabled */
+		bus_release_resource(dev, SYS_RES_MEMORY,
+		    rid, sc->msix_mem);
+		sc->msix_mem = NULL;
+		goto fail;
+	}
+
+	/* Figure out a reasonable auto config value */
+	queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus;
+
+	/* Override with hardcoded value if sane */
+	if ((ixlv_max_queues != 0) && (ixlv_max_queues <= queues)) 
+		queues = ixlv_max_queues;
+
+	/* Enforce the VF max value */
+	if (queues > IXLV_MAX_QUEUES)
+		queues = IXLV_MAX_QUEUES;
+
+	/*
+	** Want one vector (RX/TX pair) per queue
+	** plus an additional for the admin queue.
+	*/
+	want = queues + 1;
+	if (want <= available)	/* Have enough */
+		vectors = want;
+	else {
+		device_printf(sc->dev,
+		    "MSIX Configuration Problem, "
+		    "%d vectors available but %d wanted!\n",
+		    available, want);
+		goto fail;
+	}
+
+	if (pci_alloc_msix(dev, &vectors) == 0) {
+		device_printf(sc->dev,
+		    "Using MSIX interrupts with %d vectors\n", vectors);
+		sc->msix = vectors;
+		sc->vsi.num_queues = queues;
+	}
+
+	/*
+	** Explicitly set the guest PCI BUSMASTER capability
+	** and we must rewrite the ENABLE in the MSIX control
+	** register again at this point to cause the host to
+	** successfully initialize us.
+	*/
+	{
+		u16 pci_cmd_word;
+		int msix_ctrl;
+		pci_cmd_word = pci_read_config(dev, PCIR_COMMAND, 2);
+		pci_cmd_word |= PCIM_CMD_BUSMASTEREN;
+		pci_write_config(dev, PCIR_COMMAND, pci_cmd_word, 2);
+		pci_find_cap(dev, PCIY_MSIX, &rid);
+		rid += PCIR_MSIX_CTRL;
+		msix_ctrl = pci_read_config(dev, rid, 2);
+		msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
+		pci_write_config(dev, rid, msix_ctrl, 2);
+	}
+
+	/* Next we need to setup the vector for the Admin Queue */
+	rid = 1;	// zero vector + 1
+	sc->res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+	    &rid, RF_SHAREABLE | RF_ACTIVE);
+	if (sc->res == NULL) {
+		device_printf(dev,"Unable to allocate"
+		    " bus resource: AQ interrupt \n");
+		goto fail;
+	}
+	if (bus_setup_intr(dev, sc->res,
+	    INTR_TYPE_NET | INTR_MPSAFE, NULL,
+	    ixlv_msix_adminq, sc, &sc->tag)) {
+		sc->res = NULL;
+		device_printf(dev, "Failed to register AQ handler");
+		goto fail;
+	}
+	bus_describe_intr(dev, sc->res, sc->tag, "adminq");
+
+	return (vectors);
+
+fail:
+	/* The VF driver MUST use MSIX */
+	return (0);
+}
+
+static int
+ixlv_allocate_pci_resources(struct ixlv_sc *sc)
+{
+	int             rid;
+	device_t        dev = sc->dev;
+
+	rid = PCIR_BAR(0);
+	sc->pci_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &rid, RF_ACTIVE);
+
+	if (!(sc->pci_mem)) {
+		device_printf(dev,"Unable to allocate bus resource: memory\n");
+		return (ENXIO);
+	}
+
+	sc->osdep.mem_bus_space_tag =
+		rman_get_bustag(sc->pci_mem);
+	sc->osdep.mem_bus_space_handle =
+		rman_get_bushandle(sc->pci_mem);
+	sc->osdep.mem_bus_space_size = rman_get_size(sc->pci_mem);
+	sc->hw.hw_addr = (u8 *) &sc->osdep.mem_bus_space_handle;
+
+	sc->hw.back = &sc->osdep;
+
+	/* May need to pre-emptively disable adminq interrupts */
+	ixlv_disable_adminq_irq(&sc->hw);
+
+	/*
+	** Now setup MSI/X, it will return
+	** us the number of supported vectors
+	*/
+	sc->msix = ixlv_init_msix(sc);
+
+	/* We fail without MSIX support */
+	if (sc->msix == 0)
+		return (ENXIO);
+
+	return (0);
+}
+
+static void
+ixlv_free_pci_resources(struct ixlv_sc *sc)
+{
+	struct ixl_vsi         *vsi = &sc->vsi;
+	struct ixl_queue       *que = vsi->queues;
+	device_t                dev = sc->dev;
+
+	/* We may get here before stations are setup */
+	if (que == NULL)
+		goto early;
+
+	/*
+	**  Release all msix queue resources:
+	*/
+	for (int i = 0; i < vsi->num_queues; i++, que++) {
+		int rid = que->msix + 1;
+		if (que->tag != NULL) {
+			bus_teardown_intr(dev, que->res, que->tag);
+			que->tag = NULL;
+		}
+		if (que->res != NULL)
+			bus_release_resource(dev, SYS_RES_IRQ, rid, que->res);
+	}
+        
+early:
+	/* Clean the AdminQ interrupt */
+	if (sc->tag != NULL) {
+		bus_teardown_intr(dev, sc->res, sc->tag);
+		sc->tag = NULL;
+	}
+	if (sc->res != NULL)
+		bus_release_resource(dev, SYS_RES_IRQ, 1, sc->res);
+
+	pci_release_msi(dev);
+
+	if (sc->msix_mem != NULL)
+		bus_release_resource(dev, SYS_RES_MEMORY,
+		    PCIR_BAR(IXL_BAR), sc->msix_mem);
+
+	if (sc->pci_mem != NULL)
+		bus_release_resource(dev, SYS_RES_MEMORY,
+		    PCIR_BAR(0), sc->pci_mem);
+
+	return;
+}
+
+static int
+ixlv_init_taskqueue(struct ixlv_sc *sc)
+{
+	int error = 0;
+
+	/* Tasklet for AQ Interrupts */
+	TASK_INIT(&sc->aq_irq, 0, ixlv_do_adminq, sc);
+
+	sc->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT,
+	    taskqueue_thread_enqueue, &sc->tq);
+	taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s sc->tq",
+	    device_get_nameunit(sc->dev));
+
+	return (error);
+}
+
+/*********************************************************************
+ *
+ *  Setup MSIX Interrupt resources and handlers for the VSI queues
+ *
+ **********************************************************************/
+static int
+ixlv_assign_msix(struct ixlv_sc *sc)
+{
+	device_t	dev = sc->dev;
+	struct 		ixl_vsi *vsi = &sc->vsi;
+	struct 		ixl_queue *que = vsi->queues;
+	struct		tx_ring	 *txr;
+	int 		error, rid, vector = 1;
+
+	for (int i = 0; i < vsi->num_queues; i++, vector++, que++) {
+		rid = vector + 1;
+		txr = &que->txr;
+		que->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
+		    RF_SHAREABLE | RF_ACTIVE);
+		if (que->res == NULL) {
+			device_printf(dev,"Unable to allocate"
+		    	    " bus resource: que interrupt [%d]\n", vector);
+			return (ENXIO);
+		}
+		/* Set the handler function */
+		error = bus_setup_intr(dev, que->res,
+		    INTR_TYPE_NET | INTR_MPSAFE, NULL,
+		    ixlv_msix_que, que, &que->tag);
+		if (error) {
+			que->res = NULL;
+			device_printf(dev, "Failed to register que handler");
+			return (error);
+		}
+		bus_describe_intr(dev, que->res, que->tag, "que %d", i);
+		/* Bind the vector to a CPU */
+		bus_bind_intr(dev, que->res, i);
+		que->msix = vector;
+        	vsi->que_mask |= (u64)(1 << que->msix);
+		TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que);
+		TASK_INIT(&que->task, 0, ixlv_handle_que, que);
+		que->tq = taskqueue_create_fast("ixlv_que", M_NOWAIT,
+		    taskqueue_thread_enqueue, &que->tq);
+		taskqueue_start_threads(&que->tq, 1, PI_NET, "%s que",
+		    device_get_nameunit(sc->dev));
+	}
+
+	return (0);
+}
+
+/*
+** XXX: Assumes the vf's admin queue has been initialized.
+*/
+static int
+ixlv_reset(struct ixlv_sc *sc)
+{
+	struct i40e_hw	*hw = &sc->hw;
+	device_t	dev = sc->dev;
+	int		error = 0;
+
+	/* Ask the PF to reset us if we are initiating */
+	if (sc->init_state != IXLV_RESET_PENDING)
+		ixlv_request_reset(sc);
+
+	i40e_msec_delay(100);
+	error = ixlv_reset_complete(hw);
+	if (error) {
+		device_printf(dev, "%s: VF reset failed\n",
+		    __func__);
+		return (error);
+	}
+
+	error = i40e_shutdown_adminq(hw);
+	if (error) {
+		device_printf(dev, "%s: shutdown_adminq failed: %d\n",
+		    __func__, error);
+		return (error);
+	}
+
+	error = i40e_init_adminq(hw);
+	if (error) {
+		device_printf(dev, "%s: init_adminq failed: %d\n",
+		    __func__, error);
+		return(error);
+	}
+
+	return (0);
+}
+
+static int
+ixlv_reset_complete(struct i40e_hw *hw)
+{
+	u32 reg;
+
+	for (int i = 0; i < 100; i++) {
+		reg = rd32(hw, I40E_VFGEN_RSTAT) &
+		    I40E_VFGEN_RSTAT_VFR_STATE_MASK;
+
+                if ((reg == I40E_VFR_VFACTIVE) ||
+		    (reg == I40E_VFR_COMPLETED))
+			return (0);
+		i40e_usec_delay(20);
+	}
+
+	return (EBUSY);
+}
+
+
+/*********************************************************************
+ *
+ *  Setup networking device structure and register an interface.
+ *
+ **********************************************************************/
+static int
+ixlv_setup_interface(device_t dev, struct ixlv_sc *sc)
+{
+	struct ifnet		*ifp;
+	struct ixl_vsi		*vsi = &sc->vsi;
+	struct ixl_queue	*que = vsi->queues;
+
+	INIT_DBG_DEV(dev, "begin");
+
+	ifp = vsi->ifp = if_alloc(IFT_ETHER);
+	if (ifp == NULL) {
+		device_printf(dev, "can not allocate ifnet structure\n");
+		return (-1);
+	}
+
+	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
+
+	ifp->if_mtu = ETHERMTU;
+	ifp->if_baudrate = 4000000000;  // ??
+	ifp->if_init = ixlv_init;
+	ifp->if_softc = vsi;
+	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
+	ifp->if_ioctl = ixlv_ioctl;
+
+	ifp->if_transmit = ixl_mq_start;
+
+	ifp->if_qflush = ixl_qflush;
+	ifp->if_snd.ifq_maxlen = que->num_desc - 2;
+
+	ether_ifattach(ifp, sc->hw.mac.addr);
+
+	vsi->max_frame_size =
+	    ifp->if_mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
+	    + ETHER_VLAN_ENCAP_LEN;
+
+	/*
+	 * Tell the upper layer(s) we support long frames.
+	 */
+	ifp->if_data.ifi_hdrlen = sizeof(struct ether_vlan_header);
+
+	ifp->if_capabilities |= IFCAP_HWCSUM;
+	ifp->if_capabilities |= IFCAP_HWCSUM_IPV6;
+	ifp->if_capabilities |= IFCAP_TSO;
+	ifp->if_capabilities |= IFCAP_JUMBO_MTU;
+
+	ifp->if_capabilities |= IFCAP_VLAN_HWTAGGING
+			     |  IFCAP_VLAN_HWTSO
+			     |  IFCAP_VLAN_MTU
+			     |  IFCAP_VLAN_HWCSUM
+			     |  IFCAP_LRO;
+	ifp->if_capenable = ifp->if_capabilities;
+
+	/*
+	** Don't turn this on by default, if vlans are
+	** created on another pseudo device (eg. lagg)
+	** then vlan events are not passed thru, breaking
+	** operation, but with HW FILTER off it works. If
+	** using vlans directly on the ixl driver you can
+	** enable this and get full hardware tag filtering.
+	*/
+	ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;
+
+	/*
+	 * Specify the media types supported by this adapter and register
+	 * callbacks to update media and link information
+	 */
+	ifmedia_init(&sc->media, IFM_IMASK, ixlv_media_change,
+		     ixlv_media_status);
+
+	// JFV Add media types later?
+
+	ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
+	ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
+
+	INIT_DBG_DEV(dev, "end");
+	return (0);
+}
+
+/*
+** Allocate and setup the interface queues
+*/
+static int
+ixlv_setup_queues(struct ixlv_sc *sc)
+{
+	device_t		dev = sc->dev;
+	struct ixl_vsi		*vsi;
+	struct ixl_queue	*que;
+	struct tx_ring		*txr;
+	struct rx_ring		*rxr;
+	int 			rsize, tsize;
+	int			error = I40E_SUCCESS;
+
+	vsi = &sc->vsi;
+	vsi->back = (void *)sc;
+	vsi->hw = &sc->hw;
+	vsi->num_vlans = 0;
+
+	/* Get memory for the station queues */
+	if (!(vsi->queues =
+		(struct ixl_queue *) malloc(sizeof(struct ixl_queue) *
+		vsi->num_queues, M_DEVBUF, M_NOWAIT | M_ZERO))) {
+			device_printf(dev, "Unable to allocate queue memory\n");
+			error = ENOMEM;
+			goto early;
+	}
+
+	for (int i = 0; i < vsi->num_queues; i++) {
+		que = &vsi->queues[i];
+		que->num_desc = ixlv_ringsz;
+		que->me = i;
+		que->vsi = vsi;
+		/* mark the queue as active */
+		vsi->active_queues |= (u64)1 << que->me;
+
+		txr = &que->txr;
+		txr->que = que;
+		txr->tail = I40E_QTX_TAIL1(que->me);
+		/* Initialize the TX lock */
+		snprintf(txr->mtx_name, sizeof(txr->mtx_name), "%s:tx(%d)",
+		    device_get_nameunit(dev), que->me);
+		mtx_init(&txr->mtx, txr->mtx_name, NULL, MTX_DEF);
+		/*
+		** Create the TX descriptor ring, the extra int is
+		** added as the location for HEAD WB.
+		*/
+		tsize = roundup2((que->num_desc *
+		    sizeof(struct i40e_tx_desc)) +
+		    sizeof(u32), DBA_ALIGN);
+		if (i40e_allocate_dma(&sc->hw,
+		    &txr->dma, tsize, DBA_ALIGN)) {
+			device_printf(dev,
+			    "Unable to allocate TX Descriptor memory\n");
+			error = ENOMEM;
+			goto fail;
+		}
+		txr->base = (struct i40e_tx_desc *)txr->dma.va;
+		bzero((void *)txr->base, tsize);
+		/* Now allocate transmit soft structs for the ring */
+		if (ixl_allocate_tx_data(que)) {
+			device_printf(dev,
+			    "Critical Failure setting up TX structures\n");
+			error = ENOMEM;
+			goto fail;
+		}
+		/* Allocate a buf ring */
+		txr->br = buf_ring_alloc(ixlv_txbrsz, M_DEVBUF,
+		    M_WAITOK, &txr->mtx);
+		if (txr->br == NULL) {
+			device_printf(dev,
+			    "Critical Failure setting up TX buf ring\n");
+			error = ENOMEM;
+			goto fail;
+		}
+
+		/*
+		 * Next the RX queues...
+		 */ 
+		rsize = roundup2(que->num_desc *
+		    sizeof(union i40e_rx_desc), DBA_ALIGN);
+		rxr = &que->rxr;
+		rxr->que = que;
+		rxr->tail = I40E_QRX_TAIL1(que->me);
+
+		/* Initialize the RX side lock */
+		snprintf(rxr->mtx_name, sizeof(rxr->mtx_name), "%s:rx(%d)",
+		    device_get_nameunit(dev), que->me);
+		mtx_init(&rxr->mtx, rxr->mtx_name, NULL, MTX_DEF);
+
+		if (i40e_allocate_dma(&sc->hw,
+		    &rxr->dma, rsize, 4096)) { //JFV - should this be DBA?
+			device_printf(dev,
+			    "Unable to allocate RX Descriptor memory\n");
+			error = ENOMEM;
+			goto fail;
+		}
+		rxr->base = (union i40e_rx_desc *)rxr->dma.va;
+		bzero((void *)rxr->base, rsize);
+
+		/* Allocate receive soft structs for the ring*/
+		if (ixl_allocate_rx_data(que)) {
+			device_printf(dev,
+			    "Critical Failure setting up receive structs\n");
+			error = ENOMEM;
+			goto fail;
+		}
+	}
+
+	return (0);
+
+fail:
+	free(vsi->queues, M_DEVBUF);
+	for (int i = 0; i < vsi->num_queues; i++) {
+		que = &vsi->queues[i];
+		rxr = &que->rxr;
+		txr = &que->txr;
+		if (rxr->base)
+			i40e_free_dma(&sc->hw, &rxr->dma);
+		if (txr->base)
+			i40e_free_dma(&sc->hw, &txr->dma);
+	}
+
+early:
+	return (error);
+}
+
+/*
+** This routine is run via an vlan config EVENT,
+** it enables us to use the HW Filter table since
+** we can get the vlan id. This just creates the
+** entry in the soft version of the VFTA, init will
+** repopulate the real table.
+*/
+static void
+ixlv_register_vlan(void *arg, struct ifnet *ifp, u16 vtag)
+{
+	struct ixl_vsi			*vsi = ifp->if_softc;
+	struct ixlv_sc		*sc = vsi->back;
+	struct ixlv_vlan_filter	*v;
+
+
+	if (ifp->if_softc !=  arg)   /* Not our event */
+		return;
+
+	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
+		return;
+
+	/* Sanity check - make sure it doesn't already exist */
+	SLIST_FOREACH(v, sc->vlan_filters, next) {
+		if (v->vlan == vtag)
+			return;
+	}
+
+	mtx_lock(&sc->mtx);
+	++vsi->num_vlans;
+	v = malloc(sizeof(struct ixlv_vlan_filter), M_DEVBUF, M_NOWAIT | M_ZERO);
+	SLIST_INSERT_HEAD(sc->vlan_filters, v, next);
+	v->vlan = vtag;
+	v->flags = IXL_FILTER_ADD;
+	sc->aq_required |= IXLV_FLAG_AQ_ADD_VLAN_FILTER;
+	mtx_unlock(&sc->mtx);
+	return;
+}
+
+/*
+** This routine is run via an vlan
+** unconfig EVENT, remove our entry
+** in the soft vfta.
+*/
+static void
+ixlv_unregister_vlan(void *arg, struct ifnet *ifp, u16 vtag)
+{
+	struct ixl_vsi			*vsi = ifp->if_softc;
+	struct ixlv_sc		*sc = vsi->back;
+	struct ixlv_vlan_filter	*v;
+	int				i = 0;
+	
+	if (ifp->if_softc !=  arg)
+		return;
+
+	if ((vtag == 0) || (vtag > 4095))	/* Invalid */
+		return;
+
+	mtx_lock(&sc->mtx);
+	SLIST_FOREACH(v, sc->vlan_filters, next) {
+		if (v->vlan == vtag) {
+			v->flags = IXL_FILTER_DEL;
+			++i;
+			--vsi->num_vlans;
+		}
+	}
+	if (i)
+		sc->aq_required |= IXLV_FLAG_AQ_DEL_VLAN_FILTER;
+	mtx_unlock(&sc->mtx);
+	return;
+}
+
+/*
+** Get a new filter and add it to the mac filter list.
+*/
+static struct ixlv_mac_filter *
+ixlv_get_mac_filter(struct ixlv_sc *sc)
+{
+	struct ixlv_mac_filter	*f;
+
+	f = malloc(sizeof(struct ixlv_mac_filter), M_DEVBUF, M_NOWAIT | M_ZERO);
+	SLIST_INSERT_HEAD(sc->mac_filters, f, next);
+
+	return (f);
+}
+
+/*
+** Find the filter with matching MAC address
+*/
+static struct ixlv_mac_filter *
+ixlv_find_mac_filter(struct ixlv_sc *sc, u8 *macaddr)
+{
+	struct ixlv_mac_filter	*f;
+	bool				match = FALSE;
+
+	SLIST_FOREACH(f, sc->mac_filters, next) {
+		if (cmp_etheraddr(f->macaddr, macaddr)) {
+			match = TRUE;
+			break;
+		}
+	}	
+
+	if (!match)
+		f = NULL;
+	return (f);
+}
+
+/*
+** Admin Queue interrupt handler
+*/
+static void
+ixlv_msix_adminq(void *arg)
+{
+	struct ixlv_sc	*sc = arg;
+	struct i40e_hw		*hw = &sc->hw;
+	u32			reg, mask;
+
+        reg = rd32(hw, I40E_VFINT_ICR01);
+        mask = rd32(hw, I40E_VFINT_ICR0_ENA1);
+
+        reg = rd32(hw, I40E_VFINT_DYN_CTL01);
+        reg |= I40E_PFINT_DYN_CTL0_CLEARPBA_MASK;
+        wr32(hw, I40E_VFINT_DYN_CTL01, reg);
+
+        /* re-enable interrupt causes */
+        wr32(hw, I40E_VFINT_ICR0_ENA1, mask);
+        wr32(hw, I40E_VFINT_DYN_CTL01, I40E_VFINT_DYN_CTL01_INTENA_MASK);
+
+	/* schedule task */
+	taskqueue_enqueue(sc->tq, &sc->aq_irq);
+	return;
+}
+
+void
+ixlv_enable_intr(struct ixl_vsi *vsi)
+{
+	struct i40e_hw		*hw = vsi->hw;
+	struct ixl_queue	*que = vsi->queues;
+
+	ixlv_enable_adminq_irq(hw);
+	for (int i = 0; i < vsi->num_queues; i++, que++)
+		ixlv_enable_queue_irq(hw, que->me);
+}
+
+void
+ixlv_disable_intr(struct ixl_vsi *vsi)
+{
+        struct i40e_hw          *hw = vsi->hw;
+        struct ixl_queue       *que = vsi->queues;
+
+	ixlv_disable_adminq_irq(hw);
+	for (int i = 0; i < vsi->num_queues; i++, que++)
+		ixlv_disable_queue_irq(hw, que->me);
+}
+
+
+static void
+ixlv_disable_adminq_irq(struct i40e_hw *hw)
+{
+	wr32(hw, I40E_VFINT_DYN_CTL01, 0);
+	wr32(hw, I40E_VFINT_ICR0_ENA1, 0);
+	/* flush */
+	rd32(hw, I40E_VFGEN_RSTAT);
+	return;
+}
+
+static void
+ixlv_enable_adminq_irq(struct i40e_hw *hw)
+{
+	wr32(hw, I40E_VFINT_DYN_CTL01,
+	    I40E_VFINT_DYN_CTL01_INTENA_MASK |
+	    I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
+	wr32(hw, I40E_VFINT_ICR0_ENA1, I40E_VFINT_ICR0_ENA_ADMINQ_MASK);
+	/* flush */
+	rd32(hw, I40E_VFGEN_RSTAT);
+	return;
+}
+
+static void
+ixlv_enable_queue_irq(struct i40e_hw *hw, int id)
+{
+	u32		reg;
+
+	reg = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
+	    I40E_VFINT_DYN_CTLN_CLEARPBA_MASK; 
+	wr32(hw, I40E_VFINT_DYN_CTLN1(id), reg);
+}
+
+static void
+ixlv_disable_queue_irq(struct i40e_hw *hw, int id)
+{
+	wr32(hw, I40E_VFINT_DYN_CTLN1(id), 0);
+	rd32(hw, I40E_VFGEN_RSTAT);
+	return;
+}
+
+
+/*
+** Provide a update to the queue RX
+** interrupt moderation value.
+*/
+static void
+ixlv_set_queue_rx_itr(struct ixl_queue *que)
+{
+	struct ixl_vsi	*vsi = que->vsi;
+	struct i40e_hw	*hw = vsi->hw;
+	struct rx_ring	*rxr = &que->rxr;
+	u16		rx_itr;
+	u16		rx_latency = 0;
+	int		rx_bytes;
+
+
+	/* Idle, do nothing */
+	if (rxr->bytes == 0)
+		return;
+
+	if (ixlv_dynamic_rx_itr) {
+		rx_bytes = rxr->bytes/rxr->itr;
+		rx_itr = rxr->itr;
+
+		/* Adjust latency range */
+		switch (rxr->latency) {
+		case IXL_LOW_LATENCY:
+			if (rx_bytes > 10) {
+				rx_latency = IXL_AVE_LATENCY;
+				rx_itr = IXL_ITR_20K;
+			}
+			break;
+		case IXL_AVE_LATENCY:
+			if (rx_bytes > 20) {
+				rx_latency = IXL_BULK_LATENCY;
+				rx_itr = IXL_ITR_8K;
+			} else if (rx_bytes <= 10) {
+				rx_latency = IXL_LOW_LATENCY;
+				rx_itr = IXL_ITR_100K;
+			}
+			break;
+		case IXL_BULK_LATENCY:
+			if (rx_bytes <= 20) {
+				rx_latency = IXL_AVE_LATENCY;
+				rx_itr = IXL_ITR_20K;
+			}
+			break;
+       		 }
+
+		rxr->latency = rx_latency;
+
+		if (rx_itr != rxr->itr) {
+			/* do an exponential smoothing */
+			rx_itr = (10 * rx_itr * rxr->itr) /
+			    ((9 * rx_itr) + rxr->itr);
+			rxr->itr = rx_itr & IXL_MAX_ITR;
+			wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR,
+			    que->me), rxr->itr);
+		}
+	} else { /* We may have have toggled to non-dynamic */
+		if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC)
+			vsi->rx_itr_setting = ixlv_rx_itr;
+		/* Update the hardware if needed */
+		if (rxr->itr != vsi->rx_itr_setting) {
+			rxr->itr = vsi->rx_itr_setting;
+			wr32(hw, I40E_VFINT_ITRN1(IXL_RX_ITR,
+			    que->me), rxr->itr);
+		}
+	}
+	rxr->bytes = 0;
+	rxr->packets = 0;
+	return;
+}
+
+
+/*
+** Provide a update to the queue TX
+** interrupt moderation value.
+*/
+static void
+ixlv_set_queue_tx_itr(struct ixl_queue *que)
+{
+	struct ixl_vsi	*vsi = que->vsi;
+	struct i40e_hw	*hw = vsi->hw;
+	struct tx_ring	*txr = &que->txr;
+	u16		tx_itr;
+	u16		tx_latency = 0;
+	int		tx_bytes;
+
+
+	/* Idle, do nothing */
+	if (txr->bytes == 0)
+		return;
+
+	if (ixlv_dynamic_tx_itr) {
+		tx_bytes = txr->bytes/txr->itr;
+		tx_itr = txr->itr;
+
+		switch (txr->latency) {
+		case IXL_LOW_LATENCY:
+			if (tx_bytes > 10) {
+				tx_latency = IXL_AVE_LATENCY;
+				tx_itr = IXL_ITR_20K;
+			}
+			break;
+		case IXL_AVE_LATENCY:
+			if (tx_bytes > 20) {
+				tx_latency = IXL_BULK_LATENCY;
+				tx_itr = IXL_ITR_8K;
+			} else if (tx_bytes <= 10) {
+				tx_latency = IXL_LOW_LATENCY;
+				tx_itr = IXL_ITR_100K;
+			}
+			break;
+		case IXL_BULK_LATENCY:
+			if (tx_bytes <= 20) {
+				tx_latency = IXL_AVE_LATENCY;
+				tx_itr = IXL_ITR_20K;
+			}
+			break;
+		}
+
+		txr->latency = tx_latency;
+
+		if (tx_itr != txr->itr) {
+       	         /* do an exponential smoothing */
+			tx_itr = (10 * tx_itr * txr->itr) /
+			    ((9 * tx_itr) + txr->itr);
+			txr->itr = tx_itr & IXL_MAX_ITR;
+			wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR,
+			    que->me), txr->itr);
+		}
+
+	} else { /* We may have have toggled to non-dynamic */
+		if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC)
+			vsi->tx_itr_setting = ixlv_tx_itr;
+		/* Update the hardware if needed */
+		if (txr->itr != vsi->tx_itr_setting) {
+			txr->itr = vsi->tx_itr_setting;
+			wr32(hw, I40E_VFINT_ITRN1(IXL_TX_ITR,
+			    que->me), txr->itr);
+		}
+	}
+	txr->bytes = 0;
+	txr->packets = 0;
+	return;
+}
+
+
+/*
+**
+** MSIX Interrupt Handlers and Tasklets
+**
+*/
+static void
+ixlv_handle_que(void *context, int pending)
+{
+	struct ixl_queue *que = context;
+	struct ixl_vsi *vsi = que->vsi;
+	struct i40e_hw  *hw = vsi->hw;
+	struct tx_ring  *txr = &que->txr;
+	struct ifnet    *ifp = vsi->ifp;
+	bool		more;
+
+	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
+		more = ixl_rxeof(que, IXL_RX_LIMIT);
+		mtx_lock(&txr->mtx);
+		ixl_txeof(que);
+		if (!drbr_empty(ifp, txr->br))
+			ixl_mq_start_locked(ifp, txr);
+		mtx_unlock(&txr->mtx);
+		if (more) {
+			taskqueue_enqueue(que->tq, &que->task);
+			return;
+		}
+	}
+
+	/* Reenable this interrupt - hmmm */
+	ixlv_enable_queue_irq(hw, que->me);
+	return;
+}
+
+
+/*********************************************************************
+ *
+ *  MSIX Queue Interrupt Service routine
+ *
+ **********************************************************************/
+static void
+ixlv_msix_que(void *arg)
+{
+	struct ixl_queue	*que = arg;
+	struct ixl_vsi	*vsi = que->vsi;
+	struct i40e_hw	*hw = vsi->hw;
+	struct tx_ring	*txr = &que->txr;
+	bool		more_tx, more_rx;
+
+	/* Spurious interrupts are ignored */
+	if (!(vsi->ifp->if_drv_flags & IFF_DRV_RUNNING))
+		return;
+
+	++que->irqs;
+
+	more_rx = ixl_rxeof(que, IXL_RX_LIMIT);
+
+	mtx_lock(&txr->mtx);
+	more_tx = ixl_txeof(que);
+	/*
+	** Make certain that if the stack 
+	** has anything queued the task gets
+	** scheduled to handle it.
+	*/
+	if (!drbr_empty(vsi->ifp, txr->br))
+		more_tx = 1;
+	mtx_unlock(&txr->mtx);
+
+	ixlv_set_queue_rx_itr(que);
+	ixlv_set_queue_tx_itr(que);
+
+	if (more_tx || more_rx)
+		taskqueue_enqueue(que->tq, &que->task);
+	else
+		ixlv_enable_queue_irq(hw, que->me);
+
+	return;
+}
+
+
+/*********************************************************************
+ *
+ *  Media Ioctl callback
+ *
+ *  This routine is called whenever the user queries the status of
+ *  the interface using ifconfig.
+ *
+ **********************************************************************/
+static void
+ixlv_media_status(struct ifnet * ifp, struct ifmediareq * ifmr)
+{
+	struct ixl_vsi		*vsi = ifp->if_softc;
+	struct ixlv_sc	*sc = vsi->back;
+
+	INIT_DBG_IF(ifp, "begin");
+
+	mtx_lock(&sc->mtx);
+
+	ixlv_update_link_status(sc);
+
+	ifmr->ifm_status = IFM_AVALID;
+	ifmr->ifm_active = IFM_ETHER;
+
+	if (!vsi->link_up) {
+		mtx_unlock(&sc->mtx);
+		INIT_DBG_IF(ifp, "end: link not up");
+		return;
+	}
+
+	ifmr->ifm_status |= IFM_ACTIVE;
+	/* Hardware is always full-duplex */
+	ifmr->ifm_active |= IFM_FDX;
+	mtx_unlock(&sc->mtx);
+	INIT_DBG_IF(ifp, "end");
+	return;
+}
+
+/*********************************************************************
+ *
+ *  Media Ioctl callback
+ *
+ *  This routine is called when the user changes speed/duplex using
+ *  media/mediopt option with ifconfig.
+ *
+ **********************************************************************/
+static int
+ixlv_media_change(struct ifnet * ifp)
+{
+	struct ixl_vsi *vsi = ifp->if_softc;
+	struct ifmedia *ifm = &vsi->media;
+
+	INIT_DBG_IF(ifp, "begin");
+
+	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
+		return (EINVAL);
+
+	INIT_DBG_IF(ifp, "end");
+	return (0);
+}
+
+
+/*********************************************************************
+ *  Multicast Initialization
+ *
+ *  This routine is called by init to reset a fresh state.
+ *
+ **********************************************************************/
+
+static void
+ixlv_init_multi(struct ixl_vsi *vsi)
+{
+	struct ixlv_mac_filter *f;
+	struct ixlv_sc	*sc = vsi->back;
+	int			mcnt = 0;
+
+	IOCTL_DBG_IF(vsi->ifp, "begin");
+
+	/* First clear any multicast filters */
+	SLIST_FOREACH(f, sc->mac_filters, next) {
+		if ((f->flags & IXL_FILTER_USED)
+		    && (f->flags & IXL_FILTER_MC)) {
+			f->flags |= IXL_FILTER_DEL;
+			mcnt++;
+		}
+	}
+	if (mcnt > 0)
+		sc->aq_required |= IXLV_FLAG_AQ_DEL_MAC_FILTER;
+
+	IOCTL_DBG_IF(vsi->ifp, "end");
+}
+
+static void
+ixlv_add_multi(struct ixl_vsi *vsi)
+{
+	struct ifmultiaddr	*ifma;
+	struct ifnet		*ifp = vsi->ifp;
+	struct ixlv_sc	*sc = vsi->back;
+	int			mcnt = 0;
+
+	IOCTL_DBG_IF(ifp, "begin");
+
+	if_maddr_rlock(ifp);
+	/*
+	** Get a count, to decide if we
+	** simply use multicast promiscuous.
+	*/
+	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+		if (ifma->ifma_addr->sa_family != AF_LINK)
+			continue;
+		mcnt++;
+	}
+	if_maddr_runlock(ifp);
+
+	if (__predict_false(mcnt >= MAX_MULTICAST_ADDR)) {
+		/* delete all multicast filters */
+		ixlv_init_multi(vsi);
+		sc->promiscuous_flags |= I40E_FLAG_VF_MULTICAST_PROMISC;
+		sc->aq_required |= IXLV_FLAG_AQ_CONFIGURE_PROMISC;
+		IOCTL_DEBUGOUT("%s: end: too many filters", __func__);
+		return;
+	}
+
+	mcnt = 0;
+	if_maddr_rlock(ifp);
+	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+		if (ifma->ifma_addr->sa_family != AF_LINK)
+			continue;
+		if (!ixlv_add_mac_filter(sc,
+		    (u8*)LLADDR((struct sockaddr_dl *) ifma->ifma_addr),
+		    IXL_FILTER_MC))
+			mcnt++;
+	}
+	if_maddr_runlock(ifp);
+	/*
+	** Notify AQ task that sw filters need to be
+	** added to hw list
+	*/
+	if (mcnt > 0)
+		sc->aq_required |= IXLV_FLAG_AQ_ADD_MAC_FILTER;
+
+	IOCTL_DBG_IF(ifp, "end");
+}
+
+static void
+ixlv_del_multi(struct ixl_vsi *vsi)
+{
+	struct ixlv_mac_filter *f;
+	struct ifmultiaddr	*ifma;
+	struct ifnet		*ifp = vsi->ifp;
+	struct ixlv_sc	*sc = vsi->back;
+	int			mcnt = 0;
+	bool		match = FALSE;
+
+	IOCTL_DBG_IF(ifp, "begin");
+
+	/* Search for removed multicast addresses */
+	if_maddr_rlock(ifp);
+	SLIST_FOREACH(f, sc->mac_filters, next) {
+		if ((f->flags & IXL_FILTER_USED)
+		    && (f->flags & IXL_FILTER_MC)) {
+			/* check if mac address in filter is in sc's list */
+			match = FALSE;
+			TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
+				if (ifma->ifma_addr->sa_family != AF_LINK)
+					continue;
+				u8 *mc_addr =
+				    (u8 *)LLADDR((struct sockaddr_dl *)ifma->ifma_addr);
+				if (cmp_etheraddr(f->macaddr, mc_addr)) {
+					match = TRUE;
+					break;
+				}
+			}
+			/* if this filter is not in the sc's list, remove it */
+			if (match == FALSE && !(f->flags & IXL_FILTER_DEL)) {
+				f->flags |= IXL_FILTER_DEL;
+				mcnt++;
+				IOCTL_DBG_IF(ifp, "marked: " MAC_FORMAT,
+				    MAC_FORMAT_ARGS(f->macaddr));
+			}
+			else if (match == FALSE)
+				IOCTL_DBG_IF(ifp, "exists: " MAC_FORMAT,
+				    MAC_FORMAT_ARGS(f->macaddr));
+		}
+	}
+	if_maddr_runlock(ifp);
+
+	if (mcnt > 0)
+		sc->aq_required |= IXLV_FLAG_AQ_DEL_MAC_FILTER;
+
+	IOCTL_DBG_IF(ifp, "end");
+}
+
+/*********************************************************************
+ *  Timer routine
+ *
+ *  This routine checks for link status,updates statistics,
+ *  and runs the watchdog check.
+ *
+ **********************************************************************/
+
+static void
+ixlv_local_timer(void *arg)
+{
+	struct ixlv_sc	*sc = arg;
+	struct i40e_hw		*hw = &sc->hw;
+	struct ixl_vsi		*vsi = &sc->vsi;
+	struct ixl_queue	*que = vsi->queues;
+	device_t		dev = sc->dev;
+	int			hung = 0;
+	u32			mask, val, oldval;
+
+	mtx_assert(&sc->mtx, MA_OWNED);
+
+	/* If Reset is in progress just bail */
+	if (sc->init_state == IXLV_RESET_PENDING)
+		return;
+
+	/* Check for when PF triggers a VF reset */
+	val = rd32(hw, I40E_VFGEN_RSTAT) &
+	    I40E_VFGEN_RSTAT_VFR_STATE_MASK;
+
+	if (val != I40E_VFR_VFACTIVE
+	    && val != I40E_VFR_COMPLETED) {
+#ifdef IXL_DEBUG
+		device_printf(dev, "%s: reset in progress! (%d)\n",
+		    __func__, val);
+#endif
+		return;
+	}
+
+	/* check for Admin queue errors */
+	val = rd32(hw, hw->aq.arq.len);
+	oldval = val;
+	if (val & I40E_VF_ARQLEN_ARQVFE_MASK) {
+		device_printf(dev, "ARQ VF Error detected\n");
+		val &= ~I40E_VF_ARQLEN_ARQVFE_MASK;
+	}
+	if (val & I40E_VF_ARQLEN_ARQOVFL_MASK) {
+		device_printf(dev, "ARQ Overflow Error detected\n");
+		val &= ~I40E_VF_ARQLEN_ARQOVFL_MASK;
+	}
+	if (val & I40E_VF_ARQLEN_ARQCRIT_MASK) {
+		device_printf(dev, "ARQ Critical Error detected\n");
+		val &= ~I40E_VF_ARQLEN_ARQCRIT_MASK;
+	}
+	if (oldval != val)
+		wr32(hw, hw->aq.arq.len, val);
+
+	val = rd32(hw, hw->aq.asq.len);
+	oldval = val;
+	if (val & I40E_VF_ATQLEN_ATQVFE_MASK) {
+		device_printf(dev, "ASQ VF Error detected\n");
+		val &= ~I40E_VF_ATQLEN_ATQVFE_MASK;
+	}
+	if (val & I40E_VF_ATQLEN_ATQOVFL_MASK) {
+		device_printf(dev, "ASQ Overflow Error detected\n");
+		val &= ~I40E_VF_ATQLEN_ATQOVFL_MASK;
+	}
+	if (val & I40E_VF_ATQLEN_ATQCRIT_MASK) {
+		device_printf(dev, "ASQ Critical Error detected\n");
+		val &= ~I40E_VF_ATQLEN_ATQCRIT_MASK;
+	}
+	if (oldval != val)
+		wr32(hw, hw->aq.asq.len, val);
+
+	/* clean and process any events */
+	taskqueue_enqueue(sc->tq, &sc->aq_irq);
+
+	/*
+	** Check status on the queues for a hang
+	*/
+	mask = (I40E_VFINT_DYN_CTLN_INTENA_MASK |
+	    I40E_VFINT_DYN_CTLN_SWINT_TRIG_MASK);
+
+	for (int i = 0; i < vsi->num_queues; i++,que++) {
+		/* Any queues with outstanding work get a sw irq */
+		if (que->busy)
+			wr32(hw, I40E_VFINT_DYN_CTLN1(que->me), mask);
+		/*
+		** Each time txeof runs without cleaning, but there
+		** are uncleaned descriptors it increments busy. If
+		** we get to 5 we declare it hung.
+		*/
+		if (que->busy == IXL_QUEUE_HUNG) {
+			++hung;
+			/* Mark the queue as inactive */
+			vsi->active_queues &= ~((u64)1 << que->me);
+			continue;
+		} else {
+			/* Check if we've come back from hung */
+			if ((vsi->active_queues & ((u64)1 << que->me)) == 0)
+     				vsi->active_queues |= ((u64)1 << que->me);
+		}
+		if (que->busy >= IXL_MAX_TX_BUSY) {
+			device_printf(dev,"Warning queue %d "
+			    "appears to be hung!\n", i);
+			que->busy = IXL_QUEUE_HUNG;
+			++hung;
+		}
+	}
+	/* Only reset when all queues show hung */
+	if (hung == vsi->num_queues)
+		goto hung;
+	callout_reset(&sc->timer, hz, ixlv_local_timer, sc);
+	return;
+
+hung:
+	device_printf(dev, "Local Timer: TX HANG DETECTED - Resetting!!\n");
+	sc->init_state = IXLV_RESET_REQUIRED;
+	ixlv_init_locked(sc);
+}
+
+/*
+** Note: this routine updates the OS on the link state
+**	the real check of the hardware only happens with
+**	a link interrupt.
+*/
+static void
+ixlv_update_link_status(struct ixlv_sc *sc)
+{
+	struct ixl_vsi		*vsi = &sc->vsi;
+	struct ifnet		*ifp = vsi->ifp;
+	device_t		 dev = sc->dev;
+
+	if (vsi->link_up){ 
+		if (vsi->link_active == FALSE) {
+			if (bootverbose)
+				device_printf(dev,"Link is Up, %d Gbps\n",
+				    (vsi->link_speed == I40E_LINK_SPEED_40GB) ? 40:10);
+			vsi->link_active = TRUE;
+			if_link_state_change(ifp, LINK_STATE_UP);
+		}
+	} else { /* Link down */
+		if (vsi->link_active == TRUE) {
+			if (bootverbose)
+				device_printf(dev,"Link is Down\n");
+			if_link_state_change(ifp, LINK_STATE_DOWN);
+			vsi->link_active = FALSE;
+		}
+	}
+
+	return;
+}
+
+/*********************************************************************
+ *
+ *  This routine disables all traffic on the adapter by issuing a
+ *  global reset on the MAC and deallocates TX/RX buffers.
+ *
+ **********************************************************************/
+
+static void
+ixlv_stop(struct ixlv_sc *sc)
+{
+	mtx_assert(&sc->sc_mtx, MA_OWNED);
+
+	INIT_DBG_IF(&sc->vsi->ifp, "begin");
+
+	sc->aq_required |= IXLV_FLAG_AQ_DISABLE_QUEUES;
+	callout_reset(&sc->aq_task, IXLV_CALLOUT_TIMO,
+	    ixlv_sched_aq, sc);
+
+	/* Stop the local timer */
+	callout_stop(&sc->timer);
+
+	INIT_DBG_IF(&sc->vsi->ifp, "end");
+}
+
+
+/*********************************************************************
+ *
+ *  Free all station queue structs.
+ *
+ **********************************************************************/
+static void
+ixlv_free_queues(struct ixl_vsi *vsi)
+{
+	struct ixlv_sc	*sc = (struct ixlv_sc *)vsi->back;
+	struct ixl_queue	*que = vsi->queues;
+
+	for (int i = 0; i < vsi->num_queues; i++, que++) {
+		struct tx_ring *txr = &que->txr;
+		struct rx_ring *rxr = &que->rxr;
+	
+		if (!mtx_initialized(&txr->mtx)) /* uninitialized */
+			continue;
+		IXL_TX_LOCK(txr);
+		ixl_free_que_tx(que);
+		if (txr->base)
+			i40e_free_dma(&sc->hw, &txr->dma);
+		IXL_TX_UNLOCK(txr);
+		IXL_TX_LOCK_DESTROY(txr);
+
+		if (!mtx_initialized(&rxr->mtx)) /* uninitialized */
+			continue;
+		IXL_RX_LOCK(rxr);
+		ixl_free_que_rx(que);
+		if (rxr->base)
+			i40e_free_dma(&sc->hw, &rxr->dma);
+		IXL_RX_UNLOCK(rxr);
+		IXL_RX_LOCK_DESTROY(rxr);
+		
+	}
+	free(vsi->queues, M_DEVBUF);
+}
+
+
+/*
+** ixlv_config_rss - setup RSS 
+*/
+static void
+ixlv_config_rss(struct ixlv_sc *sc)
+{
+	struct i40e_hw	*hw = &sc->hw;
+	struct ixl_vsi	*vsi = &sc->vsi;
+	u32		lut = 0;
+	u64		set_hena, hena;
+	int		i, j;
+
+	/* set up random bits */
+	static const u32 seed[I40E_VFQF_HKEY_MAX_INDEX + 1] = {
+	    0x794221b4, 0xbca0c5ab, 0x6cd5ebd9, 0x1ada6127,
+	    0x983b3aa1, 0x1c4e71eb, 0x7f6328b2, 0xfcdc0da0,
+	    0xc135cafa, 0x7a6f7e2d, 0xe7102d28, 0x163cd12e,
+	    0x4954b126 };
+
+	/* Fill out hash function seed */
+	for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
+                wr32(hw, I40E_VFQF_HKEY(i), seed[i]);
+
+	/* Enable PCTYPES for RSS: */
+	set_hena =
+		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_UDP) |
+		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_TCP) |
+		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_SCTP) |
+		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV4_OTHER) |
+		((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV4) |
+		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_UDP) |
+		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_TCP) |
+		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_SCTP) |
+		((u64)1 << I40E_FILTER_PCTYPE_NONF_IPV6_OTHER) |
+		((u64)1 << I40E_FILTER_PCTYPE_FRAG_IPV6) |
+		((u64)1 << I40E_FILTER_PCTYPE_L2_PAYLOAD);
+
+	hena = (u64)rd32(hw, I40E_VFQF_HENA(0)) |
+	    ((u64)rd32(hw, I40E_VFQF_HENA(1)) << 32);
+	hena |= set_hena;
+	wr32(hw, I40E_VFQF_HENA(0), (u32)hena);
+	wr32(hw, I40E_VFQF_HENA(1), (u32)(hena >> 32));
+
+	/* Populate the LUT with max no. of queues in round robin fashion */
+	for (i = j = 0; i < hw->func_caps.rss_table_size; i++, j++) {
+                if (j == vsi->num_queues)
+                        j = 0;
+                /* lut = 4-byte sliding window of 4 lut entries */
+                lut = (lut << 8) | (j &
+                         ((0x1 << hw->func_caps.rss_table_entry_width) - 1));
+                /* On i = 3, we have 4 entries in lut; write to the register */
+                if ((i & 3) == 3)
+                        wr32(hw, I40E_VFQF_HLUT(i >> 2), lut);
+        }
+	ixl_flush(hw);
+}
+
+
+/*
+** This routine refreshes vlan filters, called by init
+** it scans the filter table and then updates the AQ
+*/
+static void
+ixlv_setup_vlan_filters(struct ixlv_sc *sc)
+{
+	struct ixl_vsi			*vsi = &sc->vsi;
+	struct ixlv_vlan_filter	*f;
+	int				cnt = 0;
+
+	if (vsi->num_vlans == 0)
+		return;
+	/*
+	** Scan the filter table for vlan entries,
+	** and if found call for the AQ update.
+	*/
+	SLIST_FOREACH(f, sc->vlan_filters, next)
+                if (f->flags & IXL_FILTER_ADD)
+			cnt++;
+	if (cnt == 0)
+		return;
+
+	sc->aq_required |= IXLV_FLAG_AQ_ADD_VLAN_FILTER;
+	return;
+}
+
+
+/*
+** This routine adds new MAC filters to the sc's list;
+** these are later added in hardware by the periodic
+** aq task.
+*/
+static int
+ixlv_add_mac_filter(struct ixlv_sc *sc, u8 *macaddr, u16 flags)
+{
+	struct ixlv_mac_filter	*f;
+	device_t			dev = sc->dev;
+
+	/* Does one already exist? */
+	f = ixlv_find_mac_filter(sc, macaddr);
+	if (f != NULL) {
+		IDPRINTF(sc->vsi.ifp, "exists: " MAC_FORMAT,
+		    MAC_FORMAT_ARGS(macaddr));
+		return (EEXIST);
+	}
+
+	/* If not, get a new empty filter */
+	f = ixlv_get_mac_filter(sc);
+	if (f == NULL) {
+		device_printf(dev, "%s: no filters available!!\n",
+		    __func__);
+		return (ENOMEM);
+	}
+
+	IDPRINTF(sc->vsi.ifp, "marked: " MAC_FORMAT,
+	    MAC_FORMAT_ARGS(macaddr));
+
+	bcopy(macaddr, f->macaddr, ETHER_ADDR_LEN);
+	f->flags |= (IXL_FILTER_ADD | IXL_FILTER_USED);
+	f->flags |= flags;
+	return (0);
+}
+
+/*
+** Tasklet handler for MSIX Adminq interrupts
+**  - done outside interrupt context since it might sleep
+*/
+static void
+ixlv_do_adminq(void *context, int pending)
+{
+	struct ixlv_sc		*sc = context;
+	struct i40e_hw			*hw = &sc->hw;
+	struct i40e_arq_event_info	event;
+	struct i40e_virtchnl_msg	*v_msg;
+	i40e_status			ret;
+	u16				result = 0;
+
+
+	event.buf_len = IXL_AQ_BUF_SZ;
+        event.msg_buf = malloc(event.buf_len,
+	    M_DEVBUF, M_NOWAIT | M_ZERO);
+	if (!event.msg_buf) {
+		printf("Unable to allocate adminq memory\n");
+		return;
+	}
+	v_msg = (struct i40e_virtchnl_msg *)&event.desc;
+
+	mtx_lock(&sc->mtx);
+	/* clean and process any events */
+	do {
+		ret = i40e_clean_arq_element(hw, &event, &result);
+		if (ret)
+			break;
+		ixlv_vc_completion(sc, v_msg->v_opcode,
+		    v_msg->v_retval, event.msg_buf, event.msg_len);
+		if (result != 0)
+			bzero(event.msg_buf, IXL_AQ_BUF_SZ);
+	} while (result);
+
+	ixlv_enable_adminq_irq(hw);
+	free(event.msg_buf, M_DEVBUF);
+	mtx_unlock(&sc->mtx);
+	return;
+}
+
+/*
+** ixlv_sched_aq - Periodic scheduling tasklet
+** 
+*/
+static void
+ixlv_sched_aq(void *context)
+{
+	struct ixlv_sc	*sc = context;
+	struct ixl_vsi		*vsi = &sc->vsi;
+
+	/* This is driven by a callout, don't spin */
+	if (!mtx_trylock(&sc->mtx))
+		goto done_nolock;
+
+	if (sc->init_state == IXLV_RESET_PENDING)
+		goto done;
+
+	/* Process requested admin queue tasks */
+	if (sc->aq_pending)
+		goto done;
+
+	if (sc->aq_required & IXLV_FLAG_AQ_MAP_VECTORS) {
+		ixlv_map_queues(sc);
+		goto done;
+	}
+
+	if (sc->aq_required & IXLV_FLAG_AQ_ADD_MAC_FILTER) {
+		ixlv_add_ether_filters(sc);
+		goto done;
+	}
+
+	if (sc->aq_required & IXLV_FLAG_AQ_ADD_VLAN_FILTER) {
+		ixlv_add_vlans(sc);
+		goto done;
+	}
+
+	if (sc->aq_required & IXLV_FLAG_AQ_DEL_MAC_FILTER) {
+		ixlv_del_ether_filters(sc);
+		goto done;
+	}
+
+	if (sc->aq_required & IXLV_FLAG_AQ_DEL_VLAN_FILTER) {
+		ixlv_del_vlans(sc);
+		goto done;
+	}
+
+	if (sc->aq_required & IXLV_FLAG_AQ_CONFIGURE_QUEUES) {
+		ixlv_configure_queues(sc);
+		goto done;
+	}
+
+	if (sc->aq_required & IXLV_FLAG_AQ_DISABLE_QUEUES) {
+		ixlv_disable_queues(sc);
+		goto done;
+	}
+
+	if (sc->aq_required & IXLV_FLAG_AQ_ENABLE_QUEUES) {
+		ixlv_enable_queues(sc);
+		goto done;
+	}
+
+	/* Do stats request only if no other AQ operations requested */
+	if (vsi->ifp->if_drv_flags & IFF_DRV_RUNNING)
+		ixlv_request_stats(sc);
+
+done:
+	mtx_unlock(&sc->mtx);
+done_nolock:
+	if (sc->aq_required) /* Reschedule */
+		callout_reset(&sc->aq_task, IXLV_CALLOUT_TIMO,
+		    ixlv_sched_aq, sc);
+	else
+		callout_reset(&sc->aq_task, 2 * hz, ixlv_sched_aq, sc);
+}
+
+static void
+ixlv_add_stats_sysctls(struct ixlv_sc *sc)
+{
+	device_t dev = sc->dev;
+	struct ixl_vsi *vsi = &sc->vsi;
+	struct i40e_eth_stats *es = &vsi->eth_stats;
+
+	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
+	struct sysctl_oid *tree = device_get_sysctl_tree(dev);
+	struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree);
+
+	struct sysctl_oid *vsi_node, *queue_node;
+	struct sysctl_oid_list *vsi_list, *queue_list;
+
+#define QUEUE_NAME_LEN 32
+	char queue_namebuf[QUEUE_NAME_LEN];
+
+	struct ixl_queue *queues = vsi->queues;
+	struct tx_ring *txr;
+	struct rx_ring *rxr;
+
+	/* Driver statistics */
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "watchdog_events",
+			CTLFLAG_RD, &sc->watchdog_events,
+			"Watchdog timeouts");
+	SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "admin_irq",
+			CTLFLAG_RD, &sc->admin_irq,
+			"Admin Queue IRQ Handled");
+
+	/* VSI statistics */
+	vsi_node = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "vsi",
+				   CTLFLAG_RD, NULL, "VSI-specific statistics");
+	vsi_list = SYSCTL_CHILDREN(vsi_node);
+
+	struct ixl_sysctl_info ctls[] =
+	{
+		{&es->rx_bytes, "good_octets_rcvd", "Good Octets Received"},
+		{&es->rx_unicast, "ucast_pkts_rcvd",
+			"Unicast Packets Received"},
+		{&es->rx_multicast, "mcast_pkts_rcvd",
+			"Multicast Packets Received"},
+		{&es->rx_broadcast, "bcast_pkts_rcvd",
+			"Broadcast Packets Received"},
+		{&es->rx_discards, "rx_discards", "Discarded RX packets"},
+		{&es->tx_bytes, "good_octets_txd", "Good Octets Transmitted"},
+		{&es->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted"},
+		{&es->tx_multicast, "mcast_pkts_txd",
+			"Multicast Packets Transmitted"},
+		{&es->tx_broadcast, "bcast_pkts_txd",
+			"Broadcast Packets Transmitted"},
+		{&es->tx_discards, "tx_discards", "Discarded TX packets"},
+		// end
+		{0,0,0}
+	};
+	struct ixl_sysctl_info *entry = ctls;
+	while (entry->stat != 0)
+	{
+		SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, entry->name,
+				CTLFLAG_RD, entry->stat,
+				entry->description);
+		entry++;
+	}
+
+	/* Queue statistics */
+	for (int q = 0; q < vsi->num_queues; q++) {
+		snprintf(queue_namebuf, QUEUE_NAME_LEN, "que%d", q);
+		queue_node = SYSCTL_ADD_NODE(ctx, vsi_list, OID_AUTO, queue_namebuf,
+					     CTLFLAG_RD, NULL, "Queue Name");
+		queue_list = SYSCTL_CHILDREN(queue_node);
+
+		txr = &(queues[q].txr);
+		rxr = &(queues[q].rxr);
+
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "mbuf_defrag_failed",
+				CTLFLAG_RD, &(queues[q].mbuf_defrag_failed),
+				"m_defrag() failed");
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "dropped",
+				CTLFLAG_RD, &(queues[q].dropped_pkts),
+				"Driver dropped packets");
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "irqs",
+				CTLFLAG_RD, &(queues[q].irqs),
+				"irqs on this queue");
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tso_tx",
+				CTLFLAG_RD, &(queues[q].tso),
+				"TSO");
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_dma_setup",
+				CTLFLAG_RD, &(queues[q].tx_dma_setup),
+				"Driver tx dma failure in xmit");
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "no_desc_avail",
+				CTLFLAG_RD, &(txr->no_desc),
+				"Queue No Descriptor Available");
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_packets",
+				CTLFLAG_RD, &(txr->total_packets),
+				"Queue Packets Transmitted");
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "tx_bytes",
+				CTLFLAG_RD, &(txr->tx_bytes),
+				"Queue Bytes Transmitted");
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_packets",
+				CTLFLAG_RD, &(rxr->rx_packets),
+				"Queue Packets Received");
+		SYSCTL_ADD_UQUAD(ctx, queue_list, OID_AUTO, "rx_bytes",
+				CTLFLAG_RD, &(rxr->rx_bytes),
+				"Queue Bytes Received");
+	}
+}
+
+static void
+ixlv_init_filters(struct ixlv_sc *sc)
+{
+	sc->mac_filters = malloc(sizeof(struct ixlv_mac_filter),
+	    M_DEVBUF, M_NOWAIT | M_ZERO);
+	SLIST_INIT(sc->mac_filters);
+	sc->vlan_filters = malloc(sizeof(struct ixlv_vlan_filter),
+	    M_DEVBUF, M_NOWAIT | M_ZERO);
+	SLIST_INIT(sc->vlan_filters);
+	return;
+}
+
+static void
+ixlv_free_filters(struct ixlv_sc *sc)
+{
+	struct ixlv_mac_filter *f;
+	struct ixlv_vlan_filter *v;
+
+	while (!SLIST_EMPTY(sc->mac_filters)) {
+		f = SLIST_FIRST(sc->mac_filters);
+		SLIST_REMOVE_HEAD(sc->mac_filters, next);
+		free(f, M_DEVBUF);
+	}
+	while (!SLIST_EMPTY(sc->vlan_filters)) {
+		v = SLIST_FIRST(sc->vlan_filters);
+		SLIST_REMOVE_HEAD(sc->vlan_filters, next);
+		free(v, M_DEVBUF);
+	}
+	return;
+}
+