From 84f670189b8c0dfd3dbaf12da8c946225f4011d3 Mon Sep 17 00:00:00 2001 From: Hariprasad Shenai Date: Tue, 8 Mar 2016 10:50:17 +0530 Subject: cxgb4vf: Enable interrupts before we register our network devices This avoids a race condition where a system that has network devices set up to be automatically configured and we get the first Port Link Status message from the firmware on the Asynchronous Firmware Event Queue before we've enabled interrupts. If that happens, we end up losing the interrupt and never realizing that the links has actually come up. Signed-off-by: Hariprasad Shenai Signed-off-by: David S. Miller --- .../net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c | 51 +++++++++++----------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c index 91857b8..fcafe34 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -2771,6 +2771,24 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, } } + /* See what interrupts we'll be using. If we've been configured to + * use MSI-X interrupts, try to enable them but fall back to using + * MSI interrupts if we can't enable MSI-X interrupts. If we can't + * get MSI interrupts we bail with the error. + */ + if (msi == MSI_MSIX && enable_msix(adapter) == 0) + adapter->flags |= USING_MSIX; + else { + err = pci_enable_msi(pdev); + if (err) { + dev_err(&pdev->dev, "Unable to allocate %s interrupts;" + " err=%d\n", + msi == MSI_MSIX ? "MSI-X or MSI" : "MSI", err); + goto err_free_dev; + } + adapter->flags |= USING_MSI; + } + /* * The "card" is now ready to go. If any errors occur during device * registration we do not fail the whole "card" but rather proceed @@ -2793,7 +2811,7 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, } if (adapter->registered_device_map == 0) { dev_err(&pdev->dev, "could not register any net devices\n"); - goto err_free_dev; + goto err_disable_interrupts; } /* @@ -2811,25 +2829,6 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, } /* - * See what interrupts we'll be using. If we've been configured to - * use MSI-X interrupts, try to enable them but fall back to using - * MSI interrupts if we can't enable MSI-X interrupts. If we can't - * get MSI interrupts we bail with the error. - */ - if (msi == MSI_MSIX && enable_msix(adapter) == 0) - adapter->flags |= USING_MSIX; - else { - err = pci_enable_msi(pdev); - if (err) { - dev_err(&pdev->dev, "Unable to allocate %s interrupts;" - " err=%d\n", - msi == MSI_MSIX ? "MSI-X or MSI" : "MSI", err); - goto err_free_debugfs; - } - adapter->flags |= USING_MSI; - } - - /* * Now that we know how many "ports" we have and what their types are, * and how many Queue Sets we can support, we can configure our queue * resources. @@ -2856,11 +2855,13 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, * Error recovery and exit code. Unwind state that's been created * so far and return the error. */ - -err_free_debugfs: - if (!IS_ERR_OR_NULL(adapter->debugfs_root)) { - cleanup_debugfs(adapter); - debugfs_remove_recursive(adapter->debugfs_root); +err_disable_interrupts: + if (adapter->flags & USING_MSIX) { + pci_disable_msix(adapter->pdev); + adapter->flags &= ~USING_MSIX; + } else if (adapter->flags & USING_MSI) { + pci_disable_msi(adapter->pdev); + adapter->flags &= ~USING_MSI; } err_free_dev: -- cgit v1.1 From 495c22bbb2b03548f6aa870faa7f6b601cb41c85 Mon Sep 17 00:00:00 2001 From: Hariprasad Shenai Date: Tue, 8 Mar 2016 10:50:18 +0530 Subject: cxgb4vf: Configure queue based on resource and interrupt type The Queue Set Configuration code was always reserving room for a Forwarded interrupt Queue even in the cases where we weren't using it. Figure out how many Ports and Queue Sets we can support. This depends on knowing our Virtual Function Resources and may be called a second time if we fall back from MSI-X to MSI Interrupt Mode. This change fixes that problem. Signed-off-by: Hariprasad Shenai Signed-off-by: David S. Miller --- .../net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c | 165 ++++++++++++--------- 1 file changed, 94 insertions(+), 71 deletions(-) diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c index fcafe34..17a3153 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -2176,6 +2176,73 @@ static void cleanup_debugfs(struct adapter *adapter) /* nothing to do */ } +/* Figure out how many Ports and Queue Sets we can support. This depends on + * knowing our Virtual Function Resources and may be called a second time if + * we fall back from MSI-X to MSI Interrupt Mode. + */ +static void size_nports_qsets(struct adapter *adapter) +{ + struct vf_resources *vfres = &adapter->params.vfres; + unsigned int ethqsets, pmask_nports; + + /* The number of "ports" which we support is equal to the number of + * Virtual Interfaces with which we've been provisioned. + */ + adapter->params.nports = vfres->nvi; + if (adapter->params.nports > MAX_NPORTS) { + dev_warn(adapter->pdev_dev, "only using %d of %d maximum" + " allowed virtual interfaces\n", MAX_NPORTS, + adapter->params.nports); + adapter->params.nports = MAX_NPORTS; + } + + /* We may have been provisioned with more VIs than the number of + * ports we're allowed to access (our Port Access Rights Mask). + * This is obviously a configuration conflict but we don't want to + * crash the kernel or anything silly just because of that. + */ + pmask_nports = hweight32(adapter->params.vfres.pmask); + if (pmask_nports < adapter->params.nports) { + dev_warn(adapter->pdev_dev, "only using %d of %d provissioned" + " virtual interfaces; limited by Port Access Rights" + " mask %#x\n", pmask_nports, adapter->params.nports, + adapter->params.vfres.pmask); + adapter->params.nports = pmask_nports; + } + + /* We need to reserve an Ingress Queue for the Asynchronous Firmware + * Event Queue. And if we're using MSI Interrupts, we'll also need to + * reserve an Ingress Queue for a Forwarded Interrupts. + * + * The rest of the FL/Intr-capable ingress queues will be matched up + * one-for-one with Ethernet/Control egress queues in order to form + * "Queue Sets" which will be aportioned between the "ports". For + * each Queue Set, we'll need the ability to allocate two Egress + * Contexts -- one for the Ingress Queue Free List and one for the TX + * Ethernet Queue. + * + * Note that even if we're currently configured to use MSI-X + * Interrupts (module variable msi == MSI_MSIX) we may get downgraded + * to MSI Interrupts if we can't get enough MSI-X Interrupts. If that + * happens we'll need to adjust things later. + */ + ethqsets = vfres->niqflint - 1 - (msi == MSI_MSI); + if (vfres->nethctrl != ethqsets) + ethqsets = min(vfres->nethctrl, ethqsets); + if (vfres->neq < ethqsets*2) + ethqsets = vfres->neq/2; + if (ethqsets > MAX_ETH_QSETS) + ethqsets = MAX_ETH_QSETS; + adapter->sge.max_ethqsets = ethqsets; + + if (adapter->sge.max_ethqsets < adapter->params.nports) { + dev_warn(adapter->pdev_dev, "only using %d of %d available" + " virtual interfaces (too few Queue Sets)\n", + adapter->sge.max_ethqsets, adapter->params.nports); + adapter->params.nports = adapter->sge.max_ethqsets; + } +} + /* * Perform early "adapter" initialization. This is where we discover what * adapter parameters we're going to be using and initialize basic adapter @@ -2183,10 +2250,8 @@ static void cleanup_debugfs(struct adapter *adapter) */ static int adap_init0(struct adapter *adapter) { - struct vf_resources *vfres = &adapter->params.vfres; struct sge_params *sge_params = &adapter->params.sge; struct sge *s = &adapter->sge; - unsigned int ethqsets; int err; u32 param, val = 0; @@ -2295,69 +2360,18 @@ static int adap_init0(struct adapter *adapter) return err; } - /* - * The number of "ports" which we support is equal to the number of - * Virtual Interfaces with which we've been provisioned. - */ - adapter->params.nports = vfres->nvi; - if (adapter->params.nports > MAX_NPORTS) { - dev_warn(adapter->pdev_dev, "only using %d of %d allowed" - " virtual interfaces\n", MAX_NPORTS, - adapter->params.nports); - adapter->params.nports = MAX_NPORTS; - } - - /* - * We need to reserve a number of the ingress queues with Free List - * and Interrupt capabilities for special interrupt purposes (like - * asynchronous firmware messages, or forwarded interrupts if we're - * using MSI). The rest of the FL/Intr-capable ingress queues will be - * matched up one-for-one with Ethernet/Control egress queues in order - * to form "Queue Sets" which will be aportioned between the "ports". - * For each Queue Set, we'll need the ability to allocate two Egress - * Contexts -- one for the Ingress Queue Free List and one for the TX - * Ethernet Queue. - */ - ethqsets = vfres->niqflint - INGQ_EXTRAS; - if (vfres->nethctrl != ethqsets) { - dev_warn(adapter->pdev_dev, "unequal number of [available]" - " ingress/egress queues (%d/%d); using minimum for" - " number of Queue Sets\n", ethqsets, vfres->nethctrl); - ethqsets = min(vfres->nethctrl, ethqsets); - } - if (vfres->neq < ethqsets*2) { - dev_warn(adapter->pdev_dev, "Not enough Egress Contexts (%d)" - " to support Queue Sets (%d); reducing allowed Queue" - " Sets\n", vfres->neq, ethqsets); - ethqsets = vfres->neq/2; - } - if (ethqsets > MAX_ETH_QSETS) { - dev_warn(adapter->pdev_dev, "only using %d of %d allowed Queue" - " Sets\n", MAX_ETH_QSETS, adapter->sge.max_ethqsets); - ethqsets = MAX_ETH_QSETS; - } - if (vfres->niq != 0 || vfres->neq > ethqsets*2) { - dev_warn(adapter->pdev_dev, "unused resources niq/neq (%d/%d)" - " ignored\n", vfres->niq, vfres->neq - ethqsets*2); - } - adapter->sge.max_ethqsets = ethqsets; - - /* - * Check for various parameter sanity issues. Most checks simply - * result in us using fewer resources than our provissioning but we - * do need at least one "port" with which to work ... - */ - if (adapter->sge.max_ethqsets < adapter->params.nports) { - dev_warn(adapter->pdev_dev, "only using %d of %d available" - " virtual interfaces (too few Queue Sets)\n", - adapter->sge.max_ethqsets, adapter->params.nports); - adapter->params.nports = adapter->sge.max_ethqsets; - } - if (adapter->params.nports == 0) { + /* Check for various parameter sanity issues */ + if (adapter->params.vfres.nvi == 0) { dev_err(adapter->pdev_dev, "no virtual interfaces configured/" "usable!\n"); return -EINVAL; } + + /* Initialize nports and max_ethqsets now that we have our Virtual + * Function Resources. + */ + size_nports_qsets(adapter); + return 0; } @@ -2779,16 +2793,32 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, if (msi == MSI_MSIX && enable_msix(adapter) == 0) adapter->flags |= USING_MSIX; else { + if (msi == MSI_MSIX) { + dev_info(adapter->pdev_dev, + "Unable to use MSI-X Interrupts; falling " + "back to MSI Interrupts\n"); + + /* We're going to need a Forwarded Interrupt Queue so + * that may cut into how many Queue Sets we can + * support. + */ + msi = MSI_MSI; + size_nports_qsets(adapter); + } err = pci_enable_msi(pdev); if (err) { - dev_err(&pdev->dev, "Unable to allocate %s interrupts;" - " err=%d\n", - msi == MSI_MSIX ? "MSI-X or MSI" : "MSI", err); + dev_err(&pdev->dev, "Unable to allocate MSI Interrupts;" + " err=%d\n", err); goto err_free_dev; } adapter->flags |= USING_MSI; } + /* Now that we know how many "ports" we have and what interrupt + * mechanism we're going to use, we can configure our queue resources. + */ + cfg_queues(adapter); + /* * The "card" is now ready to go. If any errors occur during device * registration we do not fail the whole "card" but rather proceed @@ -2829,13 +2859,6 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, } /* - * Now that we know how many "ports" we have and what their types are, - * and how many Queue Sets we can support, we can configure our queue - * resources. - */ - cfg_queues(adapter); - - /* * Print a short notice on the existence and configuration of the new * VF network device ... */ -- cgit v1.1 From 28f71c6df4c5399e4f30ce37c1fff0f6af3d364d Mon Sep 17 00:00:00 2001 From: Hariprasad Shenai Date: Tue, 8 Mar 2016 10:50:19 +0530 Subject: cxgb4vf: Add a couple more checks for invalid provisioning configurations Signed-off-by: Hariprasad Shenai Signed-off-by: David S. Miller --- drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c index 17a3153..5a3b883 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -2361,6 +2361,11 @@ static int adap_init0(struct adapter *adapter) } /* Check for various parameter sanity issues */ + if (adapter->params.vfres.pmask == 0) { + dev_err(adapter->pdev_dev, "no port access configured\n" + "usable!\n"); + return -EINVAL; + } if (adapter->params.vfres.nvi == 0) { dev_err(adapter->pdev_dev, "no virtual interfaces configured/" "usable!\n"); -- cgit v1.1 From a8d16d08065f8d3f2fca4a4c377fc4bfc5bdfccd Mon Sep 17 00:00:00 2001 From: Hariprasad Shenai Date: Tue, 8 Mar 2016 10:50:20 +0530 Subject: cxgb4vf: Set number of queues in pci probe only Signed-off-by: Hariprasad Shenai Signed-off-by: David S. Miller --- drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c index 5a3b883..1cc8a7a 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -790,10 +790,6 @@ static int cxgb4vf_open(struct net_device *dev) /* * Note that this interface is up and start everything up ... */ - netif_set_real_num_tx_queues(dev, pi->nqsets); - err = netif_set_real_num_rx_queues(dev, pi->nqsets); - if (err) - goto err_unwind; err = link_start(dev); if (err) goto err_unwind; @@ -2831,10 +2827,14 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, * must register at least one net device. */ for_each_port(adapter, pidx) { + struct port_info *pi = netdev_priv(adapter->port[pidx]); netdev = adapter->port[pidx]; if (netdev == NULL) continue; + netif_set_real_num_tx_queues(netdev, pi->nqsets); + netif_set_real_num_rx_queues(netdev, pi->nqsets); + err = register_netdev(netdev); if (err) { dev_warn(&pdev->dev, "cannot register net device %s," -- cgit v1.1