diff options
Diffstat (limited to 'drivers/scsi/csiostor/csio_isr.c')
-rw-r--r-- | drivers/scsi/csiostor/csio_isr.c | 624 |
1 files changed, 624 insertions, 0 deletions
diff --git a/drivers/scsi/csiostor/csio_isr.c b/drivers/scsi/csiostor/csio_isr.c new file mode 100644 index 0000000..7ee9777 --- /dev/null +++ b/drivers/scsi/csiostor/csio_isr.c @@ -0,0 +1,624 @@ +/* + * This file is part of the Chelsio FCoE driver for Linux. + * + * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/interrupt.h> +#include <linux/cpumask.h> +#include <linux/string.h> + +#include "csio_init.h" +#include "csio_hw.h" + +static irqreturn_t +csio_nondata_isr(int irq, void *dev_id) +{ + struct csio_hw *hw = (struct csio_hw *) dev_id; + int rv; + unsigned long flags; + + if (unlikely(!hw)) + return IRQ_NONE; + + if (unlikely(pci_channel_offline(hw->pdev))) { + CSIO_INC_STATS(hw, n_pcich_offline); + return IRQ_NONE; + } + + spin_lock_irqsave(&hw->lock, flags); + csio_hw_slow_intr_handler(hw); + rv = csio_mb_isr_handler(hw); + + if (rv == 0 && !(hw->flags & CSIO_HWF_FWEVT_PENDING)) { + hw->flags |= CSIO_HWF_FWEVT_PENDING; + spin_unlock_irqrestore(&hw->lock, flags); + schedule_work(&hw->evtq_work); + return IRQ_HANDLED; + } + spin_unlock_irqrestore(&hw->lock, flags); + return IRQ_HANDLED; +} + +/* + * csio_fwevt_handler - Common FW event handler routine. + * @hw: HW module. + * + * This is the ISR for FW events. It is shared b/w MSIX + * and INTx handlers. + */ +static void +csio_fwevt_handler(struct csio_hw *hw) +{ + int rv; + unsigned long flags; + + rv = csio_fwevtq_handler(hw); + + spin_lock_irqsave(&hw->lock, flags); + if (rv == 0 && !(hw->flags & CSIO_HWF_FWEVT_PENDING)) { + hw->flags |= CSIO_HWF_FWEVT_PENDING; + spin_unlock_irqrestore(&hw->lock, flags); + schedule_work(&hw->evtq_work); + return; + } + spin_unlock_irqrestore(&hw->lock, flags); + +} /* csio_fwevt_handler */ + +/* + * csio_fwevt_isr() - FW events MSIX ISR + * @irq: + * @dev_id: + * + * Process WRs on the FW event queue. + * + */ +static irqreturn_t +csio_fwevt_isr(int irq, void *dev_id) +{ + struct csio_hw *hw = (struct csio_hw *) dev_id; + + if (unlikely(!hw)) + return IRQ_NONE; + + if (unlikely(pci_channel_offline(hw->pdev))) { + CSIO_INC_STATS(hw, n_pcich_offline); + return IRQ_NONE; + } + + csio_fwevt_handler(hw); + + return IRQ_HANDLED; +} + +/* + * csio_fwevt_isr() - INTx wrapper for handling FW events. + * @irq: + * @dev_id: + */ +void +csio_fwevt_intx_handler(struct csio_hw *hw, void *wr, uint32_t len, + struct csio_fl_dma_buf *flb, void *priv) +{ + csio_fwevt_handler(hw); +} /* csio_fwevt_intx_handler */ + +/* + * csio_process_scsi_cmpl - Process a SCSI WR completion. + * @hw: HW module. + * @wr: The completed WR from the ingress queue. + * @len: Length of the WR. + * @flb: Freelist buffer array. + * + */ +static void +csio_process_scsi_cmpl(struct csio_hw *hw, void *wr, uint32_t len, + struct csio_fl_dma_buf *flb, void *cbfn_q) +{ + struct csio_ioreq *ioreq; + uint8_t *scsiwr; + uint8_t subop; + void *cmnd; + unsigned long flags; + + ioreq = csio_scsi_cmpl_handler(hw, wr, len, flb, NULL, &scsiwr); + if (likely(ioreq)) { + if (unlikely(*scsiwr == FW_SCSI_ABRT_CLS_WR)) { + subop = FW_SCSI_ABRT_CLS_WR_SUB_OPCODE_GET( + ((struct fw_scsi_abrt_cls_wr *) + scsiwr)->sub_opcode_to_chk_all_io); + + csio_dbg(hw, "%s cmpl recvd ioreq:%p status:%d\n", + subop ? "Close" : "Abort", + ioreq, ioreq->wr_status); + + spin_lock_irqsave(&hw->lock, flags); + if (subop) + csio_scsi_closed(ioreq, + (struct list_head *)cbfn_q); + else + csio_scsi_aborted(ioreq, + (struct list_head *)cbfn_q); + /* + * We call scsi_done for I/Os that driver thinks aborts + * have timed out. If there is a race caused by FW + * completing abort at the exact same time that the + * driver has deteced the abort timeout, the following + * check prevents calling of scsi_done twice for the + * same command: once from the eh_abort_handler, another + * from csio_scsi_isr_handler(). This also avoids the + * need to check if csio_scsi_cmnd(req) is NULL in the + * fast path. + */ + cmnd = csio_scsi_cmnd(ioreq); + if (unlikely(cmnd == NULL)) + list_del_init(&ioreq->sm.sm_list); + + spin_unlock_irqrestore(&hw->lock, flags); + + if (unlikely(cmnd == NULL)) + csio_put_scsi_ioreq_lock(hw, + csio_hw_to_scsim(hw), ioreq); + } else { + spin_lock_irqsave(&hw->lock, flags); + csio_scsi_completed(ioreq, (struct list_head *)cbfn_q); + spin_unlock_irqrestore(&hw->lock, flags); + } + } +} + +/* + * csio_scsi_isr_handler() - Common SCSI ISR handler. + * @iq: Ingress queue pointer. + * + * Processes SCSI completions on the SCSI IQ indicated by scm->iq_idx + * by calling csio_wr_process_iq_idx. If there are completions on the + * isr_cbfn_q, yank them out into a local queue and call their io_cbfns. + * Once done, add these completions onto the freelist. + * This routine is shared b/w MSIX and INTx. + */ +static inline irqreturn_t +csio_scsi_isr_handler(struct csio_q *iq) +{ + struct csio_hw *hw = (struct csio_hw *)iq->owner; + LIST_HEAD(cbfn_q); + struct list_head *tmp; + struct csio_scsim *scm; + struct csio_ioreq *ioreq; + int isr_completions = 0; + + scm = csio_hw_to_scsim(hw); + + if (unlikely(csio_wr_process_iq(hw, iq, csio_process_scsi_cmpl, + &cbfn_q) != 0)) + return IRQ_NONE; + + /* Call back the completion routines */ + list_for_each(tmp, &cbfn_q) { + ioreq = (struct csio_ioreq *)tmp; + isr_completions++; + ioreq->io_cbfn(hw, ioreq); + /* Release ddp buffer if used for this req */ + if (unlikely(ioreq->dcopy)) + csio_put_scsi_ddp_list_lock(hw, scm, &ioreq->gen_list, + ioreq->nsge); + } + + if (isr_completions) { + /* Return the ioreqs back to ioreq->freelist */ + csio_put_scsi_ioreq_list_lock(hw, scm, &cbfn_q, + isr_completions); + } + + return IRQ_HANDLED; +} + +/* + * csio_scsi_isr() - SCSI MSIX handler + * @irq: + * @dev_id: + * + * This is the top level SCSI MSIX handler. Calls csio_scsi_isr_handler() + * for handling SCSI completions. + */ +static irqreturn_t +csio_scsi_isr(int irq, void *dev_id) +{ + struct csio_q *iq = (struct csio_q *) dev_id; + struct csio_hw *hw; + + if (unlikely(!iq)) + return IRQ_NONE; + + hw = (struct csio_hw *)iq->owner; + + if (unlikely(pci_channel_offline(hw->pdev))) { + CSIO_INC_STATS(hw, n_pcich_offline); + return IRQ_NONE; + } + + csio_scsi_isr_handler(iq); + + return IRQ_HANDLED; +} + +/* + * csio_scsi_intx_handler() - SCSI INTx handler + * @irq: + * @dev_id: + * + * This is the top level SCSI INTx handler. Calls csio_scsi_isr_handler() + * for handling SCSI completions. + */ +void +csio_scsi_intx_handler(struct csio_hw *hw, void *wr, uint32_t len, + struct csio_fl_dma_buf *flb, void *priv) +{ + struct csio_q *iq = priv; + + csio_scsi_isr_handler(iq); + +} /* csio_scsi_intx_handler */ + +/* + * csio_fcoe_isr() - INTx/MSI interrupt service routine for FCoE. + * @irq: + * @dev_id: + * + * + */ +static irqreturn_t +csio_fcoe_isr(int irq, void *dev_id) +{ + struct csio_hw *hw = (struct csio_hw *) dev_id; + struct csio_q *intx_q = NULL; + int rv; + irqreturn_t ret = IRQ_NONE; + unsigned long flags; + + if (unlikely(!hw)) + return IRQ_NONE; + + if (unlikely(pci_channel_offline(hw->pdev))) { + CSIO_INC_STATS(hw, n_pcich_offline); + return IRQ_NONE; + } + + /* Disable the interrupt for this PCI function. */ + if (hw->intr_mode == CSIO_IM_INTX) + csio_wr_reg32(hw, 0, MYPF_REG(PCIE_PF_CLI)); + + /* + * The read in the following function will flush the + * above write. + */ + if (csio_hw_slow_intr_handler(hw)) + ret = IRQ_HANDLED; + + /* Get the INTx Forward interrupt IQ. */ + intx_q = csio_get_q(hw, hw->intr_iq_idx); + + CSIO_DB_ASSERT(intx_q); + + /* IQ handler is not possible for intx_q, hence pass in NULL */ + if (likely(csio_wr_process_iq(hw, intx_q, NULL, NULL) == 0)) + ret = IRQ_HANDLED; + + spin_lock_irqsave(&hw->lock, flags); + rv = csio_mb_isr_handler(hw); + if (rv == 0 && !(hw->flags & CSIO_HWF_FWEVT_PENDING)) { + hw->flags |= CSIO_HWF_FWEVT_PENDING; + spin_unlock_irqrestore(&hw->lock, flags); + schedule_work(&hw->evtq_work); + return IRQ_HANDLED; + } + spin_unlock_irqrestore(&hw->lock, flags); + + return ret; +} + +static void +csio_add_msix_desc(struct csio_hw *hw) +{ + int i; + struct csio_msix_entries *entryp = &hw->msix_entries[0]; + int k = CSIO_EXTRA_VECS; + int len = sizeof(entryp->desc) - 1; + int cnt = hw->num_sqsets + k; + + /* Non-data vector */ + memset(entryp->desc, 0, len + 1); + snprintf(entryp->desc, len, "csio-%02x:%02x:%x-nondata", + CSIO_PCI_BUS(hw), CSIO_PCI_DEV(hw), CSIO_PCI_FUNC(hw)); + + entryp++; + memset(entryp->desc, 0, len + 1); + snprintf(entryp->desc, len, "csio-%02x:%02x:%x-fwevt", + CSIO_PCI_BUS(hw), CSIO_PCI_DEV(hw), CSIO_PCI_FUNC(hw)); + entryp++; + + /* Name SCSI vecs */ + for (i = k; i < cnt; i++, entryp++) { + memset(entryp->desc, 0, len + 1); + snprintf(entryp->desc, len, "csio-%02x:%02x:%x-scsi%d", + CSIO_PCI_BUS(hw), CSIO_PCI_DEV(hw), + CSIO_PCI_FUNC(hw), i - CSIO_EXTRA_VECS); + } +} + +int +csio_request_irqs(struct csio_hw *hw) +{ + int rv, i, j, k = 0; + struct csio_msix_entries *entryp = &hw->msix_entries[0]; + struct csio_scsi_cpu_info *info; + + if (hw->intr_mode != CSIO_IM_MSIX) { + rv = request_irq(hw->pdev->irq, csio_fcoe_isr, + (hw->intr_mode == CSIO_IM_MSI) ? + 0 : IRQF_SHARED, + KBUILD_MODNAME, hw); + if (rv) { + if (hw->intr_mode == CSIO_IM_MSI) + pci_disable_msi(hw->pdev); + csio_err(hw, "Failed to allocate interrupt line.\n"); + return -EINVAL; + } + + goto out; + } + + /* Add the MSIX vector descriptions */ + csio_add_msix_desc(hw); + + rv = request_irq(entryp[k].vector, csio_nondata_isr, 0, + entryp[k].desc, hw); + if (rv) { + csio_err(hw, "IRQ request failed for vec %d err:%d\n", + entryp[k].vector, rv); + goto err; + } + + entryp[k++].dev_id = (void *)hw; + + rv = request_irq(entryp[k].vector, csio_fwevt_isr, 0, + entryp[k].desc, hw); + if (rv) { + csio_err(hw, "IRQ request failed for vec %d err:%d\n", + entryp[k].vector, rv); + goto err; + } + + entryp[k++].dev_id = (void *)hw; + + /* Allocate IRQs for SCSI */ + for (i = 0; i < hw->num_pports; i++) { + info = &hw->scsi_cpu_info[i]; + for (j = 0; j < info->max_cpus; j++, k++) { + struct csio_scsi_qset *sqset = &hw->sqset[i][j]; + struct csio_q *q = hw->wrm.q_arr[sqset->iq_idx]; + + rv = request_irq(entryp[k].vector, csio_scsi_isr, 0, + entryp[k].desc, q); + if (rv) { + csio_err(hw, + "IRQ request failed for vec %d err:%d\n", + entryp[k].vector, rv); + goto err; + } + + entryp[k].dev_id = (void *)q; + + } /* for all scsi cpus */ + } /* for all ports */ + +out: + hw->flags |= CSIO_HWF_HOST_INTR_ENABLED; + + return 0; + +err: + for (i = 0; i < k; i++) { + entryp = &hw->msix_entries[i]; + free_irq(entryp->vector, entryp->dev_id); + } + pci_disable_msix(hw->pdev); + + return -EINVAL; +} + +static void +csio_disable_msix(struct csio_hw *hw, bool free) +{ + int i; + struct csio_msix_entries *entryp; + int cnt = hw->num_sqsets + CSIO_EXTRA_VECS; + + if (free) { + for (i = 0; i < cnt; i++) { + entryp = &hw->msix_entries[i]; + free_irq(entryp->vector, entryp->dev_id); + } + } + pci_disable_msix(hw->pdev); +} + +/* Reduce per-port max possible CPUs */ +static void +csio_reduce_sqsets(struct csio_hw *hw, int cnt) +{ + int i; + struct csio_scsi_cpu_info *info; + + while (cnt < hw->num_sqsets) { + for (i = 0; i < hw->num_pports; i++) { + info = &hw->scsi_cpu_info[i]; + if (info->max_cpus > 1) { + info->max_cpus--; + hw->num_sqsets--; + if (hw->num_sqsets <= cnt) + break; + } + } + } + + csio_dbg(hw, "Reduced sqsets to %d\n", hw->num_sqsets); +} + +static int +csio_enable_msix(struct csio_hw *hw) +{ + int rv, i, j, k, n, min, cnt; + struct csio_msix_entries *entryp; + struct msix_entry *entries; + int extra = CSIO_EXTRA_VECS; + struct csio_scsi_cpu_info *info; + + min = hw->num_pports + extra; + cnt = hw->num_sqsets + extra; + + /* Max vectors required based on #niqs configured in fw */ + if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS || !csio_is_hw_master(hw)) + cnt = min_t(uint8_t, hw->cfg_niq, cnt); + + entries = kzalloc(sizeof(struct msix_entry) * cnt, GFP_KERNEL); + if (!entries) + return -ENOMEM; + + for (i = 0; i < cnt; i++) + entries[i].entry = (uint16_t)i; + + csio_dbg(hw, "FW supp #niq:%d, trying %d msix's\n", hw->cfg_niq, cnt); + + while ((rv = pci_enable_msix(hw->pdev, entries, cnt)) >= min) + cnt = rv; + if (!rv) { + if (cnt < (hw->num_sqsets + extra)) { + csio_dbg(hw, "Reducing sqsets to %d\n", cnt - extra); + csio_reduce_sqsets(hw, cnt - extra); + } + } else { + if (rv > 0) { + pci_disable_msix(hw->pdev); + csio_info(hw, "Not using MSI-X, remainder:%d\n", rv); + } + + kfree(entries); + return -ENOMEM; + } + + /* Save off vectors */ + for (i = 0; i < cnt; i++) { + entryp = &hw->msix_entries[i]; + entryp->vector = entries[i].vector; + } + + /* Distribute vectors */ + k = 0; + csio_set_nondata_intr_idx(hw, entries[k].entry); + csio_set_mb_intr_idx(csio_hw_to_mbm(hw), entries[k++].entry); + csio_set_fwevt_intr_idx(hw, entries[k++].entry); + + for (i = 0; i < hw->num_pports; i++) { + info = &hw->scsi_cpu_info[i]; + + for (j = 0; j < hw->num_scsi_msix_cpus; j++) { + n = (j % info->max_cpus) + k; + hw->sqset[i][j].intr_idx = entries[n].entry; + } + + k += info->max_cpus; + } + + kfree(entries); + return 0; +} + +void +csio_intr_enable(struct csio_hw *hw) +{ + hw->intr_mode = CSIO_IM_NONE; + hw->flags &= ~CSIO_HWF_HOST_INTR_ENABLED; + + /* Try MSIX, then MSI or fall back to INTx */ + if ((csio_msi == 2) && !csio_enable_msix(hw)) + hw->intr_mode = CSIO_IM_MSIX; + else { + /* Max iqs required based on #niqs configured in fw */ + if (hw->flags & CSIO_HWF_USING_SOFT_PARAMS || + !csio_is_hw_master(hw)) { + int extra = CSIO_EXTRA_MSI_IQS; + + if (hw->cfg_niq < (hw->num_sqsets + extra)) { + csio_dbg(hw, "Reducing sqsets to %d\n", + hw->cfg_niq - extra); + csio_reduce_sqsets(hw, hw->cfg_niq - extra); + } + } + + if ((csio_msi == 1) && !pci_enable_msi(hw->pdev)) + hw->intr_mode = CSIO_IM_MSI; + else + hw->intr_mode = CSIO_IM_INTX; + } + + csio_dbg(hw, "Using %s interrupt mode.\n", + (hw->intr_mode == CSIO_IM_MSIX) ? "MSIX" : + ((hw->intr_mode == CSIO_IM_MSI) ? "MSI" : "INTx")); +} + +void +csio_intr_disable(struct csio_hw *hw, bool free) +{ + csio_hw_intr_disable(hw); + + switch (hw->intr_mode) { + case CSIO_IM_MSIX: + csio_disable_msix(hw, free); + break; + case CSIO_IM_MSI: + if (free) + free_irq(hw->pdev->irq, hw); + pci_disable_msi(hw->pdev); + break; + case CSIO_IM_INTX: + if (free) + free_irq(hw->pdev->irq, hw); + break; + default: + break; + } + hw->intr_mode = CSIO_IM_NONE; + hw->flags &= ~CSIO_HWF_HOST_INTR_ENABLED; +} |