From ed8ccaef52fa03fb03cff45b380f72c9f869f273 Mon Sep 17 00:00:00 2001 From: Tadeusz Struk Date: Fri, 7 Aug 2015 11:34:25 -0700 Subject: crypto: qat - Add support for SRIOV Add code that enables SRIOV on dh895xcc devices. Signed-off-by: Tadeusz Struk Signed-off-by: Herbert Xu --- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 24 +++- .../crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h | 4 + drivers/crypto/qat/qat_dh895xcc/adf_drv.c | 80 +++++++----- drivers/crypto/qat/qat_dh895xcc/adf_isr.c | 139 +++++++++++++++------ 4 files changed, 180 insertions(+), 67 deletions(-) (limited to 'drivers/crypto/qat/qat_dh895xcc') diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index a3b1be8..ff54257 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include +#include #include #include "adf_dh895xcc_hw_data.h" #include "adf_drv.h" @@ -161,6 +162,16 @@ void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, } } +static uint32_t get_pf2vf_offset(uint32_t i) +{ + return ADF_DH895XCC_PF2VF_OFFSET(i); +} + +static uint32_t get_vintmsk_offset(uint32_t i) +{ + return ADF_DH895XCC_VINTMSK_OFFSET(i); +} + static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_device = accel_dev->hw_device; @@ -197,11 +208,17 @@ static void adf_enable_ints(struct adf_accel_dev *accel_dev) /* Enable bundle and misc interrupts */ ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, - ADF_DH895XCC_SMIA0_MASK); + accel_dev->pf.vf_info ? 0 : + GENMASK_ULL(GET_MAX_BANKS(accel_dev) - 1, 0)); ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, ADF_DH895XCC_SMIA1_MASK); } +static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + return 0; +} + void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcc_class; @@ -221,17 +238,22 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_num_aes = get_num_aes; hw_data->get_etr_bar_id = get_etr_bar_id; hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vintmsk_offset = get_vintmsk_offset; hw_data->get_sram_bar_id = get_sram_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; hw_data->fw_mmp_name = ADF_DH895XCC_MMP; hw_data->init_admin_comms = adf_init_admin_comms; hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->disable_iov = adf_disable_sriov; hw_data->send_admin_init = adf_send_admin_init; hw_data->init_arb = adf_init_arb; hw_data->exit_arb = adf_exit_arb; hw_data->get_arb_mapping = adf_get_arbiter_mapping; hw_data->enable_ints = adf_enable_ints; + hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; } void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index d410afb..88dffb2 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -80,6 +80,10 @@ #define ADF_DH895XCC_CERRSSMSH(i) (i * 0x4000 + 0x10) #define ADF_DH895XCC_ERRSSMSH_EN BIT(3) +#define ADF_DH895XCC_ERRSOU3 (0x3A000 + 0x00C) +#define ADF_DH895XCC_ERRSOU5 (0x3A000 + 0x0D8) +#define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) +#define ADF_DH895XCC_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) /* FW names */ #define ADF_DH895XCC_FW "qat_895xcc.bin" #define ADF_DH895XCC_MMP "qat_mmp.bin" diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index 1837c32..f8dd14f 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -82,16 +82,21 @@ static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, .name = adf_driver_name, .probe = adf_probe, - .remove = adf_remove + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, }; +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; int i; - adf_dev_shutdown(accel_dev); - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; @@ -108,13 +113,11 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) break; } kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; } adf_cfg_dev_remove(accel_dev); debugfs_remove(accel_dev->debugfs_dir); - adf_devmgr_rm_dev(accel_dev); - pci_release_regions(accel_pci_dev->pci_dev); - pci_disable_device(accel_pci_dev->pci_dev); - kfree(accel_dev); + adf_devmgr_rm_dev(accel_dev, NULL); } static int adf_dev_configure(struct adf_accel_dev *accel_dev) @@ -205,7 +208,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct adf_hw_device_data *hw_data; char name[ADF_DEVICE_NAME_LENGTH]; unsigned int i, bar_nr; - int ret; + int ret, bar_mask; switch (ent->device) { case ADF_DH895XCC_PCI_DEVICE_ID: @@ -229,10 +232,12 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return -ENOMEM; INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; /* Add accel device to accel table. * This should be called before adf_cleanup_accel is called */ - if (adf_devmgr_add_dev(accel_dev)) { + if (adf_devmgr_add_dev(accel_dev, NULL)) { dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); kfree(accel_dev); return -EFAULT; @@ -255,7 +260,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) default: return -ENODEV; } - accel_pci_dev = &accel_dev->accel_pci_dev; pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET, &hw_data->fuses); @@ -264,7 +268,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); accel_pci_dev->sku = hw_data->get_sku(hw_data); - accel_pci_dev->pci_dev = pdev; /* If the device has no acceleration engines then ignore it. */ if (!hw_data->accel_mask || !hw_data->ae_mask || ((~hw_data->ae_mask) & 0x01)) { @@ -274,11 +277,14 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } /* Create dev top level debugfs entry */ - snprintf(name, sizeof(name), "%s%s_dev%d", ADF_DEVICE_NAME_PREFIX, - hw_data->dev_class->name, hw_data->instance_id); + snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", + ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, + pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); if (!accel_dev->debugfs_dir) { - dev_err(&pdev->dev, "Could not create debugfs dir\n"); + dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); ret = -EINVAL; goto out_err; } @@ -301,7 +307,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { dev_err(&pdev->dev, "No usable DMA configuration\n"); ret = -EFAULT; - goto out_err; + goto out_err_disable; } else { pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); } @@ -312,7 +318,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (pci_request_regions(pdev, adf_driver_name)) { ret = -EFAULT; - goto out_err; + goto out_err_disable; } /* Read accelerator capabilities mask */ @@ -320,19 +326,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) &hw_data->accel_capabilities_mask); /* Find and map all the device's BARS */ - for (i = 0; i < ADF_PCI_MAX_BARS; i++) { - struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, + ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; - bar_nr = i * 2; bar->base_addr = pci_resource_start(pdev, bar_nr); if (!bar->base_addr) break; bar->size = pci_resource_len(pdev, bar_nr); bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); if (!bar->virt_addr) { - dev_err(&pdev->dev, "Failed to map BAR %d\n", i); + dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); ret = -EFAULT; - goto out_err; + goto out_err_free_reg; } } pci_set_master(pdev); @@ -340,32 +348,40 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (adf_enable_aer(accel_dev, &adf_driver)) { dev_err(&pdev->dev, "Failed to enable aer\n"); ret = -EFAULT; - goto out_err; + goto out_err_free_reg; } if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state\n"); ret = -ENOMEM; - goto out_err; + goto out_err_free_reg; } ret = adf_dev_configure(accel_dev); if (ret) - goto out_err; + goto out_err_free_reg; ret = adf_dev_init(accel_dev); if (ret) - goto out_err; + goto out_err_dev_shutdown; ret = adf_dev_start(accel_dev); - if (ret) { - adf_dev_stop(accel_dev); - goto out_err; - } + if (ret) + goto out_err_dev_stop; - return 0; + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); out_err: adf_cleanup_accel(accel_dev); + kfree(accel_dev); return ret; } @@ -379,8 +395,12 @@ static void adf_remove(struct pci_dev *pdev) } if (adf_dev_stop(accel_dev)) dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + + adf_dev_shutdown(accel_dev); adf_disable_aer(accel_dev); adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); } static int __init adfdrv_init(void) diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c index 0d03c10..5570f78 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_isr.c @@ -59,21 +59,30 @@ #include #include #include "adf_drv.h" +#include "adf_dh895xcc_hw_data.h" static int adf_enable_msix(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t msix_num_entries = hw_data->num_banks + 1; - int i; - - for (i = 0; i < msix_num_entries; i++) - pci_dev_info->msix_entries.entries[i].entry = i; + u32 msix_num_entries = 1; + + /* If SR-IOV is disabled, add entries for each bank */ + if (!accel_dev->pf.vf_info) { + int i; + + msix_num_entries += hw_data->num_banks; + for (i = 0; i < msix_num_entries; i++) + pci_dev_info->msix_entries.entries[i].entry = i; + } else { + pci_dev_info->msix_entries.entries[0].entry = + hw_data->num_banks; + } if (pci_enable_msix_exact(pci_dev_info->pci_dev, pci_dev_info->msix_entries.entries, msix_num_entries)) { - dev_err(&GET_DEV(accel_dev), "Failed to enable MSIX IRQ\n"); + dev_err(&GET_DEV(accel_dev), "Failed to enable MSI-X IRQ(s)\n"); return -EFAULT; } return 0; @@ -97,9 +106,58 @@ static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) { struct adf_accel_dev *accel_dev = dev_ptr; - dev_info(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", - accel_dev->accel_id); - return IRQ_HANDLED; +#ifdef CONFIG_PCI_IOV + /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ + if (accel_dev->pf.vf_info) { + void __iomem *pmisc_bar_addr = + (&GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR])->virt_addr; + u32 vf_mask; + + /* Get the interrupt sources triggered by VFs */ + vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU5) & + 0x0000FFFF) << 16) | + ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU3) & + 0x01FFFE00) >> 9); + + if (vf_mask) { + struct adf_accel_vf_info *vf_info; + bool irq_handled = false; + int i; + + /* Disable VF2PF interrupts for VFs with pending ints */ + adf_disable_vf2pf_interrupts(accel_dev, vf_mask); + + /* + * Schedule tasklets to handle VF2PF interrupt BHs + * unless the VF is malicious and is attempting to + * flood the host OS with VF2PF interrupts. + */ + for_each_set_bit(i, (const unsigned long *)&vf_mask, + (sizeof(vf_mask) * BITS_PER_BYTE)) { + vf_info = accel_dev->pf.vf_info + i; + + if (!__ratelimit(&vf_info->vf2pf_ratelimit)) { + dev_info(&GET_DEV(accel_dev), + "Too many ints from VF%d\n", + vf_info->vf_nr + 1); + continue; + } + + /* Tasklet will re-enable ints from this VF */ + tasklet_hi_schedule(&vf_info->vf2pf_bh_tasklet); + irq_handled = true; + } + + if (irq_handled) + return IRQ_HANDLED; + } + } +#endif /* CONFIG_PCI_IOV */ + + dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n", + accel_dev->accel_id); + + return IRQ_NONE; } static int adf_request_irqs(struct adf_accel_dev *accel_dev) @@ -108,28 +166,32 @@ static int adf_request_irqs(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct msix_entry *msixe = pci_dev_info->msix_entries.entries; struct adf_etr_data *etr_data = accel_dev->transport; - int ret, i; + int ret, i = 0; char *name; - /* Request msix irq for all banks */ - for (i = 0; i < hw_data->num_banks; i++) { - struct adf_etr_bank_data *bank = &etr_data->banks[i]; - unsigned int cpu, cpus = num_online_cpus(); - - name = *(pci_dev_info->msix_entries.names + i); - snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, - "qat%d-bundle%d", accel_dev->accel_id, i); - ret = request_irq(msixe[i].vector, - adf_msix_isr_bundle, 0, name, bank); - if (ret) { - dev_err(&GET_DEV(accel_dev), - "failed to enable irq %d for %s\n", - msixe[i].vector, name); - return ret; + /* Request msix irq for all banks unless SR-IOV enabled */ + if (!accel_dev->pf.vf_info) { + for (i = 0; i < hw_data->num_banks; i++) { + struct adf_etr_bank_data *bank = &etr_data->banks[i]; + unsigned int cpu, cpus = num_online_cpus(); + + name = *(pci_dev_info->msix_entries.names + i); + snprintf(name, ADF_MAX_MSIX_VECTOR_NAME, + "qat%d-bundle%d", accel_dev->accel_id, i); + ret = request_irq(msixe[i].vector, + adf_msix_isr_bundle, 0, name, bank); + if (ret) { + dev_err(&GET_DEV(accel_dev), + "failed to enable irq %d for %s\n", + msixe[i].vector, name); + return ret; + } + + cpu = ((accel_dev->accel_id * hw_data->num_banks) + + i) % cpus; + irq_set_affinity_hint(msixe[i].vector, + get_cpu_mask(cpu)); } - - cpu = ((accel_dev->accel_id * hw_data->num_banks) + i) % cpus; - irq_set_affinity_hint(msixe[i].vector, get_cpu_mask(cpu)); } /* Request msix irq for AE */ @@ -152,11 +214,13 @@ static void adf_free_irqs(struct adf_accel_dev *accel_dev) struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct msix_entry *msixe = pci_dev_info->msix_entries.entries; struct adf_etr_data *etr_data = accel_dev->transport; - int i; + int i = 0; - for (i = 0; i < hw_data->num_banks; i++) { - irq_set_affinity_hint(msixe[i].vector, NULL); - free_irq(msixe[i].vector, &etr_data->banks[i]); + if (pci_dev_info->msix_entries.num_entries > 1) { + for (i = 0; i < hw_data->num_banks; i++) { + irq_set_affinity_hint(msixe[i].vector, NULL); + free_irq(msixe[i].vector, &etr_data->banks[i]); + } } irq_set_affinity_hint(msixe[i].vector, NULL); free_irq(msixe[i].vector, accel_dev); @@ -168,7 +232,11 @@ static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) char **names; struct msix_entry *entries; struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t msix_num_entries = hw_data->num_banks + 1; + u32 msix_num_entries = 1; + + /* If SR-IOV is disabled (vf_info is NULL), add entries for each bank */ + if (!accel_dev->pf.vf_info) + msix_num_entries += hw_data->num_banks; entries = kzalloc_node(msix_num_entries * sizeof(*entries), GFP_KERNEL, dev_to_node(&GET_DEV(accel_dev))); @@ -185,6 +253,7 @@ static int adf_isr_alloc_msix_entry_table(struct adf_accel_dev *accel_dev) if (!(*(names + i))) goto err; } + accel_dev->accel_pci_dev.msix_entries.num_entries = msix_num_entries; accel_dev->accel_pci_dev.msix_entries.entries = entries; accel_dev->accel_pci_dev.msix_entries.names = names; return 0; @@ -198,13 +267,11 @@ err: static void adf_isr_free_msix_entry_table(struct adf_accel_dev *accel_dev) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; - uint32_t msix_num_entries = hw_data->num_banks + 1; char **names = accel_dev->accel_pci_dev.msix_entries.names; int i; kfree(accel_dev->accel_pci_dev.msix_entries.entries); - for (i = 0; i < msix_num_entries; i++) + for (i = 0; i < accel_dev->accel_pci_dev.msix_entries.num_entries; i++) kfree(*(names + i)); kfree(names); } -- cgit v1.1