Save and restore SRIOV-related config registers.

Save the value of the IOV control and page size registers and restore
them (along with the VF count) in pci_cfg_save/pci_cfg_restore.  This
ensures ARI remains enabled if a PF driver resets itself during the
PCI_IOV_INIT callback.  This might also properly restore SRIOV state
across suspend/resume.

Reviewed by:	rstone, vangyzen
Differential Revision:	https://reviews.freebsd.org/D6192

3 files changed, 36 insertions, 0 deletions

diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
index 630aaa9..ae01f24 100644
--- a/sys/dev/pci/pci.c
+++ b/sys/dev/pci/pci.c
@@ -5643,6 +5643,10 @@ pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
 		pci_resume_msi(dev);
 	if (dinfo->cfg.msix.msix_location != 0)
 		pci_resume_msix(dev);
+
+	if (dinfo->cfg.iov != NULL)
+		pci_iov_cfg_restore(dev, dinfo);
+
 }
 
 static void
@@ -5755,6 +5759,9 @@ pci_cfg_save(device_t dev, struct pci_devinfo *dinfo, int setstate)
 	if (dinfo->cfg.pcix.pcix_location != 0)
 		pci_cfg_save_pcix(dev, dinfo);
 
+	if (dinfo->cfg.iov != NULL)
+		pci_iov_cfg_save(dev, dinfo);
+
 	/*
 	 * don't set the state for display devices, base peripherals and
 	 * memory devices since bad things happen when they are powered down.
diff --git a/sys/dev/pci/pci_iov.c b/sys/dev/pci/pci_iov.c
index 337441d..bba99f6 100644
--- a/sys/dev/pci/pci_iov.c
+++ b/sys/dev/pci/pci_iov.c
@@ -770,6 +770,29 @@ out:
 	return (error);
 }
 
+void
+pci_iov_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
+{
+	struct pcicfg_iov *iov;
+
+	iov = dinfo->cfg.iov;
+
+	IOV_WRITE(dinfo, PCIR_SRIOV_PAGE_SIZE, iov->iov_page_size, 4);
+	IOV_WRITE(dinfo, PCIR_SRIOV_NUM_VFS, iov->iov_num_vfs, 2);
+	IOV_WRITE(dinfo, PCIR_SRIOV_CTL, iov->iov_ctl, 2);
+}
+
+void
+pci_iov_cfg_save(device_t dev, struct pci_devinfo *dinfo)
+{
+	struct pcicfg_iov *iov;
+
+	iov = dinfo->cfg.iov;
+
+	iov->iov_page_size = IOV_READ(dinfo, PCIR_SRIOV_PAGE_SIZE, 4);
+	iov->iov_ctl = IOV_READ(dinfo, PCIR_SRIOV_CTL, 2);
+}
+
 /* Return true if child is a VF of the given PF. */
 static int
 pci_iov_is_child_vf(struct pcicfg_iov *pf, device_t child)
diff --git a/sys/dev/pci/pci_iov_private.h b/sys/dev/pci/pci_iov_private.h
index 6938e2e..2d15353 100644
--- a/sys/dev/pci/pci_iov_private.h
+++ b/sys/dev/pci/pci_iov_private.h
@@ -47,10 +47,16 @@ struct pcicfg_iov {
 	int iov_pos;
 	int iov_num_vfs;
 	uint32_t iov_flags;
+
+	uint16_t iov_ctl;
+	uint32_t iov_page_size;
 };
 
 #define	IOV_RMAN_INITED		0x0001
 #define	IOV_BUSY		0x0002
 
+void	pci_iov_cfg_restore(device_t dev, struct pci_devinfo *dinfo);
+void	pci_iov_cfg_save(device_t dev, struct pci_devinfo *dinfo);
+
 #endif