MFC 305502: Reset PCI pass through devices via PCI-e FLR during VM start/end.

Add routines to trigger a function level reset (FLR) of a PCI-express
device via the PCI-express device control register.  This also includes
support routines to wait for pending transactions to complete as well
as calculating the maximum completion timeout permitted by a device.

Change the ppt(4) driver to reset pass through devices before attaching
to a VM during startup and before detaching from a VM during shutdown.

Sponsored by:	Chelsio Communications

2 files changed, 102 insertions, 0 deletions

diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index 251a0d9..1a438c6 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -1319,7 +1319,10 @@ MLINKS+=pci.9 pci_alloc_msi.9 \
 	pci.9 pci_set_max_read_req.9 \
 	pci.9 pci_write_config.9 \
 	pci.9 pcie_adjust_config.9 \
+	pci.9 pcie_flr.9 \
+	pci.9 pcie_max_completion_timeout.9 \
 	pci.9 pcie_read_config.9 \
+	pci.9 pcie_wait_for_pending_transactions.9 \
 	pci.9 pcie_write_config.9
 MLINKS+=pci_iov_schema.9 pci_iov_schema_alloc_node.9 \
 	pci_iov_schema.9 pci_iov_schema_add_bool.9 \
diff --git a/share/man/man9/pci.9 b/share/man/man9/pci.9
index 209986b..d8b4b91 100644
--- a/share/man/man9/pci.9
+++ b/share/man/man9/pci.9
@@ -66,7 +66,10 @@
 .Nm pci_set_powerstate ,
 .Nm pci_write_config ,
 .Nm pcie_adjust_config ,
+.Nm pcie_flr ,
+.Nm pcie_get_max_completion_timeout ,
 .Nm pcie_read_config ,
+.Nm pcie_wait_for_pending_transactions ,
 .Nm pcie_write_config
 .Nd PCI bus interface
 .Sh SYNOPSIS
@@ -145,8 +148,14 @@
 .Fa "uint32_t val"
 .Fa "int width"
 .Fc
+.Ft bool
+.Fn pcie_flr "device_t dev" "u_int max_delay" "bool force"
+.Ft int
+.Fn pcie_get_max_completion_timeout "device_t dev"
 .Ft uint32_t
 .Fn pcie_read_config "device_t dev" "int reg" "int width"
+.Ft bool
+.Fn pcie_wait_for_pending_transactions "device_t dev" "u_int max_delay"
 .Ft void
 .Fn pcie_write_config "device_t dev" "int reg" "uint32_t val" "int width"
 .Ft void
@@ -431,6 +440,51 @@ keyword,
 then
 .Fn pci_get_vpd_readonly
 returns an error.
+.Pp
+The
+.Fn pcie_get_max_completion_timeout
+function returns the maximum completion timeout configured for the device
+.Fa dev
+in microseconds.
+If the
+.Fa dev
+device is not a PCI-express device,
+.Fn pcie_get_max_completion_timeout
+returns zero.
+When completion timeouts are disabled for
+.Fa dev ,
+this function returns the maxmimum timeout that would be used if timeouts
+were enabled.
+.Pp
+The
+.Fn pcie_wait_for_pending_transactions
+function waits for any pending transactions initiated by the
+.Fa dev
+device to complete.
+The function checks for pending transactions by polling the transactions
+pending flag in the PCI-express device status register.
+It returns
+.Dv true
+once the transaction pending flag is clear.
+If transactions are still pending after
+.Fa max_delay
+milliseconds,
+.Fn pcie_wait_for_pending_transactions
+returns
+.Dv false .
+If
+.Fa max_delay
+is set to zero,
+.Fn pcie_wait_for_pending_transactions
+performs a single check;
+otherwise,
+this function may sleep while polling the transactions pending flag.
+.Nm pcie_wait_for_pending_transactions
+returns
+.Dv true
+if
+.Fa dev
+is not a PCI-express device.
 .Ss Device Configuration
 The
 .Fn pci_enable_busmaster
@@ -662,6 +716,51 @@ is invoked,
 then the device will be transitioned to
 .Dv PCI_POWERSTATE_D0
 before any config registers are restored.
+.Pp
+The
+.Fn pcie_flr
+function requests a Function Level Reset
+.Pq FLR
+of
+.Fa dev .
+If
+.Fa dev
+is not a PCI-express device or does not support Function Level Resets via
+the PCI-express device control register,
+.Dv false
+is returned.
+Pending transactions are drained by disabling busmastering and calling
+.Fn pcie_wait_for_pending_transactions
+before resetting the device.
+The
+.Fa max_delay
+argument specifies the maximum timeout to wait for pending transactions as
+described for
+.Fn pcie_wait_for_pending_transactions .
+If
+.Fn pcie_wait_for_pending_transactions
+fails with a timeout and
+.Fa force
+is
+.Dv false ,
+busmastering is re-enabled and
+.Dv false
+is returned.
+If
+.Fn pcie_wait_for_pending_transactions
+fails with a timeout and
+.Fa force
+is
+.Dv true ,
+the device is reset despite the timeout.
+After the reset has been requested,
+.Nm pcie_flr
+sleeps for at least 100 milliseconds before returning
+.Dv true .
+Note that
+.Nm pcie_flr
+does not save and restore any state around the reset.
+The caller should save and restore state as needed.
 .Ss Message Signaled Interrupts
 Message Signaled Interrupts
 .Pq MSI