2 files changed, 51 insertions, 34 deletions

diff --git a/sys/amd64/vmm/io/ppt.c b/sys/amd64/vmm/io/ppt.c
index fa7083e..b789f77 100644
--- a/sys/amd64/vmm/io/ppt.c
+++ b/sys/amd64/vmm/io/ppt.c
@@ -56,7 +56,6 @@ __FBSDID("$FreeBSD$");
 
 /* XXX locking */
 
-#define	MAX_PPTDEVS	(sizeof(pptdevs) / sizeof(pptdevs[0]))
 #define	MAX_MSIMSGS	32
 
 /*
@@ -77,9 +76,10 @@ struct pptintr_arg {				/* pptintr(pptintr_arg) */
 	uint64_t	msg_data;
 };
 
-static struct pptdev {
+struct pptdev {
 	device_t	dev;
 	struct vm	*vm;			/* owner of this device */
+	TAILQ_ENTRY(pptdev)	next;
 	struct vm_memory_segment mmio[MAX_MMIOSEGS];
 	struct {
 		int	num_msgs;		/* guest state */
@@ -99,7 +99,7 @@ static struct pptdev {
 		void **cookie;
 		struct pptintr_arg *arg;
 	} msix;
-} pptdevs[64];
+};
 
 SYSCTL_DECL(_hw_vmm);
 SYSCTL_NODE(_hw_vmm, OID_AUTO, ppt, CTLFLAG_RW, 0, "bhyve passthru devices");
@@ -108,6 +108,8 @@ static int num_pptdevs;
 SYSCTL_INT(_hw_vmm_ppt, OID_AUTO, devices, CTLFLAG_RD, &num_pptdevs, 0,
     "number of pci passthru devices");
 
+static TAILQ_HEAD(, pptdev) pptdev_list = TAILQ_HEAD_INITIALIZER(pptdev_list);
+
 static int
 ppt_probe(device_t dev)
 {
@@ -125,26 +127,30 @@ ppt_probe(device_t dev)
 	 * - be allowed by administrator to be used in this role
 	 * - be an endpoint device
 	 */
-	if (vmm_is_pptdev(bus, slot, func) &&
-	    (dinfo->cfg.hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_NORMAL)
+	if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL)
+		return (ENXIO);
+	else if (vmm_is_pptdev(bus, slot, func))
 		return (0);
 	else
-		return (ENXIO);
+		/*
+		 * Returning BUS_PROBE_NOWILDCARD here matches devices that the
+		 * SR-IOV infrastructure specified as "ppt" passthrough devices.
+		 * All normal devices that did not have "ppt" specified as their
+		 * driver will not be matched by this.
+		 */
+		return (BUS_PROBE_NOWILDCARD);
 }
 
 static int
 ppt_attach(device_t dev)
 {
-	int n;
+	struct pptdev *ppt;
 
-	if (num_pptdevs >= MAX_PPTDEVS) {
-		printf("ppt_attach: maximum number of pci passthrough devices "
-		       "exceeded\n");
-		return (ENXIO);
-	}
+	ppt = device_get_softc(dev);
 
-	n = num_pptdevs++;
-	pptdevs[n].dev = dev;
+	num_pptdevs++;
+	TAILQ_INSERT_TAIL(&pptdev_list, ppt, next);
+	ppt->dev = dev;
 
 	if (bootverbose)
 		device_printf(dev, "attached\n");
@@ -155,10 +161,14 @@ ppt_attach(device_t dev)
 static int
 ppt_detach(device_t dev)
 {
-	/*
-	 * XXX check whether there are any pci passthrough devices assigned
-	 * to guests before we allow this driver to detach.
-	 */
+	struct pptdev *ppt;
+
+	ppt = device_get_softc(dev);
+
+	if (ppt->vm != NULL)
+		return (EBUSY);
+	num_pptdevs--;
+	TAILQ_REMOVE(&pptdev_list, ppt, next);
 
 	return (0);
 }
@@ -172,22 +182,23 @@ static device_method_t ppt_methods[] = {
 };
 
 static devclass_t ppt_devclass;
-DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, 0);
+DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, sizeof(struct pptdev));
 DRIVER_MODULE(ppt, pci, ppt_driver, ppt_devclass, NULL, NULL);
 
 static struct pptdev *
 ppt_find(int bus, int slot, int func)
 {
 	device_t dev;
-	int i, b, s, f;
+	struct pptdev *ppt;
+	int b, s, f;
 
-	for (i = 0; i < num_pptdevs; i++) {
-		dev = pptdevs[i].dev;
+	TAILQ_FOREACH(ppt, &pptdev_list, next) {
+		dev = ppt->dev;
 		b = pci_get_bus(dev);
 		s = pci_get_slot(dev);
 		f = pci_get_function(dev);
 		if (bus == b && slot == s && func == f)
-			return (&pptdevs[i]);
+			return (ppt);
 	}
 	return (NULL);
 }
@@ -297,11 +308,12 @@ ppt_avail_devices(void)
 int
 ppt_assigned_devices(struct vm *vm)
 {
-	int i, num;
+	struct pptdev *ppt;
+	int num;
 
 	num = 0;
-	for (i = 0; i < num_pptdevs; i++) {
-		if (pptdevs[i].vm == vm)
+	TAILQ_FOREACH(ppt, &pptdev_list, next) {
+		if (ppt->vm == vm)
 			num++;
 	}
 	return (num);
@@ -310,12 +322,11 @@ ppt_assigned_devices(struct vm *vm)
 boolean_t
 ppt_is_mmio(struct vm *vm, vm_paddr_t gpa)
 {
-	int i, n;
+	int i;
 	struct pptdev *ppt;
 	struct vm_memory_segment *seg;
 
-	for (n = 0; n < num_pptdevs; n++) {
-		ppt = &pptdevs[n];
+	TAILQ_FOREACH(ppt, &pptdev_list, next) {
 		if (ppt->vm != vm)
 			continue;
 
@@ -377,12 +388,13 @@ ppt_unassign_device(struct vm *vm, int bus, int slot, int func)
 int
 ppt_unassign_all(struct vm *vm)
 {
-	int i, bus, slot, func;
+	struct pptdev *ppt;
+	int bus, slot, func;
 	device_t dev;
 
-	for (i = 0; i < num_pptdevs; i++) {
-		if (pptdevs[i].vm == vm) {
-			dev = pptdevs[i].dev;
+	TAILQ_FOREACH(ppt, &pptdev_list, next) {
+		if (ppt->vm == vm) {
+			dev = ppt->dev;
 			bus = pci_get_bus(dev);
 			slot = pci_get_slot(dev);
 			func = pci_get_function(dev);
diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c
index 615a639..9e44549 100644
--- a/sys/amd64/vmm/vmm.c
+++ b/sys/amd64/vmm/vmm.c
@@ -217,6 +217,11 @@ SYSCTL_INT(_hw_vmm, OID_AUTO, trace_guest_exceptions, CTLFLAG_RDTUN,
     &trace_guest_exceptions, 0,
     "Trap into hypervisor on all guest exceptions and reflect them back");
 
+static int vmm_force_iommu = 0;
+TUNABLE_INT("hw.vmm.force_iommu", &vmm_force_iommu);
+SYSCTL_INT(_hw_vmm, OID_AUTO, force_iommu, CTLFLAG_RDTUN, &vmm_force_iommu, 0,
+    "Force use of I/O MMU even if no passthrough devices were found.");
+
 static void
 vcpu_cleanup(struct vm *vm, int i, bool destroy)
 {
@@ -321,7 +326,7 @@ vmm_handler(module_t mod, int what, void *arg)
 	switch (what) {
 	case MOD_LOAD:
 		vmmdev_init();
-		if (ppt_avail_devices() > 0)
+		if (vmm_force_iommu || ppt_avail_devices() > 0)
 			iommu_init();
 		error = vmm_init();
 		if (error == 0)