From 29fedc4c2cc4799d9f7f47ae227df532fae74c15 Mon Sep 17 00:00:00 2001 From: rstone Date: Sun, 1 Mar 2015 00:39:48 +0000 Subject: Allow passthrough devices to be hinted. Allow the ppt driver to attach to devices that were hinted to be passthrough devices by the PCI code creating them with a driver name of "ppt". Add a tunable that allows the IOMMU to be forced to be used. With SR-IOV passthrough devices the VFs may be created after vmm.ko is loaded. The current code will not initialize the IOMMU in that case, meaning that the passthrough devices can't actually be used. Differential Revision: https://reviews.freebsd.org/D73 Reviewed by: neel MFC after: 1 month Sponsored by: Sandvine Inc. --- sys/amd64/vmm/io/ppt.c | 78 +++++++++++++++++++++++++++++--------------------- sys/amd64/vmm/vmm.c | 7 ++++- 2 files changed, 51 insertions(+), 34 deletions(-) (limited to 'sys/amd64/vmm') diff --git a/sys/amd64/vmm/io/ppt.c b/sys/amd64/vmm/io/ppt.c index fa7083e..b789f77 100644 --- a/sys/amd64/vmm/io/ppt.c +++ b/sys/amd64/vmm/io/ppt.c @@ -56,7 +56,6 @@ __FBSDID("$FreeBSD$"); /* XXX locking */ -#define MAX_PPTDEVS (sizeof(pptdevs) / sizeof(pptdevs[0])) #define MAX_MSIMSGS 32 /* @@ -77,9 +76,10 @@ struct pptintr_arg { /* pptintr(pptintr_arg) */ uint64_t msg_data; }; -static struct pptdev { +struct pptdev { device_t dev; struct vm *vm; /* owner of this device */ + TAILQ_ENTRY(pptdev) next; struct vm_memory_segment mmio[MAX_MMIOSEGS]; struct { int num_msgs; /* guest state */ @@ -99,7 +99,7 @@ static struct pptdev { void **cookie; struct pptintr_arg *arg; } msix; -} pptdevs[64]; +}; SYSCTL_DECL(_hw_vmm); SYSCTL_NODE(_hw_vmm, OID_AUTO, ppt, CTLFLAG_RW, 0, "bhyve passthru devices"); @@ -108,6 +108,8 @@ static int num_pptdevs; SYSCTL_INT(_hw_vmm_ppt, OID_AUTO, devices, CTLFLAG_RD, &num_pptdevs, 0, "number of pci passthru devices"); +static TAILQ_HEAD(, pptdev) pptdev_list = TAILQ_HEAD_INITIALIZER(pptdev_list); + static int ppt_probe(device_t dev) { @@ -125,26 +127,30 @@ ppt_probe(device_t dev) * - be allowed by administrator to be used in this role * - be an endpoint device */ - if (vmm_is_pptdev(bus, slot, func) && - (dinfo->cfg.hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_NORMAL) + if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL) + return (ENXIO); + else if (vmm_is_pptdev(bus, slot, func)) return (0); else - return (ENXIO); + /* + * Returning BUS_PROBE_NOWILDCARD here matches devices that the + * SR-IOV infrastructure specified as "ppt" passthrough devices. + * All normal devices that did not have "ppt" specified as their + * driver will not be matched by this. + */ + return (BUS_PROBE_NOWILDCARD); } static int ppt_attach(device_t dev) { - int n; + struct pptdev *ppt; - if (num_pptdevs >= MAX_PPTDEVS) { - printf("ppt_attach: maximum number of pci passthrough devices " - "exceeded\n"); - return (ENXIO); - } + ppt = device_get_softc(dev); - n = num_pptdevs++; - pptdevs[n].dev = dev; + num_pptdevs++; + TAILQ_INSERT_TAIL(&pptdev_list, ppt, next); + ppt->dev = dev; if (bootverbose) device_printf(dev, "attached\n"); @@ -155,10 +161,14 @@ ppt_attach(device_t dev) static int ppt_detach(device_t dev) { - /* - * XXX check whether there are any pci passthrough devices assigned - * to guests before we allow this driver to detach. - */ + struct pptdev *ppt; + + ppt = device_get_softc(dev); + + if (ppt->vm != NULL) + return (EBUSY); + num_pptdevs--; + TAILQ_REMOVE(&pptdev_list, ppt, next); return (0); } @@ -172,22 +182,23 @@ static device_method_t ppt_methods[] = { }; static devclass_t ppt_devclass; -DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, 0); +DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, sizeof(struct pptdev)); DRIVER_MODULE(ppt, pci, ppt_driver, ppt_devclass, NULL, NULL); static struct pptdev * ppt_find(int bus, int slot, int func) { device_t dev; - int i, b, s, f; + struct pptdev *ppt; + int b, s, f; - for (i = 0; i < num_pptdevs; i++) { - dev = pptdevs[i].dev; + TAILQ_FOREACH(ppt, &pptdev_list, next) { + dev = ppt->dev; b = pci_get_bus(dev); s = pci_get_slot(dev); f = pci_get_function(dev); if (bus == b && slot == s && func == f) - return (&pptdevs[i]); + return (ppt); } return (NULL); } @@ -297,11 +308,12 @@ ppt_avail_devices(void) int ppt_assigned_devices(struct vm *vm) { - int i, num; + struct pptdev *ppt; + int num; num = 0; - for (i = 0; i < num_pptdevs; i++) { - if (pptdevs[i].vm == vm) + TAILQ_FOREACH(ppt, &pptdev_list, next) { + if (ppt->vm == vm) num++; } return (num); @@ -310,12 +322,11 @@ ppt_assigned_devices(struct vm *vm) boolean_t ppt_is_mmio(struct vm *vm, vm_paddr_t gpa) { - int i, n; + int i; struct pptdev *ppt; struct vm_memory_segment *seg; - for (n = 0; n < num_pptdevs; n++) { - ppt = &pptdevs[n]; + TAILQ_FOREACH(ppt, &pptdev_list, next) { if (ppt->vm != vm) continue; @@ -377,12 +388,13 @@ ppt_unassign_device(struct vm *vm, int bus, int slot, int func) int ppt_unassign_all(struct vm *vm) { - int i, bus, slot, func; + struct pptdev *ppt; + int bus, slot, func; device_t dev; - for (i = 0; i < num_pptdevs; i++) { - if (pptdevs[i].vm == vm) { - dev = pptdevs[i].dev; + TAILQ_FOREACH(ppt, &pptdev_list, next) { + if (ppt->vm == vm) { + dev = ppt->dev; bus = pci_get_bus(dev); slot = pci_get_slot(dev); func = pci_get_function(dev); diff --git a/sys/amd64/vmm/vmm.c b/sys/amd64/vmm/vmm.c index 615a639..9e44549 100644 --- a/sys/amd64/vmm/vmm.c +++ b/sys/amd64/vmm/vmm.c @@ -217,6 +217,11 @@ SYSCTL_INT(_hw_vmm, OID_AUTO, trace_guest_exceptions, CTLFLAG_RDTUN, &trace_guest_exceptions, 0, "Trap into hypervisor on all guest exceptions and reflect them back"); +static int vmm_force_iommu = 0; +TUNABLE_INT("hw.vmm.force_iommu", &vmm_force_iommu); +SYSCTL_INT(_hw_vmm, OID_AUTO, force_iommu, CTLFLAG_RDTUN, &vmm_force_iommu, 0, + "Force use of I/O MMU even if no passthrough devices were found."); + static void vcpu_cleanup(struct vm *vm, int i, bool destroy) { @@ -321,7 +326,7 @@ vmm_handler(module_t mod, int what, void *arg) switch (what) { case MOD_LOAD: vmmdev_init(); - if (ppt_avail_devices() > 0) + if (vmm_force_iommu || ppt_avail_devices() > 0) iommu_init(); error = vmm_init(); if (error == 0) -- cgit v1.1