diff options
author | neel <neel@FreeBSD.org> | 2015-06-18 06:00:17 +0000 |
---|---|---|
committer | neel <neel@FreeBSD.org> | 2015-06-18 06:00:17 +0000 |
commit | 8c70d6c7af8325d488ec8dff7b0bbe7b6dfc32f9 (patch) | |
tree | 34972705799ce1ec23ee45b1e6c0c7631ad7ed98 /usr.sbin/bhyve | |
parent | 07553194d65f7f87d0eea74f0ad0f786e9a189bb (diff) | |
download | FreeBSD-src-8c70d6c7af8325d488ec8dff7b0bbe7b6dfc32f9.zip FreeBSD-src-8c70d6c7af8325d488ec8dff7b0bbe7b6dfc32f9.tar.gz |
Restructure memory allocation in bhyve to support "devmem".
devmem is used to represent MMIO devices like the boot ROM or a VESA framebuffer
where doing a trap-and-emulate for every access is impractical. devmem is a
hybrid of system memory (sysmem) and emulated device models.
devmem is mapped in the guest address space via nested page tables similar
to sysmem. However the address range where devmem is mapped may be changed
by the guest at runtime (e.g. by reprogramming a PCI BAR). Also devmem is
usually mapped RO or RW as compared to RWX mappings for sysmem.
Each devmem segment is named (e.g. "bootrom") and this name is used to
create a device node for the devmem segment (e.g. /dev/vmm/testvm.bootrom).
The device node supports mmap(2) and this decouples the host mapping of
devmem from its mapping in the guest address space (which can change).
Reviewed by: tychon
Discussed with: grehan
Differential Revision: https://reviews.freebsd.org/D2762
MFC after: 4 weeks
Diffstat (limited to 'usr.sbin/bhyve')
-rw-r--r-- | usr.sbin/bhyve/Makefile | 1 | ||||
-rw-r--r-- | usr.sbin/bhyve/bhyve.8 | 28 | ||||
-rw-r--r-- | usr.sbin/bhyve/bhyverun.c | 93 | ||||
-rw-r--r-- | usr.sbin/bhyve/bootrom.c | 111 | ||||
-rw-r--r-- | usr.sbin/bhyve/bootrom.h | 38 | ||||
-rw-r--r-- | usr.sbin/bhyve/pci_lpc.c | 27 | ||||
-rw-r--r-- | usr.sbin/bhyve/pci_lpc.h | 1 | ||||
-rw-r--r-- | usr.sbin/bhyve/pci_passthru.c | 8 |
8 files changed, 283 insertions, 24 deletions
diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile index bb81bcb..be5cb33 100644 --- a/usr.sbin/bhyve/Makefile +++ b/usr.sbin/bhyve/Makefile @@ -13,6 +13,7 @@ SRCS= \ acpi.c \ bhyverun.c \ block_if.c \ + bootrom.c \ consport.c \ dbgport.c \ inout.c \ diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8 index ee0f2ca..c5de82a 100644 --- a/usr.sbin/bhyve/bhyve.8 +++ b/usr.sbin/bhyve/bhyve.8 @@ -32,7 +32,7 @@ .Nd "run a guest operating system inside a virtual machine" .Sh SYNOPSIS .Nm -.Op Fl abehuwxACHPWY +.Op Fl abehuwxACHPSWY .Op Fl c Ar numcpus .Op Fl g Ar gdbport .Op Fl l Ar lpcdev Ns Op , Ns Ar conf @@ -99,10 +99,12 @@ Yield the virtual CPU thread when a HLT instruction is detected. If this option is not specified, virtual CPUs will use 100% of a host CPU. .It Fl l Ar lpcdev Ns Op , Ns Ar conf Allow devices behind the LPC PCI-ISA bridge to be configured. -The only supported devices are the TTY-class devices, -.Li com1 +The only supported devices are the TTY-class devices +.Ar com1 and -.Li com2 . +.Ar com2 +and the boot ROM device +.Ar bootrom . .It Fl m Ar size Ns Op Ar K|k|M|m|G|g|T|t Guest physical memory size in bytes. This must be the same size that was given to @@ -165,8 +167,8 @@ AHCI controller attached to a SATA hard-drive. .It Li uart PCI 16550 serial device. .It Li lpc -LPC PCI-ISA bridge with COM1 and COM2 16550 serial ports. The LPC bridge -emulation can only be configured on bus 0. +LPC PCI-ISA bridge with COM1 and COM2 16550 serial ports and a boot ROM. +The LPC bridge emulation can only be configured on bus 0. .El .It Op Ar conf This optional parameter describes the backend for device emulations. @@ -224,6 +226,14 @@ the bhyve process. Use the host TTY device for serial port I/O. .El .Pp +Boot ROM device: +.Bl -tag -width 10n +.It Pa romfile +Map +.Ar romfile +in the guest address space reserved for boot firmware. +.El +.Pp Pass-through devices: .Bl -tag -width 10n .It Ns Ar slot Ns / Ns Ar bus Ns / Ns Ar function @@ -235,11 +245,17 @@ and numbers. .El .Pp +Guest memory must be wired using the +.Fl S +option when a pass-through device is configured. +.Pp The host device must have been reserved at boot-time using the .Va pptdev loader variable as described in .Xr vmm 4 . .El +.It Fl S +Wire guest memory. .It Fl u RTC keeps UTC time. .It Fl U Ar uuid diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c index ee0f106..cfb3fe4 100644 --- a/usr.sbin/bhyve/bhyverun.c +++ b/usr.sbin/bhyve/bhyverun.c @@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$"); #include <pthread.h> #include <pthread_np.h> #include <sysexits.h> +#include <stdbool.h> #include <machine/vmm.h> #include <vmmapi.h> @@ -122,7 +123,7 @@ usage(int code) { fprintf(stderr, - "Usage: %s [-abehuwxACHPWY] [-c vcpus] [-g <gdb port>] [-l <lpc>]\n" + "Usage: %s [-abehuwxACHPSWY] [-c vcpus] [-g <gdb port>] [-l <lpc>]\n" " %*s [-m mem] [-p vcpu:hostcpu] [-s <pci>] [-U uuid] <vm>\n" " -a: local apic is in xAPIC mode (deprecated)\n" " -A: create ACPI tables\n" @@ -137,6 +138,7 @@ usage(int code) " -p: pin 'vcpu' to 'hostcpu'\n" " -P: vmexit from the guest on pause\n" " -s: <slot,driver,configinfo> PCI slot config\n" + " -S: guest memory cannot be swapped\n" " -u: RTC keeps UTC time\n" " -U: uuid\n" " -w: ignore unimplemented MSRs\n" @@ -700,26 +702,82 @@ fbsdrun_set_capabilities(struct vmctx *ctx, int cpu) vm_set_capability(ctx, cpu, VM_CAP_ENABLE_INVPCID, 1); } +static struct vmctx * +do_open(const char *vmname) +{ + struct vmctx *ctx; + int error; + bool reinit, romboot; + + reinit = romboot = false; + + if (lpc_bootrom()) + romboot = true; + + error = vm_create(vmname); + if (error) { + if (errno == EEXIST) { + if (romboot) { + reinit = true; + } else { + /* + * The virtual machine has been setup by the + * userspace bootloader. + */ + } + } else { + perror("vm_create"); + exit(1); + } + } else { + if (!romboot) { + /* + * If the virtual machine was just created then a + * bootrom must be configured to boot it. + */ + fprintf(stderr, "virtual machine cannot be booted\n"); + exit(1); + } + } + + ctx = vm_open(vmname); + if (ctx == NULL) { + perror("vm_open"); + exit(1); + } + + if (reinit) { + error = vm_reinit(ctx); + if (error) { + perror("vm_reinit"); + exit(1); + } + } + return (ctx); +} + int main(int argc, char *argv[]) { int c, error, gdb_port, err, bvmcons; - int dump_guest_memory, max_vcpus, mptgen; + int max_vcpus, mptgen, memflags; int rtc_localtime; struct vmctx *ctx; uint64_t rip; size_t memsize; + char *optstr; bvmcons = 0; - dump_guest_memory = 0; progname = basename(argv[0]); gdb_port = 0; guest_ncpus = 1; memsize = 256 * MB; mptgen = 1; rtc_localtime = 1; + memflags = 0; - while ((c = getopt(argc, argv, "abehuwxACHIPWYp:g:c:s:m:l:U:")) != -1) { + optstr = "abehuwxACHIPSWYp:g:c:s:m:l:U:"; + while ((c = getopt(argc, argv, optstr)) != -1) { switch (c) { case 'a': x2apic_mode = 0; @@ -740,7 +798,7 @@ main(int argc, char *argv[]) guest_ncpus = atoi(optarg); break; case 'C': - dump_guest_memory = 1; + memflags |= VM_MEM_F_INCORE; break; case 'g': gdb_port = atoi(optarg); @@ -756,6 +814,9 @@ main(int argc, char *argv[]) exit(1); else break; + case 'S': + memflags |= VM_MEM_F_WIRED; + break; case 'm': error = vm_parse_memsize(optarg, &memsize); if (error) @@ -810,12 +871,7 @@ main(int argc, char *argv[]) usage(1); vmname = argv[0]; - - ctx = vm_open(vmname); - if (ctx == NULL) { - perror("vm_open"); - exit(1); - } + ctx = do_open(vmname); if (guest_ncpus < 1) { fprintf(stderr, "Invalid guest vCPUs (%d)\n", guest_ncpus); @@ -831,11 +887,10 @@ main(int argc, char *argv[]) fbsdrun_set_capabilities(ctx, BSP); - if (dump_guest_memory) - vm_set_memflags(ctx, VM_MEM_F_INCORE); + vm_set_memflags(ctx, memflags); err = vm_setup_memory(ctx, memsize, VM_MMAP_ALL); if (err) { - fprintf(stderr, "Unable to setup memory (%d)\n", err); + fprintf(stderr, "Unable to setup memory (%d)\n", errno); exit(1); } @@ -865,6 +920,16 @@ main(int argc, char *argv[]) if (bvmcons) init_bvmcons(); + if (lpc_bootrom()) { + if (vm_set_capability(ctx, BSP, VM_CAP_UNRESTRICTED_GUEST, 1)) { + fprintf(stderr, "ROM boot failed: unrestricted guest " + "capability not available\n"); + exit(1); + } + error = vcpu_reset(ctx, BSP); + assert(error == 0); + } + error = vm_get_register(ctx, BSP, VM_REG_GUEST_RIP, &rip); assert(error == 0); diff --git a/usr.sbin/bhyve/bootrom.c b/usr.sbin/bhyve/bootrom.c new file mode 100644 index 0000000..5e4e0e9 --- /dev/null +++ b/usr.sbin/bhyve/bootrom.c @@ -0,0 +1,111 @@ +/*- + * Copyright (c) 2015 Neel Natu <neel@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +__FBSDID("$FreeBSD$"); + +#include <sys/types.h> +#include <sys/mman.h> +#include <sys/stat.h> + +#include <machine/vmm.h> + +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <stdbool.h> + +#include <vmmapi.h> +#include "bhyverun.h" +#include "bootrom.h" + +#define MAX_BOOTROM_SIZE (16 * 1024 * 1024) /* 16 MB */ + +int +bootrom_init(struct vmctx *ctx, const char *romfile) +{ + struct stat sbuf; + vm_paddr_t gpa; + ssize_t rlen; + char *ptr; + int fd, i, rv, prot; + + rv = -1; + fd = open(romfile, O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Error opening bootrom \"%s\": %s\n", + romfile, strerror(errno)); + goto done; + } + + if (fstat(fd, &sbuf) < 0) { + fprintf(stderr, "Could not fstat bootrom file \"%s\": %s\n", + romfile, strerror(errno)); + goto done; + } + + /* + * Limit bootrom size to 16MB so it doesn't encroach into reserved + * MMIO space (e.g. APIC, HPET, MSI). + */ + if (sbuf.st_size > MAX_BOOTROM_SIZE || sbuf.st_size < PAGE_SIZE) { + fprintf(stderr, "Invalid bootrom size %ld\n", sbuf.st_size); + goto done; + } + + if (sbuf.st_size & PAGE_MASK) { + fprintf(stderr, "Bootrom size %ld is not a multiple of the " + "page size\n", sbuf.st_size); + goto done; + } + + ptr = vm_create_devmem(ctx, VM_BOOTROM, "bootrom", sbuf.st_size); + if (ptr == MAP_FAILED) + goto done; + + /* Map the bootrom into the guest address space */ + prot = PROT_READ | PROT_EXEC; + gpa = (1ULL << 32) - sbuf.st_size; + if (vm_mmap_memseg(ctx, gpa, VM_BOOTROM, 0, sbuf.st_size, prot) != 0) + goto done; + + /* Read 'romfile' into the guest address space */ + for (i = 0; i < sbuf.st_size / PAGE_SIZE; i++) { + rlen = read(fd, ptr + i * PAGE_SIZE, PAGE_SIZE); + if (rlen != PAGE_SIZE) { + fprintf(stderr, "Incomplete read of page %d of bootrom " + "file %s: %ld bytes\n", i, romfile, rlen); + goto done; + } + } + rv = 0; +done: + if (fd >= 0) + close(fd); + return (rv); +} diff --git a/usr.sbin/bhyve/bootrom.h b/usr.sbin/bhyve/bootrom.h new file mode 100644 index 0000000..af150d3 --- /dev/null +++ b/usr.sbin/bhyve/bootrom.h @@ -0,0 +1,38 @@ +/*- + * Copyright (c) 2015 Neel Natu <neel@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _BOOTROM_H_ +#define _BOOTROM_H_ + +#include <stdbool.h> + +struct vmctx; + +int bootrom_init(struct vmctx *ctx, const char *romfile); + +#endif diff --git a/usr.sbin/bhyve/pci_lpc.c b/usr.sbin/bhyve/pci_lpc.c index e98b141..2203a00 100644 --- a/usr.sbin/bhyve/pci_lpc.c +++ b/usr.sbin/bhyve/pci_lpc.c @@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$"); #include <vmmapi.h> #include "acpi.h" +#include "bootrom.h" #include "inout.h" #include "pci_emul.h" #include "pci_irq.h" @@ -62,6 +63,8 @@ SYSRES_IO(NMISC_PORT, 1); static struct pci_devinst *lpc_bridge; +static const char *romfile; + #define LPC_UART_NUM 2 static struct lpc_uart_softc { struct uart_softc *uart_softc; @@ -76,7 +79,7 @@ static const char *lpc_uart_names[LPC_UART_NUM] = { "COM1", "COM2" }; /* * LPC device configuration is in the following form: * <lpc_device_name>[,<options>] - * For e.g. "com1,stdio" + * For e.g. "com1,stdio" or "bootrom,/var/romfile" */ int lpc_device_parse(const char *opts) @@ -88,6 +91,11 @@ lpc_device_parse(const char *opts) str = cpy = strdup(opts); lpcdev = strsep(&str, ","); if (lpcdev != NULL) { + if (strcasecmp(lpcdev, "bootrom") == 0) { + romfile = str; + error = 0; + goto done; + } for (unit = 0; unit < LPC_UART_NUM; unit++) { if (strcasecmp(lpcdev, lpc_uart_names[unit]) == 0) { lpc_uart_softc[unit].opts = str; @@ -104,6 +112,13 @@ done: return (error); } +const char * +lpc_bootrom(void) +{ + + return (romfile); +} + static void lpc_uart_intr_assert(void *arg) { @@ -156,13 +171,19 @@ lpc_uart_io_handler(struct vmctx *ctx, int vcpu, int in, int port, int bytes, } static int -lpc_init(void) +lpc_init(struct vmctx *ctx) { struct lpc_uart_softc *sc; struct inout_port iop; const char *name; int unit, error; + if (romfile != NULL) { + error = bootrom_init(ctx, romfile); + if (error) + return (error); + } + /* COM1 and COM2 */ for (unit = 0; unit < LPC_UART_NUM; unit++) { sc = &lpc_uart_softc[unit]; @@ -379,7 +400,7 @@ pci_lpc_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) return (-1); } - if (lpc_init() != 0) + if (lpc_init(ctx) != 0) return (-1); /* initialize config space */ diff --git a/usr.sbin/bhyve/pci_lpc.h b/usr.sbin/bhyve/pci_lpc.h index 55a5865..431f5cf 100644 --- a/usr.sbin/bhyve/pci_lpc.h +++ b/usr.sbin/bhyve/pci_lpc.h @@ -68,5 +68,6 @@ struct lpc_sysres { int lpc_device_parse(const char *opt); char *lpc_pirq_name(int pin); void lpc_pirq_routed(void); +const char *lpc_bootrom(void); #endif diff --git a/usr.sbin/bhyve/pci_passthru.c b/usr.sbin/bhyve/pci_passthru.c index 04d68c4..5b52b05 100644 --- a/usr.sbin/bhyve/pci_passthru.c +++ b/usr.sbin/bhyve/pci_passthru.c @@ -548,12 +548,18 @@ done: static int passthru_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts) { - int bus, slot, func, error; + int bus, slot, func, error, memflags; struct passthru_softc *sc; sc = NULL; error = 1; + memflags = vm_get_memflags(ctx); + if (!(memflags & VM_MEM_F_WIRED)) { + fprintf(stderr, "passthru requires guest memory to be wired\n"); + goto done; + } + if (pcifd < 0) { pcifd = open(_PATH_DEVPCI, O_RDWR, 0); if (pcifd < 0) |