Merge remote-tracking branch 'origin/stable/10' into devel

21 files changed, 1482 insertions, 80 deletions

diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile
index 7d58988..c6d4b6e 100644
--- a/usr.sbin/Makefile
+++ b/usr.sbin/Makefile
@@ -20,6 +20,7 @@ SUBDIR=	adduser \
 	ctld \
 	daemon \
 	dconschat \
+	devctl \
 	devinfo \
 	digictl \
 	diskinfo \
diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile
index 377a2e6..33635f5 100644
--- a/usr.sbin/bhyve/Makefile
+++ b/usr.sbin/bhyve/Makefile
@@ -13,8 +13,10 @@ SRCS=	\
 	acpi.c			\
 	bhyverun.c		\
 	block_if.c		\
+	bootrom.c		\
 	consport.c		\
 	dbgport.c		\
+	fwctl.c			\
 	inout.c			\
 	ioapic.c		\
 	mem.c			\
diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8
index 37d1b86..6509fe7 100644
--- a/usr.sbin/bhyve/bhyve.8
+++ b/usr.sbin/bhyve/bhyve.8
@@ -32,7 +32,7 @@
 .Nd "run a guest operating system inside a virtual machine"
 .Sh SYNOPSIS
 .Nm
-.Op Fl abehuwxACHPWY
+.Op Fl abehuwxACHPSWY
 .Op Fl c Ar numcpus
 .Op Fl g Ar gdbport
 .Op Fl l Ar lpcdev Ns Op , Ns Ar conf
@@ -99,10 +99,12 @@ Yield the virtual CPU thread when a HLT instruction is detected.
 If this option is not specified, virtual CPUs will use 100% of a host CPU.
 .It Fl l Ar lpcdev Ns Op , Ns Ar conf
 Allow devices behind the LPC PCI-ISA bridge to be configured.
-The only supported devices are the TTY-class devices,
-.Li com1
+The only supported devices are the TTY-class devices
+.Ar com1
 and
-.Li com2 .
+.Ar com2
+and the boot ROM device
+.Ar bootrom .
 .It Fl m Ar size Ns Op Ar K|k|M|m|G|g|T|t
 Guest physical memory size in bytes.
 This must be the same size that was given to
@@ -173,8 +175,8 @@ AHCI controller attached to a SATA hard-drive.
 .It Li uart
 PCI 16550 serial device.
 .It Li lpc
-LPC PCI-ISA bridge with COM1 and COM2 16550 serial ports. The LPC bridge
-emulation can only be configured on bus 0.
+LPC PCI-ISA bridge with COM1 and COM2 16550 serial ports and a boot ROM.
+The LPC bridge emulation can only be configured on bus 0.
 .El
 .It Op Ar conf
 This optional parameter describes the backend for device emulations.
@@ -234,6 +236,14 @@ process.
 Use the host TTY device for serial port I/O.
 .El
 .Pp
+Boot ROM device:
+.Bl -tag -width 10n
+.It Pa romfile
+Map
+.Ar romfile
+in the guest address space reserved for boot firmware.
+.El
+.Pp
 Pass-through devices:
 .Bl -tag -width 10n
 .It Ns Ar slot Ns / Ns Ar bus Ns / Ns Ar function
@@ -245,11 +255,17 @@ and
 numbers.
 .El
 .Pp
+Guest memory must be wired using the
+.Fl S
+option when a pass-through device is configured.
+.Pp
 The host device must have been reserved at boot-time using the
 .Va pptdev
 loader variable as described in
 .Xr vmm 4 .
 .El
+.It Fl S
+Wire guest memory.
 .It Fl u
 RTC keeps UTC time.
 .It Fl U Ar uuid
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index d281dc5..e46b81c 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <pthread.h>
 #include <pthread_np.h>
 #include <sysexits.h>
+#include <stdbool.h>
 
 #include <machine/vmm.h>
 #include <vmmapi.h>
@@ -55,6 +56,7 @@ __FBSDID("$FreeBSD$");
 #include "acpi.h"
 #include "inout.h"
 #include "dbgport.h"
+#include "fwctl.h"
 #include "ioapic.h"
 #include "mem.h"
 #include "mevent.h"
@@ -122,7 +124,7 @@ usage(int code)
 {
 
         fprintf(stderr,
-                "Usage: %s [-abehuwxACHPWY] [-c vcpus] [-g <gdb port>] [-l <lpc>]\n"
+                "Usage: %s [-abehuwxACHPSWY] [-c vcpus] [-g <gdb port>] [-l <lpc>]\n"
 		"       %*s [-m mem] [-p vcpu:hostcpu] [-s <pci>] [-U uuid] <vm>\n"
 		"       -a: local apic is in xAPIC mode (deprecated)\n"
 		"       -A: create ACPI tables\n"
@@ -137,6 +139,7 @@ usage(int code)
 		"       -p: pin 'vcpu' to 'hostcpu'\n"
 		"       -P: vmexit from the guest on pause\n"
 		"       -s: <slot,driver,configinfo> PCI slot config\n"
+		"       -S: guest memory cannot be swapped\n"
 		"       -u: RTC keeps UTC time\n"
 		"       -U: uuid\n"
 		"       -w: ignore unimplemented MSRs\n"
@@ -698,26 +701,82 @@ fbsdrun_set_capabilities(struct vmctx *ctx, int cpu)
 	vm_set_capability(ctx, cpu, VM_CAP_ENABLE_INVPCID, 1);
 }
 
+static struct vmctx *
+do_open(const char *vmname)
+{
+	struct vmctx *ctx;
+	int error;
+	bool reinit, romboot;
+
+	reinit = romboot = false;
+
+	if (lpc_bootrom())
+		romboot = true;
+
+	error = vm_create(vmname);
+	if (error) {
+		if (errno == EEXIST) {
+			if (romboot) {
+				reinit = true;
+			} else {
+				/*
+				 * The virtual machine has been setup by the
+				 * userspace bootloader.
+				 */
+			}
+		} else {
+			perror("vm_create");
+			exit(1);
+		}
+	} else {
+		if (!romboot) {
+			/*
+			 * If the virtual machine was just created then a
+			 * bootrom must be configured to boot it.
+			 */
+			fprintf(stderr, "virtual machine cannot be booted\n");
+			exit(1);
+		}
+	}
+
+	ctx = vm_open(vmname);
+	if (ctx == NULL) {
+		perror("vm_open");
+		exit(1);
+	}
+
+	if (reinit) {
+		error = vm_reinit(ctx);
+		if (error) {
+			perror("vm_reinit");
+			exit(1);
+		}
+	}
+	return (ctx);
+}
+
 int
 main(int argc, char *argv[])
 {
 	int c, error, gdb_port, err, bvmcons;
-	int dump_guest_memory, max_vcpus, mptgen;
+	int max_vcpus, mptgen, memflags;
 	int rtc_localtime;
 	struct vmctx *ctx;
 	uint64_t rip;
 	size_t memsize;
+	char *optstr;
 
 	bvmcons = 0;
-	dump_guest_memory = 0;
 	progname = basename(argv[0]);
 	gdb_port = 0;
 	guest_ncpus = 1;
 	memsize = 256 * MB;
 	mptgen = 1;
 	rtc_localtime = 1;
+	memflags = 0;
 
-	while ((c = getopt(argc, argv, "abehuwxACHIPWYp:g:c:s:m:l:U:")) != -1) {
+	optstr = "abehuwxACHIPSWYp:g:c:s:m:l:U:";
+	while ((c = getopt(argc, argv, optstr)) != -1) {
 		switch (c) {
 		case 'a':
 			x2apic_mode = 0;
@@ -738,7 +797,7 @@ main(int argc, char *argv[])
 			guest_ncpus = atoi(optarg);
 			break;
 		case 'C':
-			dump_guest_memory = 1;
+			memflags |= VM_MEM_F_INCORE;
 			break;
 		case 'g':
 			gdb_port = atoi(optarg);
@@ -754,6 +813,9 @@ main(int argc, char *argv[])
 				exit(1);
 			else
 				break;
+		case 'S':
+			memflags |= VM_MEM_F_WIRED;
+			break;
                 case 'm':
 			error = vm_parse_memsize(optarg, &memsize);
 			if (error)
@@ -808,12 +870,7 @@ main(int argc, char *argv[])
 		usage(1);
 
 	vmname = argv[0];
-
-	ctx = vm_open(vmname);
-	if (ctx == NULL) {
-		perror("vm_open");
-		exit(1);
-	}
+	ctx = do_open(vmname);
 
 	if (guest_ncpus < 1) {
 		fprintf(stderr, "Invalid guest vCPUs (%d)\n", guest_ncpus);
@@ -829,11 +886,10 @@ main(int argc, char *argv[])
 
 	fbsdrun_set_capabilities(ctx, BSP);
 
-	if (dump_guest_memory)
-		vm_set_memflags(ctx, VM_MEM_F_INCORE);
+	vm_set_memflags(ctx, memflags);
 	err = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
 	if (err) {
-		fprintf(stderr, "Unable to setup memory (%d)\n", err);
+		fprintf(stderr, "Unable to setup memory (%d)\n", errno);
 		exit(1);
 	}
 
@@ -863,6 +919,16 @@ main(int argc, char *argv[])
 	if (bvmcons)
 		init_bvmcons();
 
+	if (lpc_bootrom()) {
+		if (vm_set_capability(ctx, BSP, VM_CAP_UNRESTRICTED_GUEST, 1)) {
+			fprintf(stderr, "ROM boot failed: unrestricted guest "
+			    "capability not available\n");
+			exit(1);
+		}
+		error = vcpu_reset(ctx, BSP);
+		assert(error == 0);
+	}
+
 	error = vm_get_register(ctx, BSP, VM_REG_GUEST_RIP, &rip);
 	assert(error == 0);
 
@@ -883,6 +949,9 @@ main(int argc, char *argv[])
 		assert(error == 0);
 	}
 
+	if (lpc_bootrom())
+		fwctl_init();
+
 	/*
 	 * Change the proc title to include the VM name.
 	 */
diff --git a/usr.sbin/bhyve/bootrom.c b/usr.sbin/bhyve/bootrom.c
new file mode 100644
index 0000000..5e4e0e9
--- /dev/null
+++ b/usr.sbin/bhyve/bootrom.c
@@ -0,0 +1,111 @@
+/*-
+ * Copyright (c) 2015 Neel Natu <neel@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+
+#include <machine/vmm.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdbool.h>
+
+#include <vmmapi.h>
+#include "bhyverun.h"
+#include "bootrom.h"
+
+#define	MAX_BOOTROM_SIZE	(16 * 1024 * 1024)	/* 16 MB */
+
+int
+bootrom_init(struct vmctx *ctx, const char *romfile)
+{
+	struct stat sbuf;
+	vm_paddr_t gpa;
+	ssize_t rlen;
+	char *ptr;
+	int fd, i, rv, prot;
+
+	rv = -1;
+	fd = open(romfile, O_RDONLY);
+	if (fd < 0) {
+		fprintf(stderr, "Error opening bootrom \"%s\": %s\n",
+		    romfile, strerror(errno));
+		goto done;
+	}
+
+        if (fstat(fd, &sbuf) < 0) {
+		fprintf(stderr, "Could not fstat bootrom file \"%s\": %s\n",
+		    romfile, strerror(errno));
+		goto done;
+        }
+
+	/*
+	 * Limit bootrom size to 16MB so it doesn't encroach into reserved
+	 * MMIO space (e.g. APIC, HPET, MSI).
+	 */
+	if (sbuf.st_size > MAX_BOOTROM_SIZE || sbuf.st_size < PAGE_SIZE) {
+		fprintf(stderr, "Invalid bootrom size %ld\n", sbuf.st_size);
+		goto done;
+	}
+
+	if (sbuf.st_size & PAGE_MASK) {
+		fprintf(stderr, "Bootrom size %ld is not a multiple of the "
+		    "page size\n", sbuf.st_size);
+		goto done;
+	}
+
+	ptr = vm_create_devmem(ctx, VM_BOOTROM, "bootrom", sbuf.st_size);
+	if (ptr == MAP_FAILED)
+		goto done;
+
+	/* Map the bootrom into the guest address space */
+	prot = PROT_READ | PROT_EXEC;
+	gpa = (1ULL << 32) - sbuf.st_size;
+	if (vm_mmap_memseg(ctx, gpa, VM_BOOTROM, 0, sbuf.st_size, prot) != 0)
+		goto done;
+
+	/* Read 'romfile' into the guest address space */
+	for (i = 0; i < sbuf.st_size / PAGE_SIZE; i++) {
+		rlen = read(fd, ptr + i * PAGE_SIZE, PAGE_SIZE);
+		if (rlen != PAGE_SIZE) {
+			fprintf(stderr, "Incomplete read of page %d of bootrom "
+			    "file %s: %ld bytes\n", i, romfile, rlen);
+			goto done;
+		}
+	}
+	rv = 0;
+done:
+	if (fd >= 0)
+		close(fd);
+	return (rv);
+}
diff --git a/usr.sbin/bhyve/bootrom.h b/usr.sbin/bhyve/bootrom.h
new file mode 100644
index 0000000..af150d3
--- /dev/null
+++ b/usr.sbin/bhyve/bootrom.h
@@ -0,0 +1,38 @@
+/*-
+ * Copyright (c) 2015 Neel Natu <neel@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef	_BOOTROM_H_
+#define	_BOOTROM_H_
+
+#include <stdbool.h>
+
+struct vmctx;
+
+int	bootrom_init(struct vmctx *ctx, const char *romfile);
+
+#endif
diff --git a/usr.sbin/bhyve/dbgport.c b/usr.sbin/bhyve/dbgport.c
index 534ae65..5be0ceb 100644
--- a/usr.sbin/bhyve/dbgport.c
+++ b/usr.sbin/bhyve/dbgport.c
@@ -116,6 +116,8 @@ SYSRES_IO(BVM_DBG_PORT, 4);
 void
 init_dbgport(int sport)
 {
+	int reuse;
+
 	conn_fd = -1;
 
 	if ((listen_fd = socket(AF_INET, SOCK_STREAM, 0)) < 0) {
@@ -128,6 +130,13 @@ init_dbgport(int sport)
 	sin.sin_addr.s_addr = htonl(INADDR_ANY);
 	sin.sin_port = htons(sport);
 
+	reuse = 1;
+	if (setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuse,
+	    sizeof(reuse)) < 0) {
+		perror("setsockopt");
+		exit(1);
+	}
+
 	if (bind(listen_fd, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
 		perror("bind");
 		exit(1);
diff --git a/usr.sbin/bhyve/fwctl.c b/usr.sbin/bhyve/fwctl.c
new file mode 100644
index 0000000..4b6164b
--- /dev/null
+++ b/usr.sbin/bhyve/fwctl.c
@@ -0,0 +1,549 @@
+/*-
+ * Copyright (c) 2015  Peter Grehan <grehan@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * Guest firmware interface. Uses i/o ports x510/x511 as Qemu does,
+ * but with a request/response messaging protocol.
+ */
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/errno.h>
+#include <sys/uio.h>
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "bhyverun.h"
+#include "inout.h"
+#include "fwctl.h"
+
+/*
+ * Messaging protocol base operations
+ */
+#define	OP_NULL		1
+#define	OP_ECHO		2
+#define	OP_GET		3
+#define	OP_GET_LEN	4
+#define	OP_SET		5
+#define	OP_MAX		OP_SET
+
+/* I/O ports */
+#define	FWCTL_OUT	0x510
+#define	FWCTL_IN	0x511
+
+/*
+ * Back-end state-machine
+ */
+enum state {
+	DORMANT,
+	IDENT_WAIT,
+	IDENT_SEND,
+	REQ,
+	RESP
+} be_state = DORMANT;
+
+static uint8_t sig[] = { 'B', 'H', 'Y', 'V' };
+static u_int ident_idx;
+
+struct op_info {
+	int op;
+	int  (*op_start)(int len);
+	void (*op_data)(uint32_t data, int len);
+	int  (*op_result)(struct iovec **data);
+	void (*op_done)(struct iovec *data);
+};
+static struct op_info *ops[OP_MAX+1];
+
+/* Return 0-padded uint32_t */
+static uint32_t
+fwctl_send_rest(uint32_t *data, size_t len)
+{
+	union {
+		uint8_t c[4];
+		uint32_t w;
+	} u;
+	uint8_t *cdata;
+	int i;
+
+	cdata = (uint8_t *) data;
+	u.w = 0;	
+
+	for (i = 0, u.w = 0; i < len; i++)
+		u.c[i] = *cdata++;
+
+	return (u.w);
+}
+
+/*
+ * error op dummy proto - drop all data sent and return an error
+*/
+static int errop_code;
+
+static void
+errop_set(int err)
+{
+
+	errop_code = err;
+}
+
+static int
+errop_start(int len)
+{
+	errop_code = ENOENT;
+
+	/* accept any length */
+	return (errop_code);
+}
+
+static void
+errop_data(uint32_t data, int len)
+{
+
+	/* ignore */
+}
+
+static int
+errop_result(struct iovec **data)
+{
+
+	/* no data to send back; always successful */
+	*data = NULL;
+	return (errop_code);
+}
+
+static void
+errop_done(struct iovec *data)
+{
+
+	/* assert data is NULL */
+}
+
+static struct op_info errop_info = {
+	.op_start  = errop_start,
+	.op_data   = errop_data,
+	.op_result = errop_result,
+	.op_done   = errop_done
+};
+
+/* OID search */
+SET_DECLARE(ctl_set, struct ctl);
+
+CTL_NODE("hw.ncpu", &guest_ncpus, sizeof(guest_ncpus));
+
+static struct ctl *
+ctl_locate(const char *str, int maxlen)
+{
+	struct ctl *cp, **cpp;
+
+	SET_FOREACH(cpp, ctl_set)  {
+		cp = *cpp;
+		if (!strncmp(str, cp->c_oid, maxlen))
+			return (cp);
+	}
+
+	return (NULL);
+}
+
+/* uefi-sysctl get-len */
+#define FGET_STRSZ	80
+static struct iovec fget_biov[2];
+static char fget_str[FGET_STRSZ];
+static struct {
+	size_t f_sz;
+	uint32_t f_data[1024];
+} fget_buf;
+static int fget_cnt;
+static size_t fget_size;
+
+static int
+fget_start(int len)
+{
+
+	if (len > FGET_STRSZ)
+		return(E2BIG);
+
+	fget_cnt = 0;
+
+	return (0);
+}
+
+static void
+fget_data(uint32_t data, int len)
+{
+
+	*((uint32_t *) &fget_str[fget_cnt]) = data;
+	fget_cnt += sizeof(uint32_t);
+}
+
+static int
+fget_result(struct iovec **data, int val)
+{
+	struct ctl *cp;
+	int err;
+
+	err = 0;
+
+	/* Locate the OID */
+	cp = ctl_locate(fget_str, fget_cnt);
+	if (cp == NULL) {
+		*data = NULL;
+		err = ENOENT;
+	} else {
+		if (val) {
+			/* For now, copy the len/data into a buffer */
+			memset(&fget_buf, 0, sizeof(fget_buf));
+			fget_buf.f_sz = cp->c_len;
+			memcpy(fget_buf.f_data, cp->c_data, cp->c_len);
+			fget_biov[0].iov_base = (char *)&fget_buf;
+			fget_biov[0].iov_len  = sizeof(fget_buf.f_sz) +
+				cp->c_len;
+		} else {
+			fget_size = cp->c_len;
+			fget_biov[0].iov_base = (char *)&fget_size;
+			fget_biov[0].iov_len  = sizeof(fget_size);
+		}
+
+		fget_biov[1].iov_base = NULL;
+		fget_biov[1].iov_len  = 0;
+		*data = fget_biov;
+	}
+
+	return (err);
+}
+
+static void
+fget_done(struct iovec *data)
+{
+
+	/* nothing needs to be freed */
+}
+
+static int
+fget_len_result(struct iovec **data)
+{
+	return (fget_result(data, 0));
+}
+
+static int
+fget_val_result(struct iovec **data)
+{
+	return (fget_result(data, 1));
+}
+
+static struct op_info fgetlen_info = {
+	.op_start  = fget_start,
+	.op_data   = fget_data,
+	.op_result = fget_len_result,
+	.op_done   = fget_done
+};
+
+static struct op_info fgetval_info = {
+	.op_start  = fget_start,
+	.op_data   = fget_data,
+	.op_result = fget_val_result,
+	.op_done   = fget_done
+};
+
+static struct req_info {
+	int      req_error;
+	u_int    req_count;
+	uint32_t req_size;
+	uint32_t req_type;
+	uint32_t req_txid;
+	struct op_info *req_op;
+	int	 resp_error;
+	int	 resp_count;
+	int	 resp_size;
+	int	 resp_off;
+	struct iovec *resp_biov;
+} rinfo;
+
+static void
+fwctl_response_done(void)
+{
+
+	(*rinfo.req_op->op_done)(rinfo.resp_biov);
+
+	/* reinit the req data struct */
+	memset(&rinfo, 0, sizeof(rinfo));
+}
+
+static void
+fwctl_request_done(void)
+{
+
+	rinfo.resp_error = (*rinfo.req_op->op_result)(&rinfo.resp_biov);
+
+	/* XXX only a single vector supported at the moment */
+	rinfo.resp_off = 0;
+	if (rinfo.resp_biov == NULL) {
+		rinfo.resp_size = 0;
+	} else {
+		rinfo.resp_size = rinfo.resp_biov[0].iov_len;
+	}
+}
+
+static int
+fwctl_request_start(void)
+{
+	int err;
+
+	/* Data size doesn't include header */
+	rinfo.req_size -= 12;
+
+	rinfo.req_op = &errop_info;
+	if (rinfo.req_type <= OP_MAX && ops[rinfo.req_type] != NULL)
+		rinfo.req_op = ops[rinfo.req_type];
+
+	err = (*rinfo.req_op->op_start)(rinfo.req_size);
+
+	if (err) {
+		errop_set(err);
+		rinfo.req_op = &errop_info;
+	}
+
+	/* Catch case of zero-length message here */
+	if (rinfo.req_size == 0) {
+		fwctl_request_done();
+		return (1);
+	}
+
+	return (0);
+}
+
+static int
+fwctl_request_data(uint32_t value)
+{
+	int remlen;
+
+	/* Make sure remaining size is >= 0 */
+	rinfo.req_size -= sizeof(uint32_t);
+	remlen = (rinfo.req_size > 0) ? rinfo.req_size: 0;
+
+	(*rinfo.req_op->op_data)(value, remlen);
+
+	if (rinfo.req_size < sizeof(uint32_t)) {
+		fwctl_request_done();
+		return (1);
+	}
+
+	return (0);
+}
+
+static int
+fwctl_request(uint32_t value)
+{
+
+	int ret;
+
+	ret = 0;
+
+	switch (rinfo.req_count) {
+	case 0:
+		/* Verify size */
+		if (value < 12) {
+			printf("msg size error");
+			exit(1);
+		}
+		rinfo.req_size = value;
+		rinfo.req_count = 1;
+		break;
+	case 1:
+		rinfo.req_type = value;
+		rinfo.req_count++;
+		break;
+	case 2:
+		rinfo.req_txid = value;
+		rinfo.req_count++;
+		ret = fwctl_request_start();
+		break;
+	default:
+		ret = fwctl_request_data(value);
+		break;
+	}
+
+	return (ret);
+}
+
+static int
+fwctl_response(uint32_t *retval)
+{
+	uint32_t *dp;
+	int remlen;
+
+	switch(rinfo.resp_count) {
+	case 0:
+		/* 4 x u32 header len + data */
+		*retval = 4*sizeof(uint32_t) +
+		    roundup(rinfo.resp_size, sizeof(uint32_t));
+		rinfo.resp_count++;
+		break;
+	case 1:
+		*retval = rinfo.req_type;
+		rinfo.resp_count++;
+		break;
+	case 2:
+		*retval = rinfo.req_txid;
+		rinfo.resp_count++;
+		break;
+	case 3:
+		*retval = rinfo.resp_error;
+		rinfo.resp_count++;
+		break;
+	default:
+		remlen = rinfo.resp_size - rinfo.resp_off;
+		dp = (uint32_t *)
+		    ((uint8_t *)rinfo.resp_biov->iov_base + rinfo.resp_off);
+		if (remlen >= sizeof(uint32_t)) {
+			*retval = *dp;
+		} else if (remlen > 0) {
+			*retval = fwctl_send_rest(dp, remlen);
+		}
+		rinfo.resp_off += sizeof(uint32_t);
+		break;
+	}
+
+	if (rinfo.resp_count > 3 &&
+	    rinfo.resp_size - rinfo.resp_off <= 0) {
+		fwctl_response_done();
+		return (1);
+	}
+
+	return (0);
+}
+
+
+/*
+ * i/o port handling.
+ */
+static uint8_t
+fwctl_inb(void)
+{
+	uint8_t retval;
+
+	retval = 0xff;
+
+	switch (be_state) {
+	case IDENT_SEND:
+		retval = sig[ident_idx++];
+		if (ident_idx >= sizeof(sig))
+			be_state = REQ;
+		break;
+	default:
+		break;
+	}
+
+	return (retval);
+}
+
+static void
+fwctl_outw(uint16_t val)
+{
+	switch (be_state) {
+	case IDENT_WAIT:
+		if (val == 0) {
+			be_state = IDENT_SEND;
+			ident_idx = 0;
+		}
+		break;
+	default:
+		/* ignore */
+		break;
+	}
+}
+
+static uint32_t
+fwctl_inl(void)
+{
+	uint32_t retval;
+
+	switch (be_state) {
+	case RESP:
+		if (fwctl_response(&retval))
+			be_state = REQ;
+		break;
+	default:
+		retval = 0xffffffff;
+		break;
+	}
+
+	return (retval);
+}
+
+static void
+fwctl_outl(uint32_t val)
+{
+
+	switch (be_state) {
+	case REQ:
+		if (fwctl_request(val))
+			be_state = RESP;
+	default:
+		break;
+	}
+
+}
+
+static int
+fwctl_handler(struct vmctx *ctx, int vcpu, int in, int port, int bytes,
+    uint32_t *eax, void *arg)
+{
+
+	if (in) {
+		if (bytes == 1)
+			*eax = fwctl_inb();
+		else if (bytes == 4)
+			*eax = fwctl_inl();
+		else
+			*eax = 0xffff;
+	} else {
+		if (bytes == 2)
+			fwctl_outw(*eax);
+		else if (bytes == 4)
+			fwctl_outl(*eax);
+	}
+
+	return (0);
+}
+INOUT_PORT(fwctl_wreg, FWCTL_OUT, IOPORT_F_INOUT, fwctl_handler);
+INOUT_PORT(fwctl_rreg, FWCTL_IN,  IOPORT_F_IN,    fwctl_handler);
+
+void
+fwctl_init(void)
+{
+
+	ops[OP_GET_LEN] = &fgetlen_info;
+	ops[OP_GET]     = &fgetval_info;
+
+	be_state = IDENT_WAIT;
+}
diff --git a/usr.sbin/bhyve/fwctl.h b/usr.sbin/bhyve/fwctl.h
new file mode 100644
index 0000000..f5f8d13
--- /dev/null
+++ b/usr.sbin/bhyve/fwctl.h
@@ -0,0 +1,54 @@
+/*-
+ * Copyright (c) 2015  Peter Grehan <grehan@freebsd.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _FWCTL_H_
+#define _FWCTL_H_
+
+#include <sys/linker_set.h>
+
+/*
+ * Linker set api for export of information to guest firmware via
+ * a sysctl-like OID interface
+ */
+struct ctl {
+	const char *c_oid;
+	const void *c_data;
+	const int c_len;
+};
+
+#define CTL_NODE(oid, data, len)				\
+	static struct ctl __CONCAT(__ctl, __LINE__) = {		\
+		oid,						\
+		(data),						\
+		(len),						\
+	};							\
+	DATA_SET(ctl_set, __CONCAT(__ctl, __LINE__))
+
+void	fwctl_init(void);
+
+#endif /* _FWCTL_H_ */
diff --git a/usr.sbin/bhyve/pci_ahci.c b/usr.sbin/bhyve/pci_ahci.c
index 35a0859..5c4743c 100644
--- a/usr.sbin/bhyve/pci_ahci.c
+++ b/usr.sbin/bhyve/pci_ahci.c
@@ -926,7 +926,7 @@ handle_identify(struct ahci_port *p, int slot, uint8_t *cfis)
 		ata_string((uint8_t *)(buf+23), "001", 8);
 		ata_string((uint8_t *)(buf+27), "BHYVE SATA DISK", 40);
 		buf[47] = (0x8000 | 128);
-		buf[48] = 0x1;
+		buf[48] = 0;
 		buf[49] = (1 << 8 | 1 << 9 | 1 << 11);
 		buf[50] = (1 << 14);
 		buf[53] = (1 << 1 | 1 << 2);
@@ -1683,15 +1683,23 @@ ahci_handle_cmd(struct ahci_port *p, int slot, uint8_t *cfis)
 	case ATA_READ_LOG_DMA_EXT:
 		ahci_handle_read_log(p, slot, cfis);
 		break;
+	case ATA_SECURITY_FREEZE_LOCK:
+	case ATA_SMART_CMD:
 	case ATA_NOP:
 		ahci_write_fis_d2h(p, slot, cfis,
 		    (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR);
 		break;
+	case ATA_CHECK_POWER_MODE:
+		cfis[12] = 0xff;	/* always on */
+		ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC);
+		break;
 	case ATA_STANDBY_CMD:
 	case ATA_STANDBY_IMMEDIATE:
 	case ATA_IDLE_CMD:
 	case ATA_IDLE_IMMEDIATE:
 	case ATA_SLEEP:
+	case ATA_READ_VERIFY:
+	case ATA_READ_VERIFY48:
 		ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC);
 		break;
 	case ATA_ATAPI_IDENTIFY:
diff --git a/usr.sbin/bhyve/pci_lpc.c b/usr.sbin/bhyve/pci_lpc.c
index e98b141..2203a00 100644
--- a/usr.sbin/bhyve/pci_lpc.c
+++ b/usr.sbin/bhyve/pci_lpc.c
@@ -40,6 +40,7 @@ __FBSDID("$FreeBSD$");
 #include <vmmapi.h>
 
 #include "acpi.h"
+#include "bootrom.h"
 #include "inout.h"
 #include "pci_emul.h"
 #include "pci_irq.h"
@@ -62,6 +63,8 @@ SYSRES_IO(NMISC_PORT, 1);
 
 static struct pci_devinst *lpc_bridge;
 
+static const char *romfile;
+
 #define	LPC_UART_NUM	2
 static struct lpc_uart_softc {
 	struct uart_softc *uart_softc;
@@ -76,7 +79,7 @@ static const char *lpc_uart_names[LPC_UART_NUM] = { "COM1", "COM2" };
 /*
  * LPC device configuration is in the following form:
  * <lpc_device_name>[,<options>]
- * For e.g. "com1,stdio"
+ * For e.g. "com1,stdio" or "bootrom,/var/romfile"
  */
 int
 lpc_device_parse(const char *opts)
@@ -88,6 +91,11 @@ lpc_device_parse(const char *opts)
 	str = cpy = strdup(opts);
 	lpcdev = strsep(&str, ",");
 	if (lpcdev != NULL) {
+		if (strcasecmp(lpcdev, "bootrom") == 0) {
+			romfile = str;
+			error = 0;
+			goto done;
+		}
 		for (unit = 0; unit < LPC_UART_NUM; unit++) {
 			if (strcasecmp(lpcdev, lpc_uart_names[unit]) == 0) {
 				lpc_uart_softc[unit].opts = str;
@@ -104,6 +112,13 @@ done:
 	return (error);
 }
 
+const char *
+lpc_bootrom(void)
+{
+
+	return (romfile);
+}
+
 static void
 lpc_uart_intr_assert(void *arg)
 {
@@ -156,13 +171,19 @@ lpc_uart_io_handler(struct vmctx *ctx, int vcpu, int in, int port, int bytes,
 }
 
 static int
-lpc_init(void)
+lpc_init(struct vmctx *ctx)
 {
 	struct lpc_uart_softc *sc;
 	struct inout_port iop;
 	const char *name;
 	int unit, error;
 
+	if (romfile != NULL) {
+		error = bootrom_init(ctx, romfile);
+		if (error)
+			return (error);
+	}
+
 	/* COM1 and COM2 */
 	for (unit = 0; unit < LPC_UART_NUM; unit++) {
 		sc = &lpc_uart_softc[unit];
@@ -379,7 +400,7 @@ pci_lpc_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 		return (-1);
 	}
 
-	if (lpc_init() != 0)
+	if (lpc_init(ctx) != 0)
 		return (-1);
 
 	/* initialize config space */
diff --git a/usr.sbin/bhyve/pci_lpc.h b/usr.sbin/bhyve/pci_lpc.h
index 55a5865..431f5cf 100644
--- a/usr.sbin/bhyve/pci_lpc.h
+++ b/usr.sbin/bhyve/pci_lpc.h
@@ -68,5 +68,6 @@ struct lpc_sysres {
 int	lpc_device_parse(const char *opt);
 char	*lpc_pirq_name(int pin);
 void	lpc_pirq_routed(void);
+const char *lpc_bootrom(void);
 
 #endif
diff --git a/usr.sbin/bhyve/pci_passthru.c b/usr.sbin/bhyve/pci_passthru.c
index 04d68c4..5b52b05 100644
--- a/usr.sbin/bhyve/pci_passthru.c
+++ b/usr.sbin/bhyve/pci_passthru.c
@@ -548,12 +548,18 @@ done:
 static int
 passthru_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 {
-	int bus, slot, func, error;
+	int bus, slot, func, error, memflags;
 	struct passthru_softc *sc;
 
 	sc = NULL;
 	error = 1;
 
+	memflags = vm_get_memflags(ctx);
+	if (!(memflags & VM_MEM_F_WIRED)) {
+		fprintf(stderr, "passthru requires guest memory to be wired\n");
+		goto done;
+	}
+
 	if (pcifd < 0) {
 		pcifd = open(_PATH_DEVPCI, O_RDWR, 0);
 		if (pcifd < 0)
diff --git a/usr.sbin/bhyve/pci_virtio_net.c b/usr.sbin/bhyve/pci_virtio_net.c
index c13940b..18ff908 100644
--- a/usr.sbin/bhyve/pci_virtio_net.c
+++ b/usr.sbin/bhyve/pci_virtio_net.c
@@ -61,7 +61,7 @@ __FBSDID("$FreeBSD$");
 
 #define VTNET_RINGSZ	1024
 
-#define VTNET_MAXSEGS	32
+#define VTNET_MAXSEGS	256
 
 /*
  * Host capabilities.  Note that we only offer a few of these.
@@ -88,7 +88,7 @@ __FBSDID("$FreeBSD$");
 
 #define VTNET_S_HOSTCAPS      \
   ( VIRTIO_NET_F_MAC | VIRTIO_NET_F_MRG_RXBUF | VIRTIO_NET_F_STATUS | \
-    VIRTIO_F_NOTIFY_ON_EMPTY)
+    VIRTIO_F_NOTIFY_ON_EMPTY | VIRTIO_RING_F_INDIRECT_DESC)
 
 /*
  * PCI config-space "registers"
diff --git a/usr.sbin/bhyve/uart_emul.c b/usr.sbin/bhyve/uart_emul.c
index 4242e5c..538bf58 100644
--- a/usr.sbin/bhyve/uart_emul.c
+++ b/usr.sbin/bhyve/uart_emul.c
@@ -272,6 +272,37 @@ uart_opentty(struct uart_softc *sc)
 	assert(sc->mev != NULL);
 }
 
+static uint8_t
+modem_status(uint8_t mcr)
+{
+	uint8_t msr;
+
+	if (mcr & MCR_LOOPBACK) {
+		/*
+		 * In the loopback mode certain bits from the MCR are
+		 * reflected back into MSR.
+		 */
+		msr = 0;
+		if (mcr & MCR_RTS)
+			msr |= MSR_CTS;
+		if (mcr & MCR_DTR)
+			msr |= MSR_DSR;
+		if (mcr & MCR_OUT1)
+			msr |= MSR_RI;
+		if (mcr & MCR_OUT2)
+			msr |= MSR_DCD;
+	} else {
+		/*
+		 * Always assert DCD and DSR so tty open doesn't block
+		 * even if CLOCAL is turned off.
+		 */
+		msr = MSR_DCD | MSR_DSR;
+	}
+	assert((msr & MSR_DELTA_MASK) == 0);
+
+	return (msr);
+}
+
 /*
  * The IIR returns a prioritized interrupt reason:
  * - receive data available
@@ -304,6 +335,7 @@ uart_reset(struct uart_softc *sc)
 	divisor = DEFAULT_RCLK / DEFAULT_BAUD / 16;
 	sc->dll = divisor;
 	sc->dlh = divisor >> 16;
+	sc->msr = modem_status(sc->mcr);
 
 	rxfifo_reset(sc, 1);	/* no fifo until enabled by software */
 }
@@ -363,7 +395,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 	uint8_t msr;
 
 	pthread_mutex_lock(&sc->mtx);
-	
+
 	/*
 	 * Take care of the special case DLAB accesses first
 	 */
@@ -426,22 +458,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 		case REG_MCR:
 			/* Apply mask so that bits 5-7 are 0 */
 			sc->mcr = value & 0x1F;
-
-			msr = 0;
-			if (sc->mcr & MCR_LOOPBACK) {
-				/*
-				 * In the loopback mode certain bits from the
-				 * MCR are reflected back into MSR
-				 */
-				if (sc->mcr & MCR_RTS)
-					msr |= MSR_CTS;
-				if (sc->mcr & MCR_DTR)
-					msr |= MSR_DSR;
-				if (sc->mcr & MCR_OUT1)
-					msr |= MSR_RI;
-				if (sc->mcr & MCR_OUT2)
-					msr |= MSR_DCD;
-			}
+			msr = modem_status(sc->mcr);
 
 			/*
 			 * Detect if there has been any change between the
diff --git a/usr.sbin/bhyvectl/bhyvectl.c b/usr.sbin/bhyvectl/bhyvectl.c
index 7d3017f..51bade1 100644
--- a/usr.sbin/bhyvectl/bhyvectl.c
+++ b/usr.sbin/bhyvectl/bhyvectl.c
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/errno.h>
 #include <sys/mman.h>
+#include <sys/cpuset.h>
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -47,10 +48,12 @@ __FBSDID("$FreeBSD$");
 #include <getopt.h>
 #include <time.h>
 #include <assert.h>
+#include <libutil.h>
 
 #include <machine/cpufunc.h>
-#include <machine/vmm.h>
 #include <machine/specialreg.h>
+#include <machine/vmm.h>
+#include <machine/vmm_dev.h>
 #include <vmmapi.h>
 
 #include "amd/vmcb.h"
@@ -236,7 +239,7 @@ static int get_stats, getcap, setcap, capval, get_gpa_pmap;
 static int inject_nmi, assert_lapic_lvt;
 static int force_reset, force_poweroff;
 static const char *capname;
-static int create, destroy, get_lowmem, get_highmem;
+static int create, destroy, get_memmap, get_memseg;
 static int get_intinfo;
 static int get_active_cpus, get_suspended_cpus;
 static uint64_t memsize;
@@ -1320,8 +1323,8 @@ setup_options(bool cpu_intel)
 		{ "get-desc-gdtr", NO_ARG,	&get_desc_gdtr, 1 },
 		{ "set-desc-idtr", NO_ARG,	&set_desc_idtr, 1 },
 		{ "get-desc-idtr", NO_ARG,	&get_desc_idtr, 1 },
-		{ "get-lowmem", NO_ARG,		&get_lowmem,	1 },
-		{ "get-highmem",NO_ARG,		&get_highmem,	1 },
+		{ "get-memmap",	NO_ARG,		&get_memmap,	1 },
+		{ "get-memseg", NO_ARG,		&get_memseg,	1 },
 		{ "get-efer",	NO_ARG,		&get_efer,	1 },
 		{ "get-cr0",	NO_ARG,		&get_cr0,	1 },
 		{ "get-cr3",	NO_ARG,		&get_cr3,	1 },
@@ -1520,18 +1523,92 @@ mon_str(int idx)
 		return ("UNK");
 }
 
+static int
+show_memmap(struct vmctx *ctx)
+{
+	char name[SPECNAMELEN + 1], numbuf[8];
+	vm_ooffset_t segoff;
+	vm_paddr_t gpa;
+	size_t maplen, seglen;
+	int error, flags, prot, segid, delim;
+
+	printf("Address     Length      Segment     Offset      ");
+	printf("Prot  Flags\n");
+
+	gpa = 0;
+	while (1) {
+		error = vm_mmap_getnext(ctx, &gpa, &segid, &segoff, &maplen,
+		    &prot, &flags);
+		if (error)
+			return (errno == ENOENT ? 0 : error);
+
+		error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
+		if (error)
+			return (error);
+
+		printf("%-12lX", gpa);
+		humanize_number(numbuf, sizeof(numbuf), maplen, "B",
+		    HN_AUTOSCALE, HN_NOSPACE);
+		printf("%-12s", numbuf);
+
+		printf("%-12s", name[0] ? name : "sysmem");
+		printf("%-12lX", segoff);
+		printf("%c%c%c   ", prot & PROT_READ ? 'R' : '-',
+		    prot & PROT_WRITE ? 'W' : '-',
+		    prot & PROT_EXEC ? 'X' : '-');
+
+		delim = '\0';
+		if (flags & VM_MEMMAP_F_WIRED) {
+			printf("%cwired", delim);
+			delim = '/';
+		}
+		if (flags & VM_MEMMAP_F_IOMMU) {
+			printf("%ciommu", delim);
+			delim = '/';
+		}
+		printf("\n");
+
+		gpa += maplen;
+	}
+}
+
+static int
+show_memseg(struct vmctx *ctx)
+{
+	char name[SPECNAMELEN + 1], numbuf[8];
+	size_t seglen;
+	int error, segid;
+
+	printf("ID  Length      Name\n");
+
+	segid = 0;
+	while (1) {
+		error = vm_get_memseg(ctx, segid, &seglen, name, sizeof(name));
+		if (error)
+			return (errno == EINVAL ? 0 : error);
+
+		if (seglen) {
+			printf("%-4d", segid);
+			humanize_number(numbuf, sizeof(numbuf), seglen, "B",
+			    HN_AUTOSCALE, HN_NOSPACE);
+			printf("%-12s", numbuf);
+			printf("%s", name[0] ? name : "sysmem");
+			printf("\n");
+		}
+		segid++;
+	}
+}
+
 int
 main(int argc, char *argv[])
 {
 	char *vmname;
 	int error, ch, vcpu, ptenum;
-	vm_paddr_t gpa, gpa_pmap;
-	size_t len;
+	vm_paddr_t gpa_pmap;
 	struct vm_exit vmexit;
 	uint64_t rax, cr0, cr3, cr4, dr7, rsp, rip, rflags, efer, pat;
 	uint64_t eptp, bm, addr, u64, pteval[4], *pte, info[2];
 	struct vmctx *ctx;
-	int wired;
 	cpuset_t cpus;
 	bool cpu_intel;
 	uint64_t cs, ds, es, fs, gs, ss, tr, ldtr;
@@ -1703,7 +1780,7 @@ main(int argc, char *argv[])
 	}
 
 	if (!error && memsize)
-		error = vm_setup_memory(ctx, memsize, VM_MMAP_NONE);
+		error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
 
 	if (!error && set_efer)
 		error = vm_set_register(ctx, vcpu, VM_REG_GUEST_EFER, efer);
@@ -1838,21 +1915,11 @@ main(int argc, char *argv[])
 		error = vm_lapic_local_irq(ctx, vcpu, assert_lapic_lvt);
 	}
 
-	if (!error && (get_lowmem || get_all)) {
-		gpa = 0;
-		error = vm_get_memory_seg(ctx, gpa, &len, &wired);
-		if (error == 0)
-			printf("lowmem\t\t0x%016lx/%ld%s\n", gpa, len,
-			    wired ? " wired" : "");
-	}
+	if (!error && (get_memseg || get_all))
+		error = show_memseg(ctx);
 
-	if (!error && (get_highmem || get_all)) {
-		gpa = 4 * GB;
-		error = vm_get_memory_seg(ctx, gpa, &len, &wired);
-		if (error == 0)
-			printf("highmem\t\t0x%016lx/%ld%s\n", gpa, len,
-			    wired ? " wired" : "");
-	}
+	if (!error && (get_memmap || get_all))
+		error = show_memmap(ctx);
 
 	if (!error)
 		error = get_all_registers(ctx, vcpu);
diff --git a/usr.sbin/bhyveload/bhyveload.8 b/usr.sbin/bhyveload/bhyveload.8
index 0bdf151..62b6b27 100644
--- a/usr.sbin/bhyveload/bhyveload.8
+++ b/usr.sbin/bhyveload/bhyveload.8
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 7, 2012
+.Dd October 7, 2015
 .Dt BHYVELOAD 8
 .Os
 .Sh NAME
@@ -35,10 +35,12 @@
 guest inside a bhyve virtual machine
 .Sh SYNOPSIS
 .Nm
+.Op Fl S
 .Op Fl c Ar cons-dev
 .Op Fl d Ar disk-path
 .Op Fl e Ar name=value
 .Op Fl h Ar host-path
+.Op Fl l Ar os-loader
 .Op Fl m Ar mem-size
 .Ar vmname
 .Sh DESCRIPTION
@@ -55,6 +57,7 @@ is based on
 and will present an interface identical to the
 .Fx
 loader on the user's terminal.
+This behavior can be changed by specifying a different OS loader.
 .Pp
 The virtual machine is identified as
 .Ar vmname
@@ -77,7 +80,9 @@ The
 .Ar disk-path
 is the pathname of the guest's boot disk image.
 .It Fl e Ar name=value
-Set the FreeBSD loader environment variable
+Set the
+.Fx
+loader environment variable
 .Ar name
 to
 .Ar value .
@@ -88,6 +93,15 @@ variable.
 The
 .Ar host-path
 is the directory at the top of the guest's boot filesystem.
+.It Fl l Ar os-loader
+Specify a different OS loader.
+By default
+.Nm
+will use
+.Pa /boot/userboot.so ,
+which presents a standard
+.Fx
+loader.
 .It Fl m Ar mem-size Xo
 .Sm off
 .Op Cm K | k | M | m | G | g | T | t
@@ -111,8 +125,10 @@ respectively.
 The default value of
 .Ar mem-size
 is 256M.
-.El
+.It Fl S
+Wire guest memory.
 .Sh EXAMPLES
+.El
 To create a virtual machine named
 .Ar freebsd-vm
 that boots off the ISO image
diff --git a/usr.sbin/bhyveload/bhyveload.c b/usr.sbin/bhyveload/bhyveload.c
index 8ebf116..badf5f4 100644
--- a/usr.sbin/bhyveload/bhyveload.c
+++ b/usr.sbin/bhyveload/bhyveload.c
@@ -629,7 +629,7 @@ usage(void)
 {
 
 	fprintf(stderr,
-	    "usage: %s [-c <console-device>] [-d <disk-path>] [-e <name=value>]\n"
+	    "usage: %s [-S][-c <console-device>] [-d <disk-path>] [-e <name=value>]\n"
 	    "       %*s [-h <host-path>] [-m mem-size] <vmname>\n",
 	    progname,
 	    (int)strlen(progname), "");
@@ -639,19 +639,23 @@ usage(void)
 int
 main(int argc, char** argv)
 {
+	char *loader;
 	void *h;
 	void (*func)(struct loader_callbacks *, void *, int, int);
 	uint64_t mem_size;
-	int opt, error, need_reinit;
+	int opt, error, need_reinit, memflags;
 
 	progname = basename(argv[0]);
 
+	loader = NULL;
+
+	memflags = 0;
 	mem_size = 256 * MB;
 
 	consin_fd = STDIN_FILENO;
 	consout_fd = STDOUT_FILENO;
 
-	while ((opt = getopt(argc, argv, "c:d:e:h:m:")) != -1) {
+	while ((opt = getopt(argc, argv, "Sc:d:e:h:l:m:")) != -1) {
 		switch (opt) {
 		case 'c':
 			error = altcons_open(optarg);
@@ -673,11 +677,22 @@ main(int argc, char** argv)
 			host_base = optarg;
 			break;
 
+		case 'l':
+			if (loader != NULL)
+				errx(EX_USAGE, "-l can only be given once");
+			loader = strdup(optarg);
+			if (loader == NULL)
+				err(EX_OSERR, "malloc");
+			break;
+
 		case 'm':
 			error = vm_parse_memsize(optarg, &mem_size);
 			if (error != 0)
 				errx(EX_USAGE, "Invalid memsize '%s'", optarg);
 			break;
+		case 'S':
+			memflags |= VM_MEM_F_WIRED;
+			break;
 		case '?':
 			usage();
 		}
@@ -715,32 +730,43 @@ main(int argc, char** argv)
 		}
 	}
 
+	vm_set_memflags(ctx, memflags);
 	error = vm_setup_memory(ctx, mem_size, VM_MMAP_ALL);
 	if (error) {
 		perror("vm_setup_memory");
 		exit(1);
 	}
 
-	tcgetattr(consout_fd, &term);
-	oldterm = term;
-	cfmakeraw(&term);
-	term.c_cflag |= CLOCAL;
-	
-	tcsetattr(consout_fd, TCSAFLUSH, &term);
-
-	h = dlopen("/boot/userboot.so", RTLD_LOCAL);
+	if (loader == NULL) {
+		loader = strdup("/boot/userboot.so");
+		if (loader == NULL)
+			err(EX_OSERR, "malloc");
+	}
+	h = dlopen(loader, RTLD_LOCAL);
 	if (!h) {
 		printf("%s\n", dlerror());
+		free(loader);
 		return (1);
 	}
 	func = dlsym(h, "loader_main");
 	if (!func) {
 		printf("%s\n", dlerror());
+		free(loader);
 		return (1);
 	}
 
+	tcgetattr(consout_fd, &term);
+	oldterm = term;
+	cfmakeraw(&term);
+	term.c_cflag |= CLOCAL;
+
+	tcsetattr(consout_fd, TCSAFLUSH, &term);
+
 	addenv("smbios.bios.vendor=BHYVE");
 	addenv("boot_serial=1");
 
 	func(&cb, NULL, USERBOOT_VERSION_3, ndisks);
+
+	free(loader);
+	return (0);
 }
diff --git a/usr.sbin/devctl/Makefile b/usr.sbin/devctl/Makefile
new file mode 100644
index 0000000..6193100
--- /dev/null
+++ b/usr.sbin/devctl/Makefile
@@ -0,0 +1,9 @@
+# $FreeBSD$
+
+PROG=	devctl
+MAN=	devctl.8
+
+DPADD=	${LIBDEVCTL}
+LDADD=	-ldevctl
+
+.include <bsd.prog.mk>
diff --git a/usr.sbin/devctl/devctl.8 b/usr.sbin/devctl/devctl.8
new file mode 100644
index 0000000..add170d
--- /dev/null
+++ b/usr.sbin/devctl/devctl.8
@@ -0,0 +1,126 @@
+.\"
+.\" Copyright (c) 2015 John Baldwin <jhb@FreeBSD.org>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd February 5, 2015
+.Dt DEVCTL 8
+.Os
+.Sh NAME
+.Nm devctl
+.Nd device control utility
+.Sh SYNOPSIS
+.Nm
+.Cm attach
+.Ar device
+.Nm
+.Cm detach
+.Op Fl f
+.Ar device
+.Nm
+.Cm disable
+.Op Fl f
+.Ar device
+.Nm
+.Cm enable
+.Ar device
+.Nm
+.Cm set driver
+.Op Fl f
+.Ar device driver
+.Sh DESCRIPTION
+The
+.Nm
+utility adjusts the state of individual devices in the kernel's
+internal device hierarchy.
+Each invocation of
+.Nm
+consists of a single command followed by command-specific arguments.
+Each command operates on a single device specified via the
+.Ar device
+argument.
+The
+.Ar device
+may be specified either as the name of an existing device or as a
+bus-specific address.
+More details on supported address formats can be found in
+.Xr devctl 3 .
+.Pp
+The following commands are supported:
+.Bl -tag -width indent
+.It Cm attach Ar device
+Force the kernel to re-probe the device.
+If a suitable driver is found,
+it is attached to the device.
+.It Xo Cm detach
+.Op Fl f
+.Ar device
+.Xc
+Detach the device from its current device driver.
+If the
+.Fl f
+flag is specified,
+the device driver will be detached even if the device is busy.
+.It Xo Cm disable
+.Op Fl f
+.Ar device
+.Xc
+Disable a device.
+If the device is currently attached to a device driver,
+the device driver will be detached from the device,
+but the device will retain its current name.
+If the
+.Fl f
+flag is specified,
+the device driver will be detached even if the device is busy.
+.It Cm enable Ar device
+Enable a device.
+The device will probe and attach if a suitable device driver is found.
+Note that this can re-enable a device disabled at boot time via a
+loader tunable.
+.It Xo Cm set driver
+.Op Fl f
+.Ar device driver
+.Xc
+Force the device to use a device driver named
+.Ar driver .
+If the device is already attached to a device driver and the
+.Fl f
+flag is specified,
+the device will be detached from its current device driver before it is
+attached to the new device driver.
+If the device is already attached to a device driver and the
+.Fl f
+flag is not specified,
+the device will not be changed.
+.El
+.Sh SEE ALSO
+.Xr devctl 3 ,
+.Xr devinfo 8
+.Sh HISTORY
+The
+.Nm
+utility first appeared in
+.Fx 11.0 .
diff --git a/usr.sbin/devctl/devctl.c b/usr.sbin/devctl/devctl.c
new file mode 100644
index 0000000..fdabd4f
--- /dev/null
+++ b/usr.sbin/devctl/devctl.c
@@ -0,0 +1,256 @@
+/*-
+ * Copyright (c) 2014 John Baldwin <jhb@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/linker_set.h>
+#include <devctl.h>
+#include <err.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <unistd.h>
+
+struct devctl_command {
+	const char *name;
+	int (*handler)(int ac, char **av);
+};
+
+#define	DEVCTL_DATASET(name)	devctl_ ## name ## _table
+
+#define	DEVCTL_COMMAND(set, name, function)				\
+	static struct devctl_command function ## _devctl_command =	\
+	{ #name, function };						\
+	DATA_SET(DEVCTL_DATASET(set), function ## _devctl_command)
+
+#define	DEVCTL_TABLE(set, name)						\
+	SET_DECLARE(DEVCTL_DATASET(name), struct devctl_command);	\
+									\
+	static int							\
+	devctl_ ## name ## _table_handler(int ac, char **av)		\
+	{								\
+		return (devctl_table_handler(SET_BEGIN(DEVCTL_DATASET(name)), \
+		    SET_LIMIT(DEVCTL_DATASET(name)), ac, av));		\
+	}								\
+	DEVCTL_COMMAND(set, name, devctl_ ## name ## _table_handler)
+
+static int	devctl_table_handler(struct devctl_command **start,
+    struct devctl_command **end, int ac, char **av);
+
+SET_DECLARE(DEVCTL_DATASET(top), struct devctl_command);
+
+DEVCTL_TABLE(top, set);
+
+static void
+usage(void)
+{
+	fprintf(stderr, "%s\n%s\n%s\n%s\n%s\n",
+	    "usage: devctl attach device",
+	    "       devctl detach [-f] device",
+	    "       devctl disable [-f] device",
+	    "       devctl enable device",
+	    "       devctl set driver [-f] device driver");
+	exit(1);
+}
+
+static int
+devctl_table_handler(struct devctl_command **start,
+    struct devctl_command **end, int ac, char **av)
+{
+	struct devctl_command **cmd;
+
+	if (ac < 2) {
+		warnx("The %s command requires a sub-command.", av[0]);
+		return (EINVAL);
+	}
+	for (cmd = start; cmd < end; cmd++) {
+		if (strcmp((*cmd)->name, av[1]) == 0)
+			return ((*cmd)->handler(ac - 1, av + 1));
+	}
+
+	warnx("%s is not a valid sub-command of %s.", av[1], av[0]);
+	return (ENOENT);
+}
+
+static int
+help(int ac __unused, char **av __unused)
+{
+
+	usage();
+	return (0);
+}
+DEVCTL_COMMAND(top, help, help);
+
+static int
+attach(int ac, char **av)
+{
+
+	if (ac != 2)
+		usage();
+	if (devctl_attach(av[1]) < 0)
+		err(1, "Failed to attach %s", av[1]);
+	return (0);
+}
+DEVCTL_COMMAND(top, attach, attach);
+
+static void
+detach_usage(void)
+{
+
+	fprintf(stderr, "usage: devctl detach [-f] device\n");
+	exit(1);
+}
+
+static int
+detach(int ac, char **av)
+{
+	bool force;
+	int ch;
+
+	force = false;
+	while ((ch = getopt(ac, av, "f")) != -1)
+		switch (ch) {
+		case 'f':
+			force = true;
+			break;
+		default:
+			detach_usage();
+		}
+	ac -= optind;
+	av += optind;
+
+	if (ac != 1)
+		detach_usage();
+	if (devctl_detach(av[0], force) < 0)
+		err(1, "Failed to detach %s", av[0]);
+	return (0);
+}
+DEVCTL_COMMAND(top, detach, detach);
+
+static void
+disable_usage(void)
+{
+
+	fprintf(stderr, "usage: devctl disable [-f] device\n");
+	exit(1);
+}
+
+static int
+disable(int ac, char **av)
+{
+	bool force;
+	int ch;
+
+	force = false;
+	while ((ch = getopt(ac, av, "f")) != -1)
+		switch (ch) {
+		case 'f':
+			force = true;
+			break;
+		default:
+			disable_usage();
+		}
+	ac -= optind;
+	av += optind;
+
+	if (ac != 1)
+		disable_usage();
+	if (devctl_disable(av[0], force) < 0)
+		err(1, "Failed to disable %s", av[0]);
+	return (0);
+}
+DEVCTL_COMMAND(top, disable, disable);
+
+static int
+enable(int ac, char **av)
+{
+
+	if (ac != 2)
+		usage();
+	if (devctl_enable(av[1]) < 0)
+		err(1, "Failed to enable %s", av[1]);
+	return (0);
+}
+DEVCTL_COMMAND(top, enable, enable);
+
+static void
+set_driver_usage(void)
+{
+
+	fprintf(stderr, "usage: devctl set driver [-f] device driver\n");
+	exit(1);
+}
+
+static int
+set_driver(int ac, char **av)
+{
+	bool force;
+	int ch;
+
+	force = false;
+	while ((ch = getopt(ac, av, "f")) != -1)
+		switch (ch) {
+		case 'f':
+			force = true;
+			break;
+		default:
+			set_driver_usage();
+		}
+	ac -= optind;
+	av += optind;
+
+	if (ac != 2)
+		set_driver_usage();
+	if (devctl_set_driver(av[0], av[1], force) < 0)
+		err(1, "Failed to set %s driver to %s", av[0], av[1]);
+	return (0);
+}
+DEVCTL_COMMAND(set, driver, set_driver);
+
+int
+main(int ac, char *av[])
+{
+	struct devctl_command **cmd;
+
+	if (ac == 1)
+		usage();
+	ac--;
+	av++;
+
+	SET_FOREACH(cmd, DEVCTL_DATASET(top)) {
+		if (strcmp((*cmd)->name, av[0]) == 0) {
+			if ((*cmd)->handler(ac, av) != 0)
+				return (1);
+			else
+				return (0);
+		}
+	}
+	warnx("Unknown command %s.", av[0]);
+	return (1);
+}