Merge from head

53 files changed, 328 insertions, 157 deletions

diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile
index b06dba7..fa0c995 100644
--- a/usr.sbin/Makefile
+++ b/usr.sbin/Makefile
@@ -1,7 +1,7 @@
 #	From: @(#)Makefile	5.20 (Berkeley) 6/12/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 SUBDIR=	adduser \
 	arp \
diff --git a/usr.sbin/amd/Makefile.inc b/usr.sbin/amd/Makefile.inc
index cc616f1..e4425f5 100644
--- a/usr.sbin/amd/Makefile.inc
+++ b/usr.sbin/amd/Makefile.inc
@@ -9,7 +9,7 @@
 #	Portions derived from amd/libamu/Makefile
 #	$NetBSD: Makefile,v 1.8 1998/08/08 22:33:37 christos Exp $
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 CFLAGS+= -I. -I${.CURDIR} 
 CFLAGS+= -I${.CURDIR}/../include
diff --git a/usr.sbin/amd/amd/Makefile b/usr.sbin/amd/amd/Makefile
index c720003..1fc41bf 100644
--- a/usr.sbin/amd/amd/Makefile
+++ b/usr.sbin/amd/amd/Makefile
@@ -6,7 +6,7 @@
 # $FreeBSD$
 #
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../../contrib/amd/amd
 
diff --git a/usr.sbin/amd/include/Makefile b/usr.sbin/amd/include/Makefile
index df770d1..e594d71 100644
--- a/usr.sbin/amd/include/Makefile
+++ b/usr.sbin/amd/include/Makefile
@@ -7,7 +7,7 @@
 #	Portions derived from amd/libamu/Makefile
 #	$NetBSD: Makefile,v 1.8 1998/08/08 22:33:37 christos Exp $
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 SRCS=	config_local.h
 .if ${MK_NIS} == "no"
diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8
index 88e7214..aa97a67 100644
--- a/usr.sbin/bhyve/bhyve.8
+++ b/usr.sbin/bhyve/bhyve.8
@@ -35,7 +35,7 @@
 .Op Fl aehwxAHPW
 .Op Fl c Ar numcpus
 .Op Fl g Ar gdbport
-.Op Fl p Ar pinnedcpu
+.Op Fl p Ar vcpu:hostcpu
 .Op Fl s Ar slot,emulation Ns Op , Ns Ar conf
 .Op Fl l Ar lpcdev Ns Op , Ns Ar conf
 .Ar vmname
@@ -80,12 +80,11 @@ For
 allow a remote kernel kgdb to be relayed to the guest kernel gdb stub
 via a local IPv4 address and this port.
 This option will be deprecated in a future version.
-.It Fl p Ar pinnedcpu
-Force guest virtual CPUs to be pinned to host CPUs.
-Virtual CPU
-.Em n
-is pinned to host CPU
-.Em pinnedcpu+n .
+.It Fl p Ar vcpu:hostcpu
+Pin guest's virtual CPU
+.Em vcpu
+to
+.Em hostcpu .
 .It Fl P
 Force the guest virtual CPU to exit when a PAUSE instruction is detected.
 .It Fl W
@@ -236,6 +235,8 @@ This is intended for debug purposes.
 Ignore accesses to unimplemented Model Specific Registers (MSRs). This is intended for debug purposes.
 .It Fl x
 The guest's local APIC is configured in x2APIC mode.
+.It Fl Y
+Disable MPtable generation.
 .It Fl h
 Print help message and exit.
 .It Ar vmname
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index e8010e6..e662ca3 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -85,7 +85,6 @@ char *vmname;
 int guest_ncpus;
 char *guest_uuid_str;
 
-static int pincpu = -1;
 static int guest_vmexit_on_hlt, guest_vmexit_on_pause;
 static int virtio_msix = 1;
 static int x2apic_mode = 0;	/* default is xAPIC */
@@ -114,6 +113,7 @@ struct bhyvestats {
         uint64_t        cpu_switch_rotate;
         uint64_t        cpu_switch_direct;
         int             io_reset;
+	int		io_poweroff;
 } stats;
 
 struct mt_vmm_info {
@@ -122,18 +122,20 @@ struct mt_vmm_info {
 	int		mt_vcpu;	
 } mt_vmm_info[VM_MAXCPU];
 
+static cpuset_t *vcpumap[VM_MAXCPU] = { NULL };
+
 static void
 usage(int code)
 {
 
         fprintf(stderr,
-                "Usage: %s [-aehwAHIPW] [-g <gdb port>] [-s <pci>]\n"
-		"       %*s [-c vcpus] [-p pincpu] [-m mem] [-l <lpc>] <vm>\n"
+                "Usage: %s [-aehwAHIPW] [-g <gdb port>] [-s <pci>] [-c vcpus]\n"
+		"       %*s [-p vcpu:hostcpu] [-m mem] [-l <lpc>] <vm>\n"
 		"       -a: local apic is in xAPIC mode (deprecated)\n"
 		"       -A: create an ACPI table\n"
 		"       -g: gdb port\n"
 		"       -c: # cpus (default 1)\n"
-		"       -p: pin vcpu 'n' to host cpu 'pincpu + n'\n"
+		"       -p: pin 'vcpu' to 'hostcpu'\n"
 		"       -H: vmexit from the guest on hlt\n"
 		"       -P: vmexit from the guest on pause\n"
 		"       -W: force virtio to use single-vector MSI\n"
@@ -144,12 +146,46 @@ usage(int code)
 		"       -m: memory size in MB\n"
 		"       -w: ignore unimplemented MSRs\n"
 		"       -x: local apic is in x2APIC mode\n"
+		"       -Y: disable MPtable generation\n"
 		"       -U: uuid\n",
 		progname, (int)strlen(progname), "");
 
 	exit(code);
 }
 
+static int
+pincpu_parse(const char *opt)
+{
+	int vcpu, pcpu;
+
+	if (sscanf(opt, "%d:%d", &vcpu, &pcpu) != 2) {
+		fprintf(stderr, "invalid format: %s\n", opt);
+		return (-1);
+	}
+
+	if (vcpu < 0 || vcpu >= VM_MAXCPU) {
+		fprintf(stderr, "vcpu '%d' outside valid range from 0 to %d\n",
+		    vcpu, VM_MAXCPU - 1);
+		return (-1);
+	}
+
+	if (pcpu < 0 || pcpu >= CPU_SETSIZE) {
+		fprintf(stderr, "hostcpu '%d' outside valid range from "
+		    "0 to %d\n", pcpu, CPU_SETSIZE - 1);
+		return (-1);
+	}
+
+	if (vcpumap[vcpu] == NULL) {
+		if ((vcpumap[vcpu] = malloc(sizeof(cpuset_t))) == NULL) {
+			perror("malloc");
+			return (-1);
+		}
+		CPU_ZERO(vcpumap[vcpu]);
+	}
+	CPU_SET(pcpu, vcpumap[vcpu]);
+	return (0);
+}
+
 void *
 paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
 {
@@ -227,8 +263,7 @@ fbsdrun_deletecpu(struct vmctx *ctx, int vcpu)
 {
 
 	if (!CPU_ISSET(vcpu, &cpumask)) {
-		fprintf(stderr, "addcpu: attempting to delete unknown cpu %d\n",
-		    vcpu);
+		fprintf(stderr, "Attempting to delete unknown cpu %d\n", vcpu);
 		exit(1);
 	}
 
@@ -237,13 +272,6 @@ fbsdrun_deletecpu(struct vmctx *ctx, int vcpu)
 }
 
 static int
-vmexit_catch_reset(void)
-{
-        stats.io_reset++;
-        return (VMEXIT_RESET);
-}
-
-static int
 vmexit_catch_inout(void)
 {
 	return (VMEXIT_ABORT);
@@ -293,8 +321,10 @@ vmexit_inout(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
 	case INOUT_OK:
 		return (VMEXIT_CONTINUE);
 	case INOUT_RESET:
+		stats.io_reset++;
 		return (VMEXIT_RESET);
 	case INOUT_POWEROFF:
+		stats.io_poweroff++;
 		return (VMEXIT_POWEROFF);
 	default:
 		fprintf(stderr, "Unhandled %s%c 0x%04x\n",
@@ -365,17 +395,6 @@ vmexit_spinup_ap(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
 }
 
 static int
-vmexit_spindown_cpu(struct vmctx *ctx, struct vm_exit *vme, int *pvcpu)
-{
-	int lastcpu;
-
-	lastcpu = fbsdrun_deletecpu(ctx, *pvcpu);
-	if (!lastcpu)
-		pthread_exit(NULL);
-	return (vmexit_catch_reset());
-}
-
-static int
 vmexit_vmx(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 {
 
@@ -461,17 +480,17 @@ vmexit_inst_emul(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 
 static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER;
 static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER;
-static int resetcpu = -1;
 
 static int
 vmexit_suspend(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 {
-	
-	assert(resetcpu != -1);
+	enum vm_suspend_how how;
+
+	how = vmexit->u.suspended.how;
 
 	fbsdrun_deletecpu(ctx, *pvcpu);
 
-	if (*pvcpu != resetcpu) {
+	if (*pvcpu != BSP) {
 		pthread_mutex_lock(&resetcpu_mtx);
 		pthread_cond_signal(&resetcpu_cond);
 		pthread_mutex_unlock(&resetcpu_mtx);
@@ -483,7 +502,19 @@ vmexit_suspend(struct vmctx *ctx, struct vm_exit *vmexit, int *pvcpu)
 		pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx);
 	}
 	pthread_mutex_unlock(&resetcpu_mtx);
-	exit(0);
+
+	switch (how) {
+	case VM_SUSPEND_RESET:
+		exit(0);
+	case VM_SUSPEND_POWEROFF:
+		exit(1);
+	case VM_SUSPEND_HALT:
+		exit(2);
+	default:
+		fprintf(stderr, "vmexit_suspend: invalid reason %d\n", how);
+		exit(100);
+	}
+	return (0);	/* NOTREACHED */
 }
 
 static vmexit_handler_t handler[VM_EXITCODE_MAX] = {
@@ -495,22 +526,19 @@ static vmexit_handler_t handler[VM_EXITCODE_MAX] = {
 	[VM_EXITCODE_MTRAP]  = vmexit_mtrap,
 	[VM_EXITCODE_INST_EMUL] = vmexit_inst_emul,
 	[VM_EXITCODE_SPINUP_AP] = vmexit_spinup_ap,
-	[VM_EXITCODE_SPINDOWN_CPU] = vmexit_spindown_cpu,
 	[VM_EXITCODE_SUSPENDED] = vmexit_suspend
 };
 
 static void
 vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip)
 {
-	cpuset_t mask;
 	int error, rc, prevcpu;
 	enum vm_exitcode exitcode;
+	enum vm_suspend_how how;
 
-	if (pincpu >= 0) {
-		CPU_ZERO(&mask);
-		CPU_SET(pincpu + vcpu, &mask);
+	if (vcpumap[vcpu] != NULL) {
 		error = pthread_setaffinity_np(pthread_self(),
-					       sizeof(mask), &mask);
+		    sizeof(cpuset_t), vcpumap[vcpu]);
 		assert(error == 0);
 	}
 
@@ -538,10 +566,13 @@ vm_loop(struct vmctx *ctx, int vcpu, uint64_t rip)
                         rip = vmexit[vcpu].rip;
 			break;
 		case VMEXIT_RESET:
-			if (vm_suspend(ctx) == 0) {
-				assert(resetcpu == -1);
-				resetcpu = vcpu;
-			}
+		case VMEXIT_POWEROFF:
+			if (rc == VMEXIT_RESET)
+				how = VM_SUSPEND_RESET;
+			else
+				how = VM_SUSPEND_POWEROFF;
+			error = vm_suspend(ctx, how);
+			assert(error == 0 || errno == EALREADY);
                         rip = vmexit[vcpu].rip + vmexit[vcpu].inst_length;
 			break;
 		default:
@@ -616,7 +647,7 @@ int
 main(int argc, char *argv[])
 {
 	int c, error, gdb_port, err, bvmcons;
-	int max_vcpus;
+	int max_vcpus, mptgen;
 	struct vmctx *ctx;
 	uint64_t rip;
 	size_t memsize;
@@ -626,8 +657,9 @@ main(int argc, char *argv[])
 	gdb_port = 0;
 	guest_ncpus = 1;
 	memsize = 256 * MB;
+	mptgen = 1;
 
-	while ((c = getopt(argc, argv, "abehwxAHIPWp:g:c:s:m:l:U:")) != -1) {
+	while ((c = getopt(argc, argv, "abehwxAHIPWYp:g:c:s:m:l:U:")) != -1) {
 		switch (c) {
 		case 'a':
 			x2apic_mode = 0;
@@ -639,7 +671,10 @@ main(int argc, char *argv[])
 			bvmcons = 1;
 			break;
 		case 'p':
-			pincpu = atoi(optarg);
+                        if (pincpu_parse(optarg) != 0) {
+                            errx(EX_USAGE, "invalid vcpu pinning "
+                                 "configuration '%s'", optarg);
+                        }
 			break;
                 case 'c':
 			guest_ncpus = atoi(optarg);
@@ -693,6 +728,9 @@ main(int argc, char *argv[])
 		case 'x':
 			x2apic_mode = 1;
 			break;
+		case 'Y':
+			mptgen = 0;
+			break;
 		case 'h':
 			usage(0);			
 		default:
@@ -752,7 +790,11 @@ main(int argc, char *argv[])
 	/*
 	 * build the guest tables, MP etc.
 	 */
-	mptable_build(ctx, guest_ncpus);
+	if (mptgen) {
+		error = mptable_build(ctx, guest_ncpus);
+		if (error)
+			exit(1);
+	}
 
 	error = smbios_build(ctx);
 	assert(error == 0);
diff --git a/usr.sbin/bhyve/mevent.c b/usr.sbin/bhyve/mevent.c
index b83ec69..07d3baf 100644
--- a/usr.sbin/bhyve/mevent.c
+++ b/usr.sbin/bhyve/mevent.c
@@ -52,9 +52,10 @@ __FBSDID("$FreeBSD$");
 
 #define	MEVENT_MAX	64
 
-#define MEV_ENABLE	1
-#define MEV_DISABLE	2
-#define MEV_DEL_PENDING	3
+#define	MEV_ADD		1
+#define	MEV_ENABLE	2
+#define	MEV_DISABLE	3
+#define	MEV_DEL_PENDING	4
 
 extern char *vmname;
 
@@ -147,10 +148,11 @@ mevent_kq_flags(struct mevent *mevp)
 	int ret;
 
 	switch (mevp->me_state) {
+	case MEV_ADD:
+		ret = EV_ADD;		/* implicitly enabled */
+		break;
 	case MEV_ENABLE:
-		ret = EV_ADD;
-		if (mevp->me_type == EVF_TIMER)
-			ret |= EV_ENABLE;
+		ret = EV_ENABLE;
 		break;
 	case MEV_DISABLE:
 		ret = EV_DISABLE;
@@ -158,6 +160,9 @@ mevent_kq_flags(struct mevent *mevp)
 	case MEV_DEL_PENDING:
 		ret = EV_DELETE;
 		break;
+	default:
+		assert(0);
+		break;
 	}
 
 	return (ret);
@@ -284,7 +289,7 @@ mevent_add(int tfd, enum ev_type type,
 
 	LIST_INSERT_HEAD(&change_head, mevp, me_list);
 	mevp->me_cq = 1;
-	mevp->me_state = MEV_ENABLE;
+	mevp->me_state = MEV_ADD;
 	mevent_notify();
 
 exit:
diff --git a/usr.sbin/bhyve/mptbl.c b/usr.sbin/bhyve/mptbl.c
index e895835..4c2167e 100644
--- a/usr.sbin/bhyve/mptbl.c
+++ b/usr.sbin/bhyve/mptbl.c
@@ -303,16 +303,31 @@ mptable_build(struct vmctx *ctx, int ncpu)
 	proc_entry_ptr		mpep;
 	mpfps_t			mpfp;
 	int_entry_ptr		mpie;
-	int			ioints;
+	int			ioints, bus;
 	char 			*curraddr;
 	char 			*startaddr;
 
 	startaddr = paddr_guest2host(ctx, MPTABLE_BASE, MPTABLE_MAX_LENGTH);
 	if (startaddr == NULL) {
-		printf("mptable requires mapped mem\n");
+		fprintf(stderr, "mptable requires mapped mem\n");
 		return (ENOMEM);
 	}
 
+	/*
+	 * There is no way to advertise multiple PCI hierarchies via MPtable
+	 * so require that there is no PCI hierarchy with a non-zero bus
+	 * number.
+	 */
+	for (bus = 1; bus <= PCI_BUSMAX; bus++) {
+		if (pci_bus_configured(bus)) {
+			fprintf(stderr, "MPtable is incompatible with "
+			    "multiple PCI hierarchies.\r\n");
+			fprintf(stderr, "MPtable generation can be disabled "
+			    "by passing the -Y option to bhyve(8).\r\n");
+			return (EINVAL);
+		}
+	}
+
 	curraddr = startaddr;
 	mpfp = (mpfps_t)curraddr;
 	mpt_build_mpfp(mpfp, MPTABLE_BASE);
diff --git a/usr.sbin/bhyve/pci_ahci.c b/usr.sbin/bhyve/pci_ahci.c
index 3f9fc7f..9f61107 100644
--- a/usr.sbin/bhyve/pci_ahci.c
+++ b/usr.sbin/bhyve/pci_ahci.c
@@ -192,6 +192,7 @@ struct pci_ahci_softc {
 	uint32_t em_ctl;
 	uint32_t cap2;
 	uint32_t bohc;
+	uint32_t lintr;
 	struct ahci_port port[MAX_PORTS];
 };
 #define	ahci_ctx(sc)	((sc)->asc_pi->pi_vmctx)
@@ -211,8 +212,11 @@ static inline void lba_to_msf(uint8_t *buf, int lba)
 static void
 ahci_generate_intr(struct pci_ahci_softc *sc)
 {
+	struct pci_devinst *pi;
 	int i;
 
+	pi = sc->asc_pi;
+
 	for (i = 0; i < sc->ports; i++) {
 		struct ahci_port *pr;
 		pr = &sc->port[i];
@@ -222,8 +226,28 @@ ahci_generate_intr(struct pci_ahci_softc *sc)
 
 	DPRINTF("%s %x\n", __func__, sc->is);
 
-	if (sc->is && (sc->ghc & AHCI_GHC_IE))
-		pci_generate_msi(sc->asc_pi, 0);
+	if (sc->is && (sc->ghc & AHCI_GHC_IE)) {		
+		if (pci_msi_enabled(pi)) {
+			/*
+			 * Generate an MSI interrupt on every edge
+			 */
+			pci_generate_msi(pi, 0);
+		} else if (!sc->lintr) {
+			/*
+			 * Only generate a pin-based interrupt if one wasn't
+			 * in progress
+			 */
+			sc->lintr = 1;
+			pci_lintr_assert(pi);
+		}
+	} else if (sc->lintr) {
+		/*
+		 * No interrupts: deassert pin-based signal if it had
+		 * been asserted
+		 */
+		pci_lintr_deassert(pi);
+		sc->lintr = 0;
+	}
 }
 
 static void
@@ -367,6 +391,12 @@ ahci_reset(struct pci_ahci_softc *sc)
 
 	sc->ghc = AHCI_GHC_AE;
 	sc->is = 0;
+
+	if (sc->lintr) {
+		pci_lintr_deassert(sc->asc_pi);
+		sc->lintr = 0;
+	}
+
 	for (i = 0; i < sc->ports; i++) {
 		sc->port[i].ie = 0;
 		sc->port[i].is = 0;
@@ -1814,6 +1844,8 @@ pci_ahci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts, int atapi)
 	pci_emul_alloc_bar(pi, 5, PCIBAR_MEM32,
 	    AHCI_OFFSET + sc->ports * AHCI_STEP);
 
+	pci_lintr_request(pi);
+
 open_fail:
 	if (ret) {
 		blockif_close(sc->port[0].bctx);
diff --git a/usr.sbin/bhyve/pci_emul.c b/usr.sbin/bhyve/pci_emul.c
index 15f2fad..5b87da7 100644
--- a/usr.sbin/bhyve/pci_emul.c
+++ b/usr.sbin/bhyve/pci_emul.c
@@ -1261,6 +1261,13 @@ pci_write_dsdt(void)
 }
 
 int
+pci_bus_configured(int bus)
+{
+	assert(bus >= 0 && bus < MAXBUSES);
+	return (pci_businfo[bus] != NULL);
+}
+
+int
 pci_msi_enabled(struct pci_devinst *pi)
 {
 	return (pi->pi_msi.enabled);
diff --git a/usr.sbin/bhyve/pci_emul.h b/usr.sbin/bhyve/pci_emul.h
index 7ad0a5d..e1040a8 100644
--- a/usr.sbin/bhyve/pci_emul.h
+++ b/usr.sbin/bhyve/pci_emul.h
@@ -233,6 +233,7 @@ uint64_t pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int size);
 int	pci_count_lintr(int bus);
 void	pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg);
 void	pci_write_dsdt(void);
+int	pci_bus_configured(int bus);
 
 static __inline void 
 pci_set_cfgdata8(struct pci_devinst *pi, int offset, uint8_t val)
diff --git a/usr.sbin/bhyve/uart_emul.c b/usr.sbin/bhyve/uart_emul.c
index 38e14a9..4242e5c 100644
--- a/usr.sbin/bhyve/uart_emul.c
+++ b/usr.sbin/bhyve/uart_emul.c
@@ -110,6 +110,7 @@ struct uart_softc {
 	uint8_t dlh;		/* Baudrate divisor latch MSB */
 
 	struct fifo rxfifo;
+	struct mevent *mev;
 
 	struct ttyfd tty;
 	bool	thre_int_pending;	/* THRE interrupt pending */
@@ -145,34 +146,15 @@ ttyopen(struct ttyfd *tf)
 	}
 }
 
-static bool
-tty_char_available(struct ttyfd *tf)
-{
-	fd_set rfds;
-	struct timeval tv;
-
-	FD_ZERO(&rfds);
-	FD_SET(tf->fd, &rfds);
-	tv.tv_sec = 0;
-	tv.tv_usec = 0;
-	if (select(tf->fd + 1, &rfds, NULL, NULL, &tv) > 0 ) {
-		return (true);
-	} else {
-		return (false);
-	}
-}
-
 static int
 ttyread(struct ttyfd *tf)
 {
-	char rb;
+	unsigned char rb;
 
-	if (tty_char_available(tf)) {
-		read(tf->fd, &rb, 1);
-		return (rb & 0xff);
-	} else {
+	if (read(tf->fd, &rb, 1) == 1)
+		return (rb);
+	else
 		return (-1);
-	}
 }
 
 static void
@@ -183,62 +165,111 @@ ttywrite(struct ttyfd *tf, unsigned char wb)
 }
 
 static void
-fifo_reset(struct fifo *fifo, int size)
+rxfifo_reset(struct uart_softc *sc, int size)
 {
+	char flushbuf[32];
+	struct fifo *fifo;
+	ssize_t nread;
+	int error;
 
+	fifo = &sc->rxfifo;
 	bzero(fifo, sizeof(struct fifo));
 	fifo->size = size;
+
+	if (sc->tty.opened) {
+		/*
+		 * Flush any unread input from the tty buffer.
+		 */
+		while (1) {
+			nread = read(sc->tty.fd, flushbuf, sizeof(flushbuf));
+			if (nread != sizeof(flushbuf))
+				break;
+		}
+
+		/*
+		 * Enable mevent to trigger when new characters are available
+		 * on the tty fd.
+		 */
+		error = mevent_enable(sc->mev);
+		assert(error == 0);
+	}
 }
 
 static int
-fifo_putchar(struct fifo *fifo, uint8_t ch)
+rxfifo_available(struct uart_softc *sc)
 {
+	struct fifo *fifo;
+
+	fifo = &sc->rxfifo;
+	return (fifo->num < fifo->size);
+}
+
+static int
+rxfifo_putchar(struct uart_softc *sc, uint8_t ch)
+{
+	struct fifo *fifo;
+	int error;
+
+	fifo = &sc->rxfifo;
 
 	if (fifo->num < fifo->size) {
 		fifo->buf[fifo->windex] = ch;
 		fifo->windex = (fifo->windex + 1) % fifo->size;
 		fifo->num++;
+		if (!rxfifo_available(sc)) {
+			if (sc->tty.opened) {
+				/*
+				 * Disable mevent callback if the FIFO is full.
+				 */
+				error = mevent_disable(sc->mev);
+				assert(error == 0);
+			}
+		}
 		return (0);
 	} else
 		return (-1);
 }
 
 static int
-fifo_getchar(struct fifo *fifo)
+rxfifo_getchar(struct uart_softc *sc)
 {
-	int c;
+	struct fifo *fifo;
+	int c, error, wasfull;
 
+	wasfull = 0;
+	fifo = &sc->rxfifo;
 	if (fifo->num > 0) {
+		if (!rxfifo_available(sc))
+			wasfull = 1;
 		c = fifo->buf[fifo->rindex];
 		fifo->rindex = (fifo->rindex + 1) % fifo->size;
 		fifo->num--;
+		if (wasfull) {
+			if (sc->tty.opened) {
+				error = mevent_enable(sc->mev);
+				assert(error == 0);
+			}
+		}
 		return (c);
 	} else
 		return (-1);
 }
 
 static int
-fifo_numchars(struct fifo *fifo)
+rxfifo_numchars(struct uart_softc *sc)
 {
+	struct fifo *fifo = &sc->rxfifo;
 
 	return (fifo->num);
 }
 
-static int
-fifo_available(struct fifo *fifo)
-{
-
-	return (fifo->num < fifo->size);
-}
-
 static void
 uart_opentty(struct uart_softc *sc)
 {
-	struct mevent *mev;
 
 	ttyopen(&sc->tty);
-	mev = mevent_add(sc->tty.fd, EVF_READ, uart_drain, sc);
-	assert(mev);
+	sc->mev = mevent_add(sc->tty.fd, EVF_READ, uart_drain, sc);
+	assert(sc->mev != NULL);
 }
 
 /*
@@ -255,7 +286,7 @@ uart_intr_reason(struct uart_softc *sc)
 
 	if ((sc->lsr & LSR_OE) != 0 && (sc->ier & IER_ERLS) != 0)
 		return (IIR_RLS);
-	else if (fifo_numchars(&sc->rxfifo) > 0 && (sc->ier & IER_ERXRDY) != 0)
+	else if (rxfifo_numchars(sc) > 0 && (sc->ier & IER_ERXRDY) != 0)
 		return (IIR_RXTOUT);
 	else if (sc->thre_int_pending && (sc->ier & IER_ETXRDY) != 0)
 		return (IIR_TXRDY);
@@ -274,7 +305,7 @@ uart_reset(struct uart_softc *sc)
 	sc->dll = divisor;
 	sc->dlh = divisor >> 16;
 
-	fifo_reset(&sc->rxfifo, 1);	/* no fifo until enabled by software */
+	rxfifo_reset(sc, 1);	/* no fifo until enabled by software */
 }
 
 /*
@@ -315,9 +346,9 @@ uart_drain(int fd, enum ev_type ev, void *arg)
 	if ((sc->mcr & MCR_LOOPBACK) != 0) {
 		(void) ttyread(&sc->tty);
 	} else {
-		while (fifo_available(&sc->rxfifo) &&
+		while (rxfifo_available(sc) &&
 		       ((ch = ttyread(&sc->tty)) != -1)) {
-			fifo_putchar(&sc->rxfifo, ch);
+			rxfifo_putchar(sc, ch);
 		}
 		uart_toggle_intr(sc);
 	}
@@ -351,7 +382,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
         switch (offset) {
 	case REG_DATA:
 		if (sc->mcr & MCR_LOOPBACK) {
-			if (fifo_putchar(&sc->rxfifo, value) != 0)
+			if (rxfifo_putchar(sc, value) != 0)
 				sc->lsr |= LSR_OE;
 		} else if (sc->tty.opened) {
 			ttywrite(&sc->tty, value);
@@ -372,7 +403,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 			 */
 			if ((sc->fcr & FCR_ENABLE) ^ (value & FCR_ENABLE)) {
 				fifosz = (value & FCR_ENABLE) ? FIFOSZ : 1;
-				fifo_reset(&sc->rxfifo, fifosz);
+				rxfifo_reset(sc, fifosz);
 			}
 
 			/*
@@ -383,7 +414,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 				sc->fcr = 0;
 			} else {
 				if ((value & FCR_RCV_RST) != 0)
-					fifo_reset(&sc->rxfifo, FIFOSZ);
+					rxfifo_reset(sc, FIFOSZ);
 
 				sc->fcr = value &
 					 (FCR_ENABLE | FCR_DMA | FCR_RX_MASK);
@@ -480,7 +511,7 @@ uart_read(struct uart_softc *sc, int offset)
 
 	switch (offset) {
 	case REG_DATA:
-		reg = fifo_getchar(&sc->rxfifo);
+		reg = rxfifo_getchar(sc);
 		break;
 	case REG_IER:
 		reg = sc->ier;
@@ -511,7 +542,7 @@ uart_read(struct uart_softc *sc, int offset)
 		sc->lsr |= LSR_TEMT | LSR_THRE;
 
 		/* Check for new receive data */
-		if (fifo_numchars(&sc->rxfifo) > 0)
+		if (rxfifo_numchars(sc) > 0)
 			sc->lsr |= LSR_RXRDY;
 		else
 			sc->lsr &= ~LSR_RXRDY;
@@ -615,6 +646,10 @@ uart_set_backend(struct uart_softc *sc, const char *opts)
 		retval = 0;
 	}
 
+	/* Make the backend file descriptor non-blocking */
+	if (retval == 0)
+		retval = fcntl(sc->tty.fd, F_SETFL, O_NONBLOCK);
+
 	if (retval == 0)
 		uart_opentty(sc);
 
diff --git a/usr.sbin/bhyve/xmsr.c b/usr.sbin/bhyve/xmsr.c
index ba94125..63522bf 100644
--- a/usr.sbin/bhyve/xmsr.c
+++ b/usr.sbin/bhyve/xmsr.c
@@ -47,6 +47,8 @@ emulate_wrmsr(struct vmctx *ctx, int vcpu, uint32_t code, uint64_t val)
 	case 0xd04:			/* Sandy Bridge uncore PMC MSRs */
 	case 0xc24:
 		return (0);
+	case 0x79:
+		return (0);		/* IA32_BIOS_UPDT_TRIG MSR */
 	default:
 		break;
 	}
diff --git a/usr.sbin/bhyvectl/bhyvectl.c b/usr.sbin/bhyvectl/bhyvectl.c
index 2e732b5..ceee33a 100644
--- a/usr.sbin/bhyvectl/bhyvectl.c
+++ b/usr.sbin/bhyvectl/bhyvectl.c
@@ -191,13 +191,16 @@ usage(void)
 	"       [--get-highmem]\n"
 	"       [--get-gpa-pmap]\n"
 	"       [--assert-lapic-lvt=<pin>]\n"
-	"       [--inject-nmi]\n",
+	"       [--inject-nmi]\n"
+	"       [--force-reset]\n"
+	"       [--force-poweroff]\n",
 	progname);
 	exit(1);
 }
 
 static int get_stats, getcap, setcap, capval, get_gpa_pmap;
 static int inject_nmi, assert_lapic_lvt;
+static int force_reset, force_poweroff;
 static const char *capname;
 static int create, destroy, get_lowmem, get_highmem;
 static uint64_t memsize;
@@ -565,6 +568,8 @@ main(int argc, char *argv[])
 		{ "create",	NO_ARG,		&create,	1 },
 		{ "destroy",	NO_ARG,		&destroy,	1 },
 		{ "inject-nmi",	NO_ARG,		&inject_nmi,	1 },
+		{ "force-reset",	NO_ARG,	&force_reset,	1 },
+		{ "force-poweroff", NO_ARG,	&force_poweroff, 1 },
 		{ NULL,		0,		NULL,		0 }
 	};
 
@@ -1535,6 +1540,12 @@ main(int argc, char *argv[])
 			printf("vm_run error %d\n", error);
 	}
 
+	if (!error && force_reset)
+		error = vm_suspend(ctx, VM_SUSPEND_RESET);
+
+	if (!error && force_poweroff)
+		error = vm_suspend(ctx, VM_SUSPEND_POWEROFF);
+
 	if (error)
 		printf("errno = %d\n", errno);
 
diff --git a/usr.sbin/bootparamd/bootparamd/Makefile b/usr.sbin/bootparamd/bootparamd/Makefile
index a5163a3..0596993 100644
--- a/usr.sbin/bootparamd/bootparamd/Makefile
+++ b/usr.sbin/bootparamd/bootparamd/Makefile
@@ -1,7 +1,7 @@
 #	from: @(#)Makefile	5.8 (Berkeley) 7/28/90
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	bootparamd
 MAN=	bootparams.5 bootparamd.8
diff --git a/usr.sbin/bsdinstall/distextract/Makefile b/usr.sbin/bsdinstall/distextract/Makefile
index 1910b23..e359bba 100644
--- a/usr.sbin/bsdinstall/distextract/Makefile
+++ b/usr.sbin/bsdinstall/distextract/Makefile
@@ -8,7 +8,7 @@ LDADD=	-larchive -ldialog -lm
 WARNS?=	6
 MAN=
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_NCURSESW} == "no"
 DPADD+=	${LIBNCURSES}
diff --git a/usr.sbin/bsdinstall/distfetch/Makefile b/usr.sbin/bsdinstall/distfetch/Makefile
index 08eb91c..2ac9eb6 100644
--- a/usr.sbin/bsdinstall/distfetch/Makefile
+++ b/usr.sbin/bsdinstall/distfetch/Makefile
@@ -8,7 +8,7 @@ LDADD=	-lfetch -ldialog -lm
 WARNS?=	6
 MAN=
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_NCURSESW} == "no"
 DPADD+=	${LIBNCURSES}
diff --git a/usr.sbin/bsdinstall/partedit/Makefile b/usr.sbin/bsdinstall/partedit/Makefile
index 3eb0d3f..328259c 100644
--- a/usr.sbin/bsdinstall/partedit/Makefile
+++ b/usr.sbin/bsdinstall/partedit/Makefile
@@ -22,7 +22,7 @@ SRCS=	diskeditor.c partedit.c gpart_ops.c partedit_${PARTEDIT_ARCH}.c \
 WARNS?=	3
 MAN= sade.8
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_NCURSESW} == "no"
 DPADD+=	${LIBNCURSES}
diff --git a/usr.sbin/bsnmpd/bsnmpd/Makefile b/usr.sbin/bsnmpd/bsnmpd/Makefile
index 3e6df47..b20ba31 100644
--- a/usr.sbin/bsnmpd/bsnmpd/Makefile
+++ b/usr.sbin/bsnmpd/bsnmpd/Makefile
@@ -2,7 +2,7 @@
 #
 # Author: Harti Brandt <harti@freebsd.org>
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 CONTRIB=${.CURDIR}/../../../contrib/bsnmp
 .PATH: ${CONTRIB}/snmpd
diff --git a/usr.sbin/bsnmpd/modules/Makefile b/usr.sbin/bsnmpd/modules/Makefile
index 45558b7..24f9f02 100644
--- a/usr.sbin/bsnmpd/modules/Makefile
+++ b/usr.sbin/bsnmpd/modules/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../../contrib/bsnmp/snmpd
 
diff --git a/usr.sbin/bsnmpd/modules/snmp_hast/Makefile b/usr.sbin/bsnmpd/modules/snmp_hast/Makefile
index 6369918..e05ce31 100644
--- a/usr.sbin/bsnmpd/modules/snmp_hast/Makefile
+++ b/usr.sbin/bsnmpd/modules/snmp_hast/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH:	${.CURDIR}/../../../../sbin/hastd
 
diff --git a/usr.sbin/bsnmpd/tools/bsnmptools/Makefile b/usr.sbin/bsnmpd/tools/bsnmptools/Makefile
index 94a1cea..3310420 100644
--- a/usr.sbin/bsnmpd/tools/bsnmptools/Makefile
+++ b/usr.sbin/bsnmpd/tools/bsnmptools/Makefile
@@ -1,7 +1,7 @@
 # $FreeBSD$
 # Author: Shteryana Shopova <syrinx@FreeBSD.org>
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}
 
diff --git a/usr.sbin/etcupdate/Makefile b/usr.sbin/etcupdate/Makefile
index 765533a..155151d 100644
--- a/usr.sbin/etcupdate/Makefile
+++ b/usr.sbin/etcupdate/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 SCRIPTS=etcupdate.sh
 MAN=	etcupdate.8
diff --git a/usr.sbin/flowctl/Makefile b/usr.sbin/flowctl/Makefile
index a2dd377..acee16b 100644
--- a/usr.sbin/flowctl/Makefile
+++ b/usr.sbin/flowctl/Makefile
@@ -2,7 +2,7 @@
 # $FreeBSD$
 #
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 	
 PROG=	flowctl
 MAN=	flowctl.8
diff --git a/usr.sbin/gssd/Makefile b/usr.sbin/gssd/Makefile
index 3663bf9..9573702 100644
--- a/usr.sbin/gssd/Makefile
+++ b/usr.sbin/gssd/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	gssd
 MAN=	gssd.8
diff --git a/usr.sbin/ifmcstat/Makefile b/usr.sbin/ifmcstat/Makefile
index fab7ea6..5b967c7 100644
--- a/usr.sbin/ifmcstat/Makefile
+++ b/usr.sbin/ifmcstat/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.1 (Berkeley) 6/5/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	ifmcstat
 SRCS=	ifmcstat.c printb.c
diff --git a/usr.sbin/inetd/Makefile b/usr.sbin/inetd/Makefile
index 29aacd1..f31027af 100644
--- a/usr.sbin/inetd/Makefile
+++ b/usr.sbin/inetd/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.1 (Berkeley) 6/6/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	inetd
 MAN=	inetd.8
diff --git a/usr.sbin/jail/Makefile b/usr.sbin/jail/Makefile
index fb36def..58324ac 100644
--- a/usr.sbin/jail/Makefile
+++ b/usr.sbin/jail/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	jail
 MAN=	jail.8 jail.conf.5
diff --git a/usr.sbin/jls/Makefile b/usr.sbin/jls/Makefile
index b297cc4..431fd0c 100644
--- a/usr.sbin/jls/Makefile
+++ b/usr.sbin/jls/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	jls
 MAN=	jls.8
diff --git a/usr.sbin/kldxref/ef.c b/usr.sbin/kldxref/ef.c
index b683898..88fbc34 100644
--- a/usr.sbin/kldxref/ef.c
+++ b/usr.sbin/kldxref/ef.c
@@ -47,6 +47,7 @@
 
 #include "ef.h"
 
+#define	MAXSEGS 2
 struct ef_file {
 	char*		ef_name;
 	struct elf_file *ef_efile;
@@ -68,7 +69,7 @@ struct ef_file {
 	Elf_Off		ef_symoff;
 	Elf_Sym*	ef_symtab;
 	int		ef_nsegs;
-	Elf_Phdr *	ef_segs[2];
+	Elf_Phdr *	ef_segs[MAXSEGS];
 	int		ef_verbose;
 	Elf_Rel *	ef_rel;			/* relocation table */
 	int		ef_relsz;		/* number of entries */
@@ -580,12 +581,9 @@ ef_open(const char *filename, struct elf_file *efile, int verbose)
 				ef_print_phdr(phdr);
 			switch (phdr->p_type) {
 			case PT_LOAD:
-				if (nsegs == 2) {
-					warnx("%s: too many sections",
-					    filename);
-					break;
-				}
-				ef->ef_segs[nsegs++] = phdr;
+				if (nsegs < MAXSEGS)
+					ef->ef_segs[nsegs] = phdr;
+				nsegs++;
 				break;
 			case PT_PHDR:
 				break;
@@ -597,12 +595,15 @@ ef_open(const char *filename, struct elf_file *efile, int verbose)
 		}
 		if (verbose > 1)
 			printf("\n");
-		ef->ef_nsegs = nsegs;
 		if (phdyn == NULL) {
 			warnx("Skipping %s: not dynamically-linked",
 			    filename);
 			break;
+		} else if (nsegs > MAXSEGS) {
+			warnx("%s: too many sections", filename);
+			break;
 		}
+		ef->ef_nsegs = nsegs;
 		if (ef_read_entry(ef, phdyn->p_offset,
 			phdyn->p_filesz, (void**)&ef->ef_dyn) != 0) {
 			printf("ef_read_entry failed\n");
diff --git a/usr.sbin/lpr/Makefile.inc b/usr.sbin/lpr/Makefile.inc
index ab667dd..65535e6 100644
--- a/usr.sbin/lpr/Makefile.inc
+++ b/usr.sbin/lpr/Makefile.inc
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_INET6_SUPPORT} != "no"
 CFLAGS+= -DINET6
diff --git a/usr.sbin/mailwrapper/Makefile b/usr.sbin/mailwrapper/Makefile
index 8811f6f..8e49db7 100644
--- a/usr.sbin/mailwrapper/Makefile
+++ b/usr.sbin/mailwrapper/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_MAILWRAPPER} != "no"
 PROG=	mailwrapper
diff --git a/usr.sbin/mtest/Makefile b/usr.sbin/mtest/Makefile
index 9104c68..22c417c 100644
--- a/usr.sbin/mtest/Makefile
+++ b/usr.sbin/mtest/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	mtest
 MAN=	mtest.8
diff --git a/usr.sbin/newsyslog/Makefile b/usr.sbin/newsyslog/Makefile
index 4e3ae55..1c63e73 100644
--- a/usr.sbin/newsyslog/Makefile
+++ b/usr.sbin/newsyslog/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	newsyslog
 MAN=	newsyslog.8 newsyslog.conf.5
diff --git a/usr.sbin/ngctl/Makefile b/usr.sbin/ngctl/Makefile
index 071f1c3..a1d1e6b 100644
--- a/usr.sbin/ngctl/Makefile
+++ b/usr.sbin/ngctl/Makefile
@@ -1,7 +1,7 @@
 # $FreeBSD$
 # $Whistle: Makefile,v 1.3 1999/01/16 00:10:11 archie Exp $
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	ngctl
 MAN=	ngctl.8
diff --git a/usr.sbin/ntp/Makefile.inc b/usr.sbin/ntp/Makefile.inc
index 5095e89..cadd4c7 100644
--- a/usr.sbin/ntp/Makefile.inc
+++ b/usr.sbin/ntp/Makefile.inc
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 DEFS_LOCAL= -DPARSE -DHAVE_CONFIG_H
 NTPDEFS=   -DSYS_FREEBSD
diff --git a/usr.sbin/ntp/doc/Makefile b/usr.sbin/ntp/doc/Makefile
index 64e3571..9c42b09 100644
--- a/usr.sbin/ntp/doc/Makefile
+++ b/usr.sbin/ntp/doc/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 FILESDIR=	${SHAREDIR}/doc/ntp
 
diff --git a/usr.sbin/ntp/ntp-keygen/Makefile b/usr.sbin/ntp/ntp-keygen/Makefile
index 26716c2..fea148c 100644
--- a/usr.sbin/ntp/ntp-keygen/Makefile
+++ b/usr.sbin/ntp/ntp-keygen/Makefile
@@ -2,7 +2,7 @@
 
 MAN=
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../../contrib/ntp/util \
        ${.CURDIR}/../../../contrib/ntp/ntpd
diff --git a/usr.sbin/ntp/ntpd/Makefile b/usr.sbin/ntp/ntpd/Makefile
index 1ac4c72..53f537f 100644
--- a/usr.sbin/ntp/ntpd/Makefile
+++ b/usr.sbin/ntp/ntpd/Makefile
@@ -2,7 +2,7 @@
 
 MAN=
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../../contrib/ntp/ntpd
 
diff --git a/usr.sbin/pmcstat/pmcstat.8 b/usr.sbin/pmcstat/pmcstat.8
index 5555759..97a2f62 100644
--- a/usr.sbin/pmcstat/pmcstat.8
+++ b/usr.sbin/pmcstat/pmcstat.8
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 19, 2008
+.Dd April 29, 2014
 .Dt PMCSTAT 8
 .Os
 .Sh NAME
@@ -46,6 +46,7 @@
 .Op Fl S Ar event-spec
 .Op Fl T
 .Op Fl W
+.Op Fl a Ar pathname
 .Op Fl c Ar cpu-spec
 .Op Fl d
 .Op Fl f Ar pluginopt
@@ -221,6 +222,21 @@ This is an experimental feature intended to help analyse the
 dynamic behaviour of processes in the system.
 It may incur substantial overhead if enabled.
 The default is for this feature to be disabled.
+.It Fl a Ar pathname
+Perform a symbol and file:line lookup for each address in each
+callgraph and save the output to
+.Ar pathname .
+Unlike
+.Fl m
+that only resolves the first symbol in the graph, this resolves
+every node in the callgraph, or prints out addresses if no
+lookup information is available.
+This option requires the
+.Fl R
+option to read in samples that were previously collected and
+saved with the 
+.Fl o
+option.
 .It Fl c Ar cpu-spec
 Set the cpus for subsequent system mode PMCs specified on the
 command line to
diff --git a/usr.sbin/pmcstat/pmcstat.c b/usr.sbin/pmcstat/pmcstat.c
index 3edba3f..ae93b93 100644
--- a/usr.sbin/pmcstat/pmcstat.c
+++ b/usr.sbin/pmcstat/pmcstat.c
@@ -503,7 +503,7 @@ pmcstat_show_usage(void)
 	    "\t -S spec\t allocate a system-wide sampling PMC\n"
 	    "\t -T\t\t start in top mode\n"
 	    "\t -W\t\t (toggle) show counts per context switch\n"
-	    "\t -a <file>\t print sampled PCs and callgraph to \"file\"\n"
+	    "\t -a file\t print sampled PCs and callgraph to \"file\"\n"
 	    "\t -c cpu-list\t set cpus for subsequent system-wide PMCs\n"
 	    "\t -d\t\t (toggle) track descendants\n"
 	    "\t -f spec\t pass \"spec\" to as plugin option\n"
diff --git a/usr.sbin/ppp/Makefile b/usr.sbin/ppp/Makefile
index 9943665..a61ebf5 100644
--- a/usr.sbin/ppp/Makefile
+++ b/usr.sbin/ppp/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	ppp
 MAN=	ppp.8
diff --git a/usr.sbin/rpcbind/Makefile b/usr.sbin/rpcbind/Makefile
index dd042f2..2780001 100644
--- a/usr.sbin/rpcbind/Makefile
+++ b/usr.sbin/rpcbind/Makefile
@@ -1,7 +1,7 @@
 #	$NetBSD: Makefile,v 1.3 2000/06/20 13:56:43 fvdl Exp $
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	rpcbind
 MAN=	rpcbind.8
diff --git a/usr.sbin/sa/Makefile b/usr.sbin/sa/Makefile
index 23f9bf2..c3c79b0 100644
--- a/usr.sbin/sa/Makefile
+++ b/usr.sbin/sa/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../usr.bin/lastcomm
 
diff --git a/usr.sbin/sendmail/Makefile b/usr.sbin/sendmail/Makefile
index a9f7fbb..f8a910b 100644
--- a/usr.sbin/sendmail/Makefile
+++ b/usr.sbin/sendmail/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.8 (Berkeley) 3/28/97
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 SENDMAIL_DIR=${.CURDIR}/../../contrib/sendmail
 SMDIR=	${SENDMAIL_DIR}/src
diff --git a/usr.sbin/syslogd/Makefile b/usr.sbin/syslogd/Makefile
index 4764013..069e093 100644
--- a/usr.sbin/syslogd/Makefile
+++ b/usr.sbin/syslogd/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.1 (Berkeley) 6/6/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../usr.bin/wall
 
diff --git a/usr.sbin/tcpdchk/Makefile b/usr.sbin/tcpdchk/Makefile
index a49ffc6..e979916 100644
--- a/usr.sbin/tcpdchk/Makefile
+++ b/usr.sbin/tcpdchk/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../contrib/tcp_wrappers
 
diff --git a/usr.sbin/tcpdmatch/Makefile b/usr.sbin/tcpdmatch/Makefile
index caae746..f8ccf8a 100644
--- a/usr.sbin/tcpdmatch/Makefile
+++ b/usr.sbin/tcpdmatch/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .PATH: ${.CURDIR}/../../contrib/tcp_wrappers
 
diff --git a/usr.sbin/tcpdump/tcpdump/Makefile b/usr.sbin/tcpdump/tcpdump/Makefile
index ac7636ac..4982873 100644
--- a/usr.sbin/tcpdump/tcpdump/Makefile
+++ b/usr.sbin/tcpdump/tcpdump/Makefile
@@ -1,6 +1,6 @@
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 TCPDUMP_DISTDIR?= ${.CURDIR}/../../../contrib/tcpdump
 .PATH: ${TCPDUMP_DISTDIR}
diff --git a/usr.sbin/trpt/Makefile b/usr.sbin/trpt/Makefile
index 5c62ce0..e49fff1 100644
--- a/usr.sbin/trpt/Makefile
+++ b/usr.sbin/trpt/Makefile
@@ -1,7 +1,7 @@
 #	@(#)Makefile	8.1 (Berkeley) 6/6/93
 # $FreeBSD$
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 PROG=	trpt
 MAN=	trpt.8
diff --git a/usr.sbin/tzsetup/Makefile b/usr.sbin/tzsetup/Makefile
index 5c80a48..5646b2a 100644
--- a/usr.sbin/tzsetup/Makefile
+++ b/usr.sbin/tzsetup/Makefile
@@ -10,7 +10,7 @@ WARNS?=	3
 DPADD=	${LIBDIALOG} ${LIBM}
 LDADD=	-ldialog -lm
 
-.include <bsd.own.mk>
+.include <src.opts.mk>
 
 .if ${MK_NCURSESW} == "no"
 DPADD+=	${LIBNCURSES}
diff --git a/usr.sbin/wpa/hostapd/Makefile b/usr.sbin/wpa/hostapd/Makefile
index 530cf02..7f0304c 100644
--- a/usr.sbin/wpa/hostapd/Makefile
+++ b/usr.sbin/wpa/hostapd/Makefile
@@ -1,5 +1,6 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
 .include "${.CURDIR}/../Makefile.inc"
 
 .PATH.c:${HOSTAPD_DISTDIR} \
diff --git a/usr.sbin/wpa/wpa_supplicant/Makefile b/usr.sbin/wpa/wpa_supplicant/Makefile
index 3424413..11cccf3 100644
--- a/usr.sbin/wpa/wpa_supplicant/Makefile
+++ b/usr.sbin/wpa/wpa_supplicant/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 .include "${.CURDIR}/../Makefile.inc"
 
 .PATH.c:${WPA_SUPPLICANT_DISTDIR} \