Allow bhyve and bhyveload to attach to tty devices.

bhyveload: introduce the -c <device> parameter
 to select a tty for output (or "stdio")

bhyve: allow the puc and lpc-com backends to
 accept a tty in addition to "stdio"

When used in conjunction with the null-modem device,
nmdm(4), this allows attach/detach to the guest console
and multiple concurrent serial ports. kgdb on a serial
port is now functional.

Reviewed by:	neel
Requested by:	Almost everyone that has used bhyve
MFC after:	10.0

3 files changed, 149 insertions, 58 deletions

diff --git a/usr.sbin/bhyve/uart_emul.c b/usr.sbin/bhyve/uart_emul.c
index 981607b..2fbbd15 100644
--- a/usr.sbin/bhyve/uart_emul.c
+++ b/usr.sbin/bhyve/uart_emul.c
@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 #include <stdio.h>
 #include <stdlib.h>
 #include <assert.h>
+#include <fcntl.h>
 #include <termios.h>
 #include <unistd.h>
 #include <stdbool.h>
@@ -67,6 +68,7 @@ __FBSDID("$FreeBSD$");
 #define	FIFOSZ	16
 
 static bool uart_stdio;		/* stdio in use for i/o */
+static struct termios tio_stdio_orig;
 
 static struct {
 	int	baseaddr;
@@ -87,6 +89,12 @@ struct fifo {
 	int	size;		/* size of the fifo */
 };
 
+struct ttyfd {
+	bool	opened;
+	int	fd;		/* tty device file descriptor */
+	struct termios tio_orig, tio_new;    /* I/O Terminals */
+};
+
 struct uart_softc {
 	pthread_mutex_t mtx;	/* protects all softc elements */
 	uint8_t	data;		/* Data register (R/W) */
@@ -103,8 +111,7 @@ struct uart_softc {
 
 	struct fifo rxfifo;
 
-	bool	opened;
-	bool	stdio;
+	struct ttyfd tty;
 	bool	thre_int_pending;	/* THRE interrupt pending */
 
 	void	*arg;
@@ -114,38 +121,41 @@ struct uart_softc {
 
 static void uart_drain(int fd, enum ev_type ev, void *arg);
 
-static struct termios tio_orig, tio_new;	/* I/O Terminals */
-
 static void
 ttyclose(void)
 {
 
-	tcsetattr(STDIN_FILENO, TCSANOW, &tio_orig);
+	tcsetattr(STDIN_FILENO, TCSANOW, &tio_stdio_orig);
 }
 
 static void
-ttyopen(void)
+ttyopen(struct ttyfd *tf)
 {
 
-	tcgetattr(STDIN_FILENO, &tio_orig);
+	tcgetattr(tf->fd, &tf->tio_orig);
 
-	cfmakeraw(&tio_new);
-	tcsetattr(STDIN_FILENO, TCSANOW, &tio_new);
+	tf->tio_new = tf->tio_orig;
+	cfmakeraw(&tf->tio_new);
+	tf->tio_new.c_cflag |= CLOCAL;
+	tcsetattr(tf->fd, TCSANOW, &tf->tio_new);
 
-	atexit(ttyclose);
+	if (tf->fd == STDIN_FILENO) {
+		tio_stdio_orig = tf->tio_orig;
+		atexit(ttyclose);
+	}
 }
 
 static bool
-tty_char_available(void)
+tty_char_available(struct ttyfd *tf)
 {
 	fd_set rfds;
 	struct timeval tv;
 
 	FD_ZERO(&rfds);
-	FD_SET(STDIN_FILENO, &rfds);
+	FD_SET(tf->fd, &rfds);
 	tv.tv_sec = 0;
 	tv.tv_usec = 0;
-	if (select(STDIN_FILENO + 1, &rfds, NULL, NULL, &tv) > 0 ) {
+	if (select(tf->fd + 1, &rfds, NULL, NULL, &tv) > 0 ) {
 		return (true);
 	} else {
 		return (false);
@@ -153,12 +163,12 @@ tty_char_available(void)
 }
 
 static int
-ttyread(void)
+ttyread(struct ttyfd *tf)
 {
 	char rb;
 
-	if (tty_char_available()) {
-		read(STDIN_FILENO, &rb, 1);
+	if (tty_char_available(tf)) {
+		read(tf->fd, &rb, 1);
 		return (rb & 0xff);
 	} else {
 		return (-1);
@@ -166,10 +176,10 @@ ttyread(void)
 }
 
 static void
-ttywrite(unsigned char wb)
+ttywrite(struct ttyfd *tf, unsigned char wb)
 {
 
-	(void)write(STDIN_FILENO, &wb, 1);
+	(void)write(tf->fd, &wb, 1);
 }
 
 static void
@@ -226,10 +236,8 @@ uart_opentty(struct uart_softc *sc)
 {
 	struct mevent *mev;
 
-	assert(!sc->opened && sc->stdio);
-
-	ttyopen();
-	mev = mevent_add(STDIN_FILENO, EVF_READ, uart_drain, sc);
+	ttyopen(&sc->tty);
+	mev = mevent_add(sc->tty.fd, EVF_READ, uart_drain, sc);
 	assert(mev);
 }
 
@@ -294,7 +302,7 @@ uart_drain(int fd, enum ev_type ev, void *arg)
 
 	sc = arg;	
 
-	assert(fd == STDIN_FILENO);
+	assert(fd == sc->tty.fd);
 	assert(ev == EVF_READ);
 	
 	/*
@@ -305,10 +313,10 @@ uart_drain(int fd, enum ev_type ev, void *arg)
 	pthread_mutex_lock(&sc->mtx);
 
 	if ((sc->mcr & MCR_LOOPBACK) != 0) {
-		(void) ttyread();
+		(void) ttyread(&sc->tty);
 	} else {
 		while (fifo_available(&sc->rxfifo) &&
-		       ((ch = ttyread()) != -1)) {
+		       ((ch = ttyread(&sc->tty)) != -1)) {
 			fifo_putchar(&sc->rxfifo, ch);
 		}
 		uart_toggle_intr(sc);
@@ -323,12 +331,6 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 	int fifosz;
 	uint8_t msr;
 
-	/* Open terminal */
-	if (!sc->opened && sc->stdio) {
-		uart_opentty(sc);
-		sc->opened = true;
-	}
-
 	pthread_mutex_lock(&sc->mtx);
 	
 	/*
@@ -351,8 +353,8 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 		if (sc->mcr & MCR_LOOPBACK) {
 			if (fifo_putchar(&sc->rxfifo, value) != 0)
 				sc->lsr |= LSR_OE;
-		} else if (sc->stdio) {
-			ttywrite(value);
+		} else if (sc->tty.opened) {
+			ttywrite(&sc->tty, value);
 		} /* else drop on floor */
 		sc->thre_int_pending = true;
 		break;
@@ -459,12 +461,6 @@ uart_read(struct uart_softc *sc, int offset)
 {
 	uint8_t iir, intr_reason, reg;
 
-	/* Open terminal */
-	if (!sc->opened && sc->stdio) {
-		uart_opentty(sc);
-		sc->opened = true;
-	}
-
 	pthread_mutex_lock(&sc->mtx);
 
 	/*
@@ -581,19 +577,47 @@ uart_init(uart_intr_func_t intr_assert, uart_intr_func_t intr_deassert,
 	return (sc);
 }
 
+static int
+uart_tty_backend(struct uart_softc *sc, const char *opts)
+{
+	int fd;
+	int retval;
+
+	retval = -1;
+
+	fd = open(opts, O_RDWR);
+	if (fd > 0 && isatty(fd)) {
+		sc->tty.fd = fd;
+		sc->tty.opened = true;
+		retval = 0;
+	}
+	    
+	return (retval);
+}
+
 int
 uart_set_backend(struct uart_softc *sc, const char *opts)
 {
-	/*
-	 * XXX one stdio backend supported at this time.
-	 */
+	int retval;
+
+	retval = -1;
+
 	if (opts == NULL)
 		return (0);
 
-	if (strcmp("stdio", opts) == 0 && !uart_stdio) {
-		sc->stdio = true;
-		uart_stdio = true;
-		return (0);
-	} else
-		return (-1);
+	if (strcmp("stdio", opts) == 0) {
+		if (!uart_stdio) {
+			sc->tty.fd = STDIN_FILENO;
+			sc->tty.opened = true;
+			uart_stdio = true;
+			retval = 0;
+		}
+	} else if (uart_tty_backend(sc, opts) == 0) {
+		retval = 0;
+	}
+
+	if (retval == 0)
+		uart_opentty(sc);
+
+	return (retval);
 }
diff --git a/usr.sbin/bhyveload/bhyveload.8 b/usr.sbin/bhyveload/bhyveload.8
index 00ce362..4e0160f 100644
--- a/usr.sbin/bhyveload/bhyveload.8
+++ b/usr.sbin/bhyveload/bhyveload.8
@@ -39,6 +39,7 @@ guest inside a bhyve virtual machine
 .Op Fl d Ar disk-path
 .Op Fl h Ar host-path
 .Op Fl e Ar name=value
+.Op Fl c Ar cons-dev
 .Ar vmname
 .Sh DESCRIPTION
 .Nm
@@ -100,6 +101,16 @@ to
 .Pp
 The option may be used more than once to set more than one environment
 variable.
+.It Fl c Ar cons-dev
+.Ar cons-dev
+is a
+.Xr tty 4
+device to use for
+.Nm
+terminal I/O.
+.Pp
+The text string "stdio" is also accepted and selects the use of
+unbuffered standard I/O. This is the default value.
 .El
 .Sh EXAMPLES
 To create a virtual machine named
@@ -109,10 +120,23 @@ that boots off the ISO image
 and has 1GB memory allocated to it:
 .Pp
 .Dl "bhyveload -m 1G -d /freebsd/release.iso freebsd-vm"
+.Pp
+To create a virtual machine named
+.Ar test-vm
+with 256MB of memory allocated, the guest root filesystem under the host
+directory
+.Pa /user/images/test
+and terminal I/O sent to the
+.Xr nmdm 4
+device
+.Pa /dev/nmdm1B
+.Pp
+.Dl "bhyveload -m 256MB -h /usr/images/test -c /dev/nmdm1B test-vm
 .Sh SEE ALSO
 .Xr bhyve 4 ,
 .Xr bhyve 8 ,
 .Xr loader 8 ,
+.Xr nmdm 4,
 .Xr vmm 4
 .Sh HISTORY
 .Nm
diff --git a/usr.sbin/bhyveload/bhyveload.c b/usr.sbin/bhyveload/bhyveload.c
index 1b63c58..c76bd0bc 100644
--- a/usr.sbin/bhyveload/bhyveload.c
+++ b/usr.sbin/bhyveload/bhyveload.c
@@ -91,6 +91,7 @@ __FBSDID("$FreeBSD$");
 static char *host_base = "/";
 static struct termios term, oldterm;
 static int disk_fd = -1;
+static int consin_fd, consout_fd;
 
 static char *vmname, *progname;
 static struct vmctx *ctx;
@@ -108,7 +109,7 @@ cb_putc(void *arg, int ch)
 {
 	char c = ch;
 
-	write(1, &c, 1);
+	(void) write(consout_fd, &c, 1);
 }
 
 static int
@@ -116,7 +117,7 @@ cb_getc(void *arg)
 {
 	char c;
 
-	if (read(0, &c, 1) == 1)
+	if (read(consin_fd, &c, 1) == 1)
 		return (c);
 	return (-1);
 }
@@ -126,7 +127,7 @@ cb_poll(void *arg)
 {
 	int n;
 
-	if (ioctl(0, FIONREAD, &n) >= 0)
+	if (ioctl(consin_fd, FIONREAD, &n) >= 0)
 		return (n > 0);
 	return (0);
 }
@@ -488,7 +489,7 @@ static void
 cb_exit(void *arg, int v)
 {
 
-	tcsetattr(0, TCSAFLUSH, &oldterm);
+	tcsetattr(consout_fd, TCSAFLUSH, &oldterm);
 	exit(v);
 }
 
@@ -564,13 +565,45 @@ static struct loader_callbacks cb = {
 	.getenv = cb_getenv,
 };
 
+static int
+altcons_open(char *path)
+{
+	struct stat sb;
+	int err;
+	int fd;
+
+	/*
+	 * Allow stdio to be passed in so that the same string
+	 * can be used for the bhyveload console and bhyve com-port
+	 * parameters
+	 */
+	if (!strcmp(path, "stdio"))
+		return (0);
+
+	err = stat(path, &sb);
+	if (err == 0) {
+		if (!S_ISCHR(sb.st_mode))
+			err = ENOTSUP;
+		else {
+			fd = open(path, O_RDWR | O_NONBLOCK);
+			if (fd < 0)
+				err = errno;
+			else
+				consin_fd = consout_fd = fd;
+		}
+	}
+
+	return (err);
+}
+
 static void
 usage(void)
 {
 
 	fprintf(stderr,
 	    "usage: %s [-m mem-size] [-d <disk-path>] [-h <host-path>]\n"
-	    "       %*s [-e <name=value>] <vmname>\n", progname,
+	    "       %*s [-e <name=value>] [-c <console-device>] <vmname>\n",
+	    progname,
 	    (int)strlen(progname), "");
 	exit(1);
 }
@@ -589,8 +622,16 @@ main(int argc, char** argv)
 	mem_size = 256 * MB;
 	disk_image = NULL;
 
-	while ((opt = getopt(argc, argv, "d:e:h:m:")) != -1) {
+	consin_fd = STDIN_FILENO;
+	consout_fd = STDOUT_FILENO;
+
+	while ((opt = getopt(argc, argv, "c:d:e:h:m:")) != -1) {
 		switch (opt) {
+		case 'c':
+			error = altcons_open(optarg);
+			if (error != 0)
+				errx(EX_USAGE, "Could not open '%s'", optarg);
+			break;
 		case 'd':
 			disk_image = optarg;
 			break;
@@ -640,11 +681,13 @@ main(int argc, char** argv)
 		exit(1);
 	}
 
-	tcgetattr(0, &term);
+	tcgetattr(consout_fd, &term);
 	oldterm = term;
-	term.c_lflag &= ~(ICANON|ECHO);
-	term.c_iflag &= ~ICRNL;
-	tcsetattr(0, TCSAFLUSH, &term);
+	cfmakeraw(&term);
+	term.c_cflag |= CLOCAL;
+	
+	tcsetattr(consout_fd, TCSAFLUSH, &term);
+
 	h = dlopen("/boot/userboot.so", RTLD_LOCAL);
 	if (!h) {
 		printf("%s\n", dlerror());