Major update to the way synchronization is done in the kernel. Highlights

include:

* Mutual exclusion is used instead of spl*().  See mutex(9).  (Note: The
  alpha port is still in transition and currently uses both.)

* Per-CPU idle processes.

* Interrupts are run in their own separate kernel threads and can be
  preempted (i386 only).

Partially contributed by:	BSDi (BSD/OS)
Submissions by (at least):	cp, dfr, dillon, grog, jake, jhb, sheldonh

1 files changed, 115 insertions, 40 deletions

diff --git a/sys/isa/sio.c b/sys/isa/sio.c
index 2725a20..a6f05e7 100644
--- a/sys/isa/sio.c
+++ b/sys/isa/sio.c
@@ -95,16 +95,12 @@
 #endif
 #include <isa/ic/ns16550.h>
 
+/* XXX - this is ok because we only do sio fast interrupts on i386 */
 #ifndef __i386__
 #define disable_intr()
 #define enable_intr()
 #endif
 
-#ifdef SMP
-#define disable_intr()	COM_DISABLE_INTR()
-#define enable_intr()	COM_ENABLE_INTR()
-#endif /* SMP */
-
 #define	LOTS_OF_EVENTS	64	/* helps separate urgent events from input */
 
 #define	CALLOUT_MASK		0x80
@@ -760,6 +756,7 @@ sioprobe(dev, xrid)
 	u_int		flags = device_get_flags(dev);
 	int		rid;
 	struct resource *port;
+	int		intrsave;
 
 	rid = xrid;
 	port = bus_alloc_resource(dev, SYS_RES_IOPORT, &rid,
@@ -856,7 +853,9 @@ sioprobe(dev, xrid)
 	 * but mask them in the processor as well in case there are some
 	 * (misconfigured) shared interrupts.
 	 */
+	intrsave = save_intr();
 	disable_intr();
+	COM_LOCK();
 /* EXTRA DELAY? */
 
 	/*
@@ -953,7 +952,8 @@ sioprobe(dev, xrid)
 			CLR_FLAG(dev, COM_C_IIR_TXRDYBUG);
 		}
 		sio_setreg(com, com_cfcr, CFCR_8BITS);
-		enable_intr();
+		COM_UNLOCK();
+		restore_intr(intrsave);
 		bus_release_resource(dev, SYS_RES_IOPORT, rid, port);
 		return (iobase == siocniobase ? 0 : result);
 	}
@@ -993,7 +993,8 @@ sioprobe(dev, xrid)
 	irqmap[3] = isa_irq_pending();
 	failures[9] = (sio_getreg(com, com_iir) & IIR_IMASK) - IIR_NOPEND;
 
-	enable_intr();
+	COM_UNLOCK();
+	restore_intr(intrsave);
 
 	irqs = irqmap[1] & ~irqmap[0];
 	if (bus_get_resource(idev, SYS_RES_IRQ, 0, &xirq, NULL) == 0 &&
@@ -1181,7 +1182,6 @@ sioattach(dev, xrid)
 	} else
 		com->it_in.c_ispeed = com->it_in.c_ospeed = TTYDEF_SPEED;
 	if (siosetwater(com, com->it_in.c_ispeed) != 0) {
-		enable_intr();
 		/*
 		 * Leave i/o resources allocated if this is a `cn'-level
 		 * console, so that other devices can't snarf them.
@@ -1190,7 +1190,6 @@ sioattach(dev, xrid)
 			bus_release_resource(dev, SYS_RES_IOPORT, rid, port);
 		return (ENOMEM);
 	}
-	enable_intr();
 	termioschars(&com->it_in);
 	com->it_out = com->it_in;
 
@@ -1340,7 +1339,7 @@ determined_type: ;
 	    RF_ACTIVE);
 	if (com->irqres) {
 		ret = BUS_SETUP_INTR(device_get_parent(dev), dev, com->irqres,
-				     INTR_TYPE_TTY | INTR_TYPE_FAST,
+				     INTR_TYPE_TTY | INTR_FAST,
 				     siointr, com, &com->cookie);
 		if (ret) {
 			ret = BUS_SETUP_INTR(device_get_parent(dev), dev,
@@ -1424,6 +1423,8 @@ open_top:
 			goto out;
 		}
 	} else {
+		int	intrsave;
+
 		/*
 		 * The device isn't open, so there are no conflicts.
 		 * Initialize it.  Initialization is done twice in many
@@ -1483,7 +1484,9 @@ open_top:
 			}
 		}
 
+		intrsave = save_intr();
 		disable_intr();
+		COM_LOCK();
 		(void) inb(com->line_status_port);
 		(void) inb(com->data_port);
 		com->prev_modem_status = com->last_modem_status
@@ -1495,7 +1498,8 @@ open_top:
 			outb(com->intr_ctl_port, IER_ERXRDY | IER_ETXRDY
 						| IER_ERLS | IER_EMSC);
 		}
-		enable_intr();
+		COM_UNLOCK();
+		restore_intr(intrsave);
 		/*
 		 * Handle initial DCD.  Callout devices get a fake initial
 		 * DCD (trapdoor DCD).  If we are callout, then any sleeping
@@ -1716,6 +1720,9 @@ siodtrwakeup(chan)
 	wakeup(&com->dtr_wait);
 }
 
+/*
+ * Call this function with COM_LOCK.  It will return with the lock still held.
+ */
 static void
 sioinput(com)
 	struct com_s	*com;
@@ -1725,6 +1732,7 @@ sioinput(com)
 	u_char		line_status;
 	int		recv_data;
 	struct tty	*tp;
+	int		intrsave;
 
 	buf = com->ibuf;
 	tp = com->tp;
@@ -1742,6 +1750,13 @@ sioinput(com)
 		 * call overhead).
 		 */
 		do {
+			/*
+			 * This may look odd, but it is using save-and-enable
+			 * semantics instead of the save-and-disable semantics
+			 * that are used everywhere else.
+			 */
+			intrsave = save_intr();
+			COM_UNLOCK();
 			enable_intr();
 			incc = com->iptr - buf;
 			if (tp->t_rawq.c_cc + incc > tp->t_ihiwat
@@ -1763,10 +1778,18 @@ sioinput(com)
 				tp->t_lflag &= ~FLUSHO;
 				comstart(tp);
 			}
-			disable_intr();
+			restore_intr(intrsave);
+			COM_LOCK();
 		} while (buf < com->iptr);
 	} else {
 		do {
+			/*
+			 * This may look odd, but it is using save-and-enable
+			 * semantics instead of the save-and-disable semantics
+			 * that are used everywhere else.
+			 */
+			intrsave = save_intr();
+			COM_UNLOCK();
 			enable_intr();
 			line_status = buf[com->ierroff];
 			recv_data = *buf++;
@@ -1782,7 +1805,8 @@ sioinput(com)
 					recv_data |= TTY_PE;
 			}
 			(*linesw[tp->t_line].l_rint)(recv_data, tp);
-			disable_intr();
+			restore_intr(intrsave);
+			COM_LOCK();
 		} while (buf < com->iptr);
 	}
 	com_events -= (com->iptr - com->ibuf);
@@ -1893,12 +1917,16 @@ siointr1(com)
 				if (recv_data == KEY_CR) {
 					brk_state1 = recv_data;
 					brk_state2 = 0;
-				} else if (brk_state1 == KEY_CR && (recv_data == KEY_TILDE || recv_data == KEY_CRTLB)) {
+				} else if (brk_state1 == KEY_CR
+					   && (recv_data == KEY_TILDE
+					       || recv_data == KEY_CRTLB)) {
 					if (recv_data == KEY_TILDE)
 						brk_state2 = recv_data;
-					else if (brk_state2 == KEY_TILDE && recv_data == KEY_CRTLB) {
+					else if (brk_state2 == KEY_TILDE
+						 && recv_data == KEY_CRTLB) {
 							breakpoint();
-							brk_state1 = brk_state2 = 0;
+							brk_state1 = 0;
+							brk_state2 = 0;
 							goto cont;
 					} else
 						brk_state2 = 0;
@@ -1949,7 +1977,10 @@ siointr1(com)
 				if (com->do_timestamp)
 					microtime(&com->timestamp);
 				++com_events;
+/* XXX - needs to go away when alpha gets ithreads */
+#ifdef __alpha__
 				schedsofttty();
+#endif
 #if 0 /* for testing input latency vs efficiency */
 if (com->iptr - com->ibuf == 8)
 	setsofttty();
@@ -2217,10 +2248,12 @@ sioioctl(dev, cmd, data, flag, p)
 	return (0);
 }
 
+/* software interrupt handler for SWI_TTY */
 static void
 siopoll()
 {
 	int		unit;
+	int		intrsave;
 
 	if (com_events == 0)
 		return;
@@ -2239,7 +2272,9 @@ repeat:
 			 * Discard any events related to never-opened or
 			 * going-away devices.
 			 */
+			intrsave = save_intr();
 			disable_intr();
+			COM_LOCK();
 			incc = com->iptr - com->ibuf;
 			com->iptr = com->ibuf;
 			if (com->state & CS_CHECKMSR) {
@@ -2247,33 +2282,43 @@ repeat:
 				com->state &= ~CS_CHECKMSR;
 			}
 			com_events -= incc;
-			enable_intr();
+			COM_UNLOCK();
+			restore_intr(intrsave);
 			continue;
 		}
 		if (com->iptr != com->ibuf) {
+			intrsave = save_intr();
 			disable_intr();
+			COM_LOCK();
 			sioinput(com);
-			enable_intr();
+			COM_UNLOCK();
+			restore_intr(intrsave);
 		}
 		if (com->state & CS_CHECKMSR) {
 			u_char	delta_modem_status;
 
+			intrsave = save_intr();
 			disable_intr();
+			COM_LOCK();
 			delta_modem_status = com->last_modem_status
 					     ^ com->prev_modem_status;
 			com->prev_modem_status = com->last_modem_status;
 			com_events -= LOTS_OF_EVENTS;
 			com->state &= ~CS_CHECKMSR;
-			enable_intr();
+			COM_UNLOCK();
+			restore_intr(intrsave);
 			if (delta_modem_status & MSR_DCD)
 				(*linesw[tp->t_line].l_modem)
 					(tp, com->prev_modem_status & MSR_DCD);
 		}
 		if (com->state & CS_ODONE) {
+			intrsave = save_intr();
 			disable_intr();
+			COM_LOCK();
 			com_events -= LOTS_OF_EVENTS;
 			com->state &= ~CS_ODONE;
-			enable_intr();
+			COM_UNLOCK();
+			restore_intr(intrsave);
 			if (!(com->state & CS_BUSY)
 			    && !(com->extra_state & CSE_BUSYCHECK)) {
 				timeout(siobusycheck, com, hz / 100);
@@ -2301,6 +2346,7 @@ comparam(tp, t)
 	u_char		dlbl;
 	int		s;
 	int		unit;
+	int		intrsave;
 
 	/* do historical conversions */
 	if (t->c_ispeed == 0)
@@ -2367,11 +2413,10 @@ comparam(tp, t)
 		sio_setreg(com, com_fifo, com->fifo_image);
 	}
 
-	/*
-	 * This returns with interrupts disabled so that we can complete
-	 * the speed change atomically.  Keeping interrupts disabled is
-	 * especially important while com_data is hidden.
-	 */
+	intrsave = save_intr();
+	disable_intr();
+	COM_LOCK();
+
 	(void) siosetwater(com, t->c_ispeed);
 
 	if (divisor != 0) {
@@ -2459,7 +2504,8 @@ comparam(tp, t)
 	if (com->state >= (CS_BUSY | CS_TTGO))
 		siointr1(com);
 
-	enable_intr();
+	COM_UNLOCK();
+	restore_intr(intrsave);
 	splx(s);
 	comstart(tp);
 	if (com->ibufold != NULL) {
@@ -2478,6 +2524,7 @@ siosetwater(com, speed)
 	u_char		*ibuf;
 	int		ibufsize;
 	struct tty	*tp;
+	int		intrsave;
 
 	/*
 	 * Make the buffer size large enough to handle a softtty interrupt
@@ -2488,20 +2535,16 @@ siosetwater(com, speed)
 	cp4ticks = speed / 10 / hz * 4;
 	for (ibufsize = 128; ibufsize < cp4ticks;)
 		ibufsize <<= 1;
-	if (ibufsize == com->ibufsize) {
-		disable_intr();
+	if (ibufsize == com->ibufsize)
 		return (0);
-	}
 
 	/*
 	 * Allocate input buffer.  The extra factor of 2 in the size is
 	 * to allow for an error byte for each input byte.
 	 */
 	ibuf = malloc(2 * ibufsize, M_DEVBUF, M_NOWAIT);
-	if (ibuf == NULL) {
-		disable_intr();
+	if (ibuf == NULL)
 		return (ENOMEM);
-	}
 
 	/* Initialize non-critical variables. */
 	com->ibufold = com->ibuf;
@@ -2517,7 +2560,9 @@ siosetwater(com, speed)
 	 * Read current input buffer, if any.  Continue with interrupts
 	 * disabled.
 	 */
+	intrsave = save_intr();
 	disable_intr();
+	COM_LOCK();
 	if (com->iptr != com->ibuf)
 		sioinput(com);
 
@@ -2536,6 +2581,8 @@ siosetwater(com, speed)
 	com->ibufend = ibuf + ibufsize;
 	com->ierroff = ibufsize;
 	com->ihighwater = ibuf + 3 * ibufsize / 4;
+	COM_UNLOCK();
+	restore_intr(intrsave);
 	return (0);
 }
 
@@ -2546,13 +2593,16 @@ comstart(tp)
 	struct com_s	*com;
 	int		s;
 	int		unit;
+	int		intrsave;
 
 	unit = DEV_TO_UNIT(tp->t_dev);
 	com = com_addr(unit);
 	if (com == NULL)
 		return;
 	s = spltty();
+	intrsave = save_intr();
 	disable_intr();
+	COM_LOCK();
 	if (tp->t_state & TS_TTSTOP)
 		com->state &= ~CS_TTGO;
 	else
@@ -2565,7 +2615,8 @@ comstart(tp)
 		    && com->state & CS_RTS_IFLOW)
 			outb(com->modem_ctl_port, com->mcr_image |= MCR_RTS);
 	}
-	enable_intr();
+	COM_UNLOCK();
+	restore_intr(intrsave);
 	if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) {
 		ttwwakeup(tp);
 		splx(s);
@@ -2581,7 +2632,9 @@ comstart(tp)
 						  sizeof com->obuf1);
 			com->obufs[0].l_next = NULL;
 			com->obufs[0].l_queued = TRUE;
+			intrsave = save_intr();
 			disable_intr();
+			COM_LOCK();
 			if (com->state & CS_BUSY) {
 				qp = com->obufq.l_next;
 				while ((next = qp->l_next) != NULL)
@@ -2593,7 +2646,8 @@ comstart(tp)
 				com->obufq.l_next = &com->obufs[0];
 				com->state |= CS_BUSY;
 			}
-			enable_intr();
+			COM_UNLOCK();
+			restore_intr(intrsave);
 		}
 		if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) {
 			com->obufs[1].l_tail
@@ -2601,7 +2655,9 @@ comstart(tp)
 						  sizeof com->obuf2);
 			com->obufs[1].l_next = NULL;
 			com->obufs[1].l_queued = TRUE;
+			intrsave = save_intr();
 			disable_intr();
+			COM_LOCK();
 			if (com->state & CS_BUSY) {
 				qp = com->obufq.l_next;
 				while ((next = qp->l_next) != NULL)
@@ -2613,14 +2669,18 @@ comstart(tp)
 				com->obufq.l_next = &com->obufs[1];
 				com->state |= CS_BUSY;
 			}
-			enable_intr();
+			COM_UNLOCK();
+			restore_intr(intrsave);
 		}
 		tp->t_state |= TS_BUSY;
 	}
+	intrsave = save_intr();
 	disable_intr();
+	COM_LOCK();
 	if (com->state >= (CS_BUSY | CS_TTGO))
 		siointr1(com);	/* fake interrupt to start output */
-	enable_intr();
+	COM_UNLOCK();
+	restore_intr(intrsave);
 	ttwwakeup(tp);
 	splx(s);
 }
@@ -2631,11 +2691,14 @@ comstop(tp, rw)
 	int		rw;
 {
 	struct com_s	*com;
+	int		intrsave;
 
 	com = com_addr(DEV_TO_UNIT(tp->t_dev));
 	if (com == NULL || com->gone)
 		return;
+	intrsave = save_intr();
 	disable_intr();
+	COM_LOCK();
 	if (rw & FWRITE) {
 		if (com->hasfifo)
 #ifdef COM_ESP
@@ -2662,7 +2725,8 @@ comstop(tp, rw)
 		com_events -= (com->iptr - com->ibuf);
 		com->iptr = com->ibuf;
 	}
-	enable_intr();
+	COM_UNLOCK();
+	restore_intr(intrsave);
 	comstart(tp);
 }
 
@@ -2674,6 +2738,7 @@ commctl(com, bits, how)
 {
 	int	mcr;
 	int	msr;
+	int	intrsave;
 
 	if (how == DMGET) {
 		bits = TIOCM_LE;	/* XXX - always enabled while open */
@@ -2705,7 +2770,9 @@ commctl(com, bits, how)
 		mcr |= MCR_RTS;
 	if (com->gone)
 		return(0);
+	intrsave = save_intr();
 	disable_intr();
+	COM_LOCK();
 	switch (how) {
 	case DMSET:
 		outb(com->modem_ctl_port,
@@ -2718,7 +2785,8 @@ commctl(com, bits, how)
 		outb(com->modem_ctl_port, com->mcr_image &= ~mcr);
 		break;
 	}
-	enable_intr();
+	COM_UNLOCK();
+	restore_intr(intrsave);
 	return (0);
 }
 
@@ -2766,6 +2834,7 @@ comwakeup(chan)
 {
 	struct com_s	*com;
 	int		unit;
+	int		intrsave;
 
 	sio_timeout_handle = timeout(comwakeup, (void *)NULL, sio_timeout);
 
@@ -2777,9 +2846,12 @@ comwakeup(chan)
 		com = com_addr(unit);
 		if (com != NULL && !com->gone
 		    && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) {
+			intrsave = save_intr();
 			disable_intr();
+			COM_LOCK();
 			siointr1(com);
-			enable_intr();
+			COM_UNLOCK();
+			restore_intr(intrsave);
 		}
 	}
 
@@ -2801,10 +2873,13 @@ comwakeup(chan)
 			u_int	delta;
 			u_long	total;
 
+			intrsave = save_intr();
 			disable_intr();
+			COM_LOCK();
 			delta = com->delta_error_counts[errnum];
 			com->delta_error_counts[errnum] = 0;
-			enable_intr();
+			COM_UNLOCK();
+			restore_intr(intrsave);
 			if (delta == 0)
 				continue;
 			total = com->error_counts[errnum] += delta;