Preperation for moving cpl into critical region access.

Several new fine-grained locks.
New FAST_INTR() methods:
 - separate simplelock for FAST_INTR, no more giant lock.
 - FAST_INTR()s no longer checks ipending on way out of ISR.
sio made MP-safe (I hope).

5 files changed, 154 insertions, 18 deletions

diff --git a/sys/i386/isa/apic_ipl.s b/sys/i386/isa/apic_ipl.s
index b6f34e1..1412367 100644
--- a/sys/i386/isa/apic_ipl.s
+++ b/sys/i386/isa/apic_ipl.s
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: apic_ipl.s,v 1.10 1997/07/24 23:49:44 fsmp Exp $
+ *	$Id: apic_ipl.s,v 1.11 1997/07/31 17:28:56 fsmp Exp $
  */
 
 
@@ -66,11 +66,29 @@ _vec:
 	.long	 vec8,  vec9, vec10, vec11, vec12, vec13, vec14, vec15
 	.long	vec16, vec17, vec18, vec19, vec20, vec21, vec22, vec23
 
+/* various simple locks */
 	.align 2				/* MUST be 32bit aligned */
+
+/* critical region around IO APIC */
 	.globl _imen_lock
 _imen_lock:
 	.long	0
 
+/* critical region around spl & cpl */
+	.globl _cpl_lock
+_cpl_lock:
+	.long	0
+
+/* critical region around FAST_INTR() routines */
+	.globl _fast_intr_lock
+_fast_intr_lock:
+	.long	0
+
+/* critical region around INTR() routines */
+	.globl _intr_lock
+_intr_lock:
+	.long	0
+
 /*
  * Note:
  *	This is the UP equivilant of _imen.
diff --git a/sys/i386/isa/apic_vector.s b/sys/i386/isa/apic_vector.s
index f73ddc5..f2a8bce 100644
--- a/sys/i386/isa/apic_vector.s
+++ b/sys/i386/isa/apic_vector.s
@@ -1,6 +1,6 @@
 /*
  *	from: vector.s, 386BSD 0.1 unknown origin
- *	$Id: apic_vector.s,v 1.21 1997/08/10 20:47:53 smp Exp smp $
+ *	$Id: apic_vector.s,v 1.15 1997/08/10 20:58:57 fsmp Exp $
  */
 
 
@@ -11,12 +11,41 @@
 #include "i386/isa/intr_machdep.h"
 
 
+#ifdef FAST_SIMPLELOCK
+
+#define GET_FAST_INTR_LOCK						\
+	pushl	$_fast_intr_lock ;		/* address of lock */	\
+	call	_s_lock ;			/* MP-safe */		\
+	addl	$4,%esp
+
+#define REL_FAST_INTR_LOCK						\
+	pushl	$_fast_intr_lock ;		/* address of lock */	\
+	call	_s_unlock ;			/* MP-safe */		\
+	addl	$4,%esp
+
+#else /* FAST_SIMPLELOCK */
+
+#define GET_FAST_INTR_LOCK						\
+	call	_get_isrlock
+#define REL_FAST_INTR_LOCK						\
+	pushl	$_mp_lock ;	/* GIANT_LOCK */			\
+	call	_MPrellock ;						\
+	add	$4, %esp
+
+#endif /* FAST_SIMPLELOCK */
+
+#define REL_ISR_LOCK							\
+	pushl	$_mp_lock ;	/* GIANT_LOCK */			\
+	call	_MPrellock ;						\
+	add	$4, %esp
+
 /* convert an absolute IRQ# into a bitmask */
 #define IRQ_BIT(irq_num)	(1 << (irq_num))
 
 /* make an index into the IO APIC from the IRQ# */
 #define REDTBL_IDX(irq_num)	(0x10 + ((irq_num) * 2))
 
+
 /*
  * 'lazy masking' code suggested by Bruce Evans <bde@zeta.org.au>
  */
@@ -115,6 +144,8 @@
  * Macros for interrupt interrupt entry, call to handler, and exit.
  */
 
+#ifdef FAST_WITHOUTCPL
+
 #define	FAST_INTR(irq_num, vec_name)					\
 	.text ;								\
 	SUPERALIGN_TEXT ;						\
@@ -128,11 +159,45 @@ IDTVEC(vec_name) ;							\
 	movl	%ax,%ds ;						\
 	MAYBE_MOVW_AX_ES ;						\
 	FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ;			\
-	call	_get_isrlock ;						\
+	GET_FAST_INTR_LOCK ;						\
 	pushl	_intr_unit + (irq_num) * 4 ;				\
 	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
+	addl	$4, %esp ;						\
 	movl	$0, lapic_eoi ;						\
+	lock ; 								\
+	incl	_cnt+V_INTR ;	/* book-keeping can wait */		\
+	movl	_intr_countp + (irq_num) * 4, %eax ;			\
+	lock ; 								\
+	incl	(%eax) ;						\
+	MEXITCOUNT ;							\
+	REL_FAST_INTR_LOCK ;						\
+	MAYBE_POPL_ES ;							\
+	popl	%ds ;							\
+	popl	%edx ;							\
+	popl	%ecx ;							\
+	popl	%eax ;							\
+	iret
+
+#else
+
+#define	FAST_INTR(irq_num, vec_name)					\
+	.text ;								\
+	SUPERALIGN_TEXT ;						\
+IDTVEC(vec_name) ;							\
+	pushl	%eax ;		/* save only call-used registers */	\
+	pushl	%ecx ;							\
+	pushl	%edx ;							\
+	pushl	%ds ;							\
+	MAYBE_PUSHL_ES ;						\
+	movl	$KDSEL,%eax ;						\
+	movl	%ax,%ds ;						\
+	MAYBE_MOVW_AX_ES ;						\
+	FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ;			\
+	GET_FAST_INTR_LOCK ;						\
+	pushl	_intr_unit + (irq_num) * 4 ;				\
+	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
 	addl	$4,%esp ;						\
+	movl	$0, lapic_eoi ;						\
 	incl	_cnt+V_INTR ;	/* book-keeping can wait */		\
 	movl	_intr_countp + (irq_num) * 4,%eax ;			\
 	incl	(%eax) ;						\
@@ -142,9 +207,7 @@ IDTVEC(vec_name) ;							\
 	jne	2f ; 		/* yes, maybe handle them */		\
 1: ;									\
 	MEXITCOUNT ;							\
-	pushl	$_mp_lock ;	/* GIANT_LOCK */			\
-	call	_MPrellock ;						\
-	add	$4, %esp ;						\
+	REL_FAST_INTR_LOCK ;						\
 	MAYBE_POPL_ES ;							\
 	popl	%ds ;							\
 	popl	%edx ;							\
@@ -178,6 +241,9 @@ IDTVEC(vec_name) ;							\
 	MEXITCOUNT ;							\
 	jmp	_doreti
 
+#endif /** FAST_WITHOUTCPL */
+
+
 #define	INTR(irq_num, vec_name)						\
 	.text ;								\
 	SUPERALIGN_TEXT ;						\
@@ -217,9 +283,7 @@ __CONCAT(Xresume,irq_num): ;						\
 	/* XXX skip mcounting here to avoid double count */		\
 	lock ;					/* MP-safe */		\
 	orl	$IRQ_BIT(irq_num), _ipending ;				\
-	pushl	$_mp_lock ;			/* GIANT_LOCK */	\
-	call	_MPrellock ;						\
-	add	$4, %esp ;						\
+	REL_ISR_LOCK ;							\
 	popl	%es ;							\
 	popl	%ds ;							\
 	popal ;								\
diff --git a/sys/i386/isa/ipl.s b/sys/i386/isa/ipl.s
index 002e924..3b2bf73 100644
--- a/sys/i386/isa/ipl.s
+++ b/sys/i386/isa/ipl.s
@@ -36,7 +36,7 @@
  *
  *	@(#)ipl.s
  *
- *	$Id: ipl.s,v 1.7 1997/08/10 20:58:58 fsmp Exp $
+ *	$Id: ipl.s,v 1.8 1997/08/10 21:17:55 fsmp Exp $
  */
 
 
@@ -182,8 +182,7 @@ doreti_unpend:
 	 */
 	sti
 	bsfl	%ecx,%ecx		/* slow, but not worth optimizing */
-	MP_INSTR_LOCK
-	btrl	%ecx,_ipending
+	MPLOCKED btrl %ecx, _ipending
 	jnc	doreti_next		/* some intr cleared memory copy */
 	movl	ihandlers(,%ecx,4),%edx
 	testl	%edx,%edx
@@ -244,8 +243,7 @@ swi_ast_phantom:
 	 * using by using cli, but they are unavoidable for lcall entries.
 	 */
 	cli
-	MP_INSTR_LOCK
-	orl	$SWI_AST_PENDING,_ipending
+	MPLOCKED orl $SWI_AST_PENDING, _ipending
 	subl	%eax,%eax
 	jmp	doreti_exit	/* SWI_AST is highest so we must be done */
 
@@ -283,8 +281,7 @@ splz_next:
 	ALIGN_TEXT
 splz_unpend:
 	bsfl	%ecx,%ecx
-	MP_INSTR_LOCK
-	btrl	%ecx,_ipending
+	MPLOCKED btrl %ecx, _ipending
 	jnc	splz_next
 	movl	ihandlers(,%ecx,4),%edx
 	testl	%edx,%edx
diff --git a/sys/i386/isa/ipl_funcs.c b/sys/i386/isa/ipl_funcs.c
index fa3136a..c0849cd 100644
--- a/sys/i386/isa/ipl_funcs.c
+++ b/sys/i386/isa/ipl_funcs.c
@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id$
+ *	$Id: ipl_funcs.c,v 1.1 1997/05/31 08:57:05 peter Exp $
  */
 
 #include <sys/types.h>
@@ -92,3 +92,13 @@ splx(unsigned ipl)
 	if (ipending & ~ipl)
 		splz();
 }
+
+#ifdef SMP
+intrmask_t
+splq(intrmask_t mask)
+{
+	intrmask_t tmp = cpl;
+	cpl |= mask;
+	return (tmp);
+}
+#endif
diff --git a/sys/i386/isa/sio.c b/sys/i386/isa/sio.c
index 55944b2..5249698 100644
--- a/sys/i386/isa/sio.c
+++ b/sys/i386/isa/sio.c
@@ -31,7 +31,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)com.c	7.5 (Berkeley) 5/16/91
- *	$Id: sio.c,v 1.1 1997/08/04 17:58:18 smp Exp smp $
+ *	$Id: sio.c,v 1.174 1997/08/04 19:12:54 fsmp Exp $
  */
 
 #include "opt_comconsole.h"
@@ -82,6 +82,15 @@
 #include <pccard/slot.h>
 #endif
 
+#ifdef SMP
+#include <machine/smp.h>
+#define COM_LOCK() 	s_lock(&com_lock)
+#define COM_UNLOCK() 	s_unlock(&com_lock)
+#else
+#define COM_LOCK()
+#define COM_UNLOCK()
+#endif /* SMP */
+
 #ifdef APIC_IO
 /*
  * INTs are masked in the (global) IO APIC,
@@ -559,9 +568,11 @@ static int
 card_intr(struct pccard_dev *dp)
 {
 	struct com_s	*com;
+	COM_LOCK();
 	com = com_addr(dp->isahd.id_unit);
 	if (com && !com_addr(dp->isahd.id_unit)->gone)
 		siointr1(com_addr(dp->isahd.id_unit));
+	COM_UNLOCK();
 	return(1);
 }
 #endif /* NCRD > 0 */
@@ -1183,12 +1194,14 @@ open_top:
 		}
 
 		disable_intr();
+		COM_LOCK();
 		(void) inb(com->line_status_port);
 		(void) inb(com->data_port);
 		com->prev_modem_status = com->last_modem_status
 		    = inb(com->modem_status_port);
 		outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS
 				       | IER_EMSC);
+		COM_UNLOCK();
 		enable_intr();
 		/*
 		 * Handle initial DCD.  Callout devices get a fake initial
@@ -1417,7 +1430,9 @@ siointr(unit)
 	int	unit;
 {
 #ifndef COM_MULTIPORT
+	COM_LOCK();
 	siointr1(com_addr(unit));
+	COM_UNLOCK();
 #else /* COM_MULTIPORT */
 	struct com_s    *com;
 	bool_t		possibly_more_intrs;
@@ -1429,6 +1444,7 @@ siointr(unit)
 	 * devices, then the edge from one may be lost because another is
 	 * on.
 	 */
+	COM_LOCK();
 	do {
 		possibly_more_intrs = FALSE;
 		for (unit = 0; unit < NSIO; ++unit) {
@@ -1442,6 +1458,7 @@ siointr(unit)
 			}
 		}
 	} while (possibly_more_intrs);
+	COM_UNLOCK();
 #endif /* COM_MULTIPORT */
 }
 
@@ -1821,6 +1838,7 @@ repeat:
 			 * loop.
 			 */
 			disable_intr();
+			COM_LOCK();
 			incc = com->iptr - com->ibuf;
 			com->iptr = com->ibuf;
 			if (com->state & CS_CHECKMSR) {
@@ -1828,6 +1846,7 @@ repeat:
 				com->state &= ~CS_CHECKMSR;
 			}
 			com_events -= incc;
+			COM_UNLOCK();
 			enable_intr();
 			if (incc != 0)
 				log(LOG_DEBUG,
@@ -1843,6 +1862,7 @@ repeat:
 		} else {
 			buf = ibuf;
 			disable_intr();
+			COM_LOCK();
 			incc = com->iptr - buf;
 			com_events -= incc;
 			if (ibuf == com->ibuf1)
@@ -1863,6 +1883,7 @@ repeat:
 			    && !(tp->t_state & TS_TBLOCK))
 				outb(com->modem_ctl_port,
 				     com->mcr_image |= MCR_RTS);
+			COM_UNLOCK();
 			enable_intr();
 			com->ibuf = ibuf;
 		}
@@ -1871,11 +1892,13 @@ repeat:
 			u_char	delta_modem_status;
 
 			disable_intr();
+			COM_LOCK();
 			delta_modem_status = com->last_modem_status
 					     ^ com->prev_modem_status;
 			com->prev_modem_status = com->last_modem_status;
 			com_events -= LOTS_OF_EVENTS;
 			com->state &= ~CS_CHECKMSR;
+			COM_UNLOCK();
 			enable_intr();
 			if (delta_modem_status & MSR_DCD)
 				(*linesw[tp->t_line].l_modem)
@@ -1883,8 +1906,10 @@ repeat:
 		}
 		if (com->state & CS_ODONE) {
 			disable_intr();
+			COM_LOCK();
 			com_events -= LOTS_OF_EVENTS;
 			com->state &= ~CS_ODONE;
+			COM_UNLOCK();
 			enable_intr();
 			if (!(com->state & CS_BUSY)
 			    && !(com->extra_state & CSE_BUSYCHECK)) {
@@ -2040,9 +2065,11 @@ comparam(tp, t)
 	 * error bits, but that is acceptable here.
 	 */
 	disable_intr();
+	COM_LOCK();
 retry:
 	com->state &= ~CS_TTGO;
 	txtimeout = tp->t_timeout;
+	COM_UNLOCK();
 	enable_intr();
 	while ((inb(com->line_status_port) & (LSR_TSRE | LSR_TXRDY))
 	       != (LSR_TSRE | LSR_TXRDY)) {
@@ -2059,7 +2086,9 @@ retry:
 		if (error != 0 && error != EAGAIN) {
 			if (!(tp->t_state & TS_TTSTOP)) {
 				disable_intr();
+				COM_LOCK();
 				com->state |= CS_TTGO;
+				COM_UNLOCK();
 				enable_intr();
 			}
 			splx(s);
@@ -2068,6 +2097,7 @@ retry:
 	}
 
 	disable_intr();		/* very important while com_data is hidden */
+	COM_LOCK();
 
 	/*
 	 * XXX - clearing CS_TTGO is not sufficient to stop further output,
@@ -2163,6 +2193,7 @@ retry:
 	if (com->state >= (CS_BUSY | CS_TTGO))
 		siointr1(com);
 
+	COM_UNLOCK();
 	enable_intr();
 	splx(s);
 	comstart(tp);
@@ -2181,6 +2212,7 @@ comstart(tp)
 	com = com_addr(unit);
 	s = spltty();
 	disable_intr();
+	COM_LOCK();
 	if (tp->t_state & TS_TTSTOP)
 		com->state &= ~CS_TTGO;
 	else
@@ -2193,6 +2225,7 @@ comstart(tp)
 		    && com->state & CS_RTS_IFLOW)
 			outb(com->modem_ctl_port, com->mcr_image |= MCR_RTS);
 	}
+	COM_UNLOCK();
 	enable_intr();
 	if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) {
 		splx(s);
@@ -2209,6 +2242,7 @@ comstart(tp)
 			com->obufs[0].l_next = NULL;
 			com->obufs[0].l_queued = TRUE;
 			disable_intr();
+			COM_LOCK();
 			if (com->state & CS_BUSY) {
 				qp = com->obufq.l_next;
 				while ((next = qp->l_next) != NULL)
@@ -2220,6 +2254,7 @@ comstart(tp)
 				com->obufq.l_next = &com->obufs[0];
 				com->state |= CS_BUSY;
 			}
+			COM_UNLOCK();
 			enable_intr();
 		}
 		if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) {
@@ -2229,6 +2264,7 @@ comstart(tp)
 			com->obufs[1].l_next = NULL;
 			com->obufs[1].l_queued = TRUE;
 			disable_intr();
+			COM_LOCK();
 			if (com->state & CS_BUSY) {
 				qp = com->obufq.l_next;
 				while ((next = qp->l_next) != NULL)
@@ -2240,13 +2276,16 @@ comstart(tp)
 				com->obufq.l_next = &com->obufs[1];
 				com->state |= CS_BUSY;
 			}
+			COM_UNLOCK();
 			enable_intr();
 		}
 		tp->t_state |= TS_BUSY;
 	}
 	disable_intr();
+	COM_LOCK();
 	if (com->state >= (CS_BUSY | CS_TTGO))
 		siointr1(com);	/* fake interrupt to start output */
+	COM_UNLOCK();
 	enable_intr();
 	ttwwakeup(tp);
 	splx(s);
@@ -2263,6 +2302,7 @@ siostop(tp, rw)
 	if (com->gone)
 		return;
 	disable_intr();
+	COM_LOCK();
 	if (rw & FWRITE) {
 		if (com->hasfifo)
 #ifdef COM_ESP
@@ -2291,6 +2331,7 @@ siostop(tp, rw)
 		com_events -= (com->iptr - com->ibuf);
 		com->iptr = com->ibuf;
 	}
+	COM_UNLOCK();
 	enable_intr();
 	comstart(tp);
 }
@@ -2351,6 +2392,7 @@ commctl(com, bits, how)
 	if (com->gone)
 		return(0);
 	disable_intr();
+	COM_LOCK();
 	switch (how) {
 	case DMSET:
 		outb(com->modem_ctl_port,
@@ -2363,6 +2405,7 @@ commctl(com, bits, how)
 		outb(com->modem_ctl_port, com->mcr_image &= ~mcr);
 		break;
 	}
+	COM_UNLOCK();
 	enable_intr();
 	return (0);
 }
@@ -2422,7 +2465,9 @@ comwakeup(chan)
 		if (com != NULL && !com->gone
 		    && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) {
 			disable_intr();
+			COM_LOCK();
 			siointr1(com);
+			COM_UNLOCK();
 			enable_intr();
 		}
 	}
@@ -2446,8 +2491,10 @@ comwakeup(chan)
 			u_long	total;
 
 			disable_intr();
+			COM_LOCK();
 			delta = com->delta_error_counts[errnum];
 			com->delta_error_counts[errnum] = 0;
+			COM_UNLOCK();
 			enable_intr();
 			if (delta == 0)
 				continue;