Made PEND_INTS default.

Made NEW_STRATEGY default.
Removed misc. old cruft.

Centralized simple locks into mp_machdep.c
Centralized simple lock macros into param.h

More cleanup in the direction of making splxx()/cpl MP-safe.

16 files changed, 412 insertions, 580 deletions

diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s
index f2a8bce..9d1d4a0 100644
--- a/sys/i386/i386/apic_vector.s
+++ b/sys/i386/i386/apic_vector.s
@@ -1,12 +1,12 @@
 /*
  *	from: vector.s, 386BSD 0.1 unknown origin
- *	$Id: apic_vector.s,v 1.15 1997/08/10 20:58:57 fsmp Exp $
+ *	$Id: apic_vector.s,v 1.24 1997/08/21 04:52:30 smp Exp smp $
  */
 
 
 #include <machine/apic.h>
 #include <machine/smp.h>
-#include <machine/smptests.h>		/** PEND_INTS, various counters */
+#include <machine/smptests.h>			/** various things... */
 
 #include "i386/isa/intr_machdep.h"
 
@@ -27,6 +27,7 @@
 
 #define GET_FAST_INTR_LOCK						\
 	call	_get_isrlock
+
 #define REL_FAST_INTR_LOCK						\
 	pushl	$_mp_lock ;	/* GIANT_LOCK */			\
 	call	_MPrellock ;						\
@@ -34,11 +35,6 @@
 
 #endif /* FAST_SIMPLELOCK */
 
-#define REL_ISR_LOCK							\
-	pushl	$_mp_lock ;	/* GIANT_LOCK */			\
-	call	_MPrellock ;						\
-	add	$4, %esp
-
 /* convert an absolute IRQ# into a bitmask */
 #define IRQ_BIT(irq_num)	(1 << (irq_num))
 
@@ -47,105 +43,13 @@
 
 
 /*
- * 'lazy masking' code suggested by Bruce Evans <bde@zeta.org.au>
- */
-
-#ifdef PEND_INTS
-
-/*
- * the 1st version fails because masked edge-triggered INTs are lost
- * by the IO APIC.  This version tests to see whether we are handling
- * an edge or level triggered INT.  Level-triggered INTs must still be
- * masked as we don't clear the source, and the EOI cycle would allow
- * recursive INTs to occur.
- */
-#define MAYBE_MASK_IRQ(irq_num)						\
-	lock ;					/* MP-safe */		\
-	btsl	$(irq_num),iactive ;		/* lazy masking */	\
-	jc	6f ;				/* already active */	\
-	pushl	$_mp_lock ;			/* GIANT_LOCK */	\
-	call	_MPtrylock ;			/* try to get lock */	\
-	add $4,	%esp ;							\
-	testl	%eax, %eax ;			/* did we get it? */	\
-	jnz	8f ;				/* yes, enter kernel */	\
-6: ;						/* active or locked */	\
-	IMASK_LOCK ;				/* into critical reg */	\
-	testl	$IRQ_BIT(irq_num),_apic_pin_trigger ;			\
-	jz	7f ;				/* edge, don't mask */	\
-	orl	$IRQ_BIT(irq_num),_apic_imen ;	/* set the mask bit */	\
-	movl	_ioapic,%ecx ;			/* ioapic[0] addr */	\
-	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
-	orl	$IOART_INTMASK,%eax ;		/* set the mask */	\
-	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-7: ;									\
-	lock ;					/* MP-safe */		\
-	orl	$IRQ_BIT(irq_num), _ipending ;	/* set _ipending bit */	\
-	IMASK_UNLOCK ;				/* exit critical reg */	\
-	movl	$0, lapic_eoi ;			/* do the EOI */	\
-	popl	%es ;							\
-	popl	%ds ;							\
-	popal ;								\
-	addl	$4+4,%esp ;						\
-	iret ;								\
-;									\
-	ALIGN_TEXT ;							\
-8:
-
-#else /* PEND_INTS */
-
-#define MAYBE_MASK_IRQ(irq_num)						\
-	lock ;					/* MP-safe */		\
-	btsl	$(irq_num),iactive ;		/* lazy masking */	\
-	jnc	1f ;				/* NOT active */	\
-	IMASK_LOCK ;				/* enter critical reg */\
-	orl	$IRQ_BIT(irq_num),_apic_imen ;	/* set the mask bit */	\
-	movl	_ioapic,%ecx ;			/* ioapic[0]addr */	\
-	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
-	orl	$IOART_INTMASK,%eax ;		/* set the mask */	\
-	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-	lock ;					/* MP-safe */		\
-	orl	$IRQ_BIT(irq_num), _ipending ;	/* set _ipending bit */	\
-	movl	$0, lapic_eoi ;			/* do the EOI */	\
-	IMASK_UNLOCK ;				/* exit critical reg */	\
-	popl	%es ;							\
-	popl	%ds ;							\
-	popal ;								\
-	addl	$4+4,%esp ;						\
-	iret ;								\
-;									\
-	ALIGN_TEXT ;							\
-1: ;									\
-	call	_get_mplock			/* SMP Spin lock */
-
-#endif /* PEND_INTS */
-
-
-#define MAYBE_UNMASK_IRQ(irq_num)					\
-	cli ;	/* must unmask _apic_imen and IO APIC atomically */	\
-	lock ;					/* MP-safe */		\
-	andl	$~IRQ_BIT(irq_num),iactive ;				\
-	IMASK_LOCK ;				/* enter critical reg */\
-	testl	$IRQ_BIT(irq_num),_apic_imen ;				\
-	je	9f ;							\
-	andl	$~IRQ_BIT(irq_num),_apic_imen ;	/* clear mask bit */	\
-	movl	_ioapic,%ecx ;			/* ioapic[0]addr */	\
-	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
-	andl	$~IOART_INTMASK,%eax ;		/* clear the mask */	\
-	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-9: ;									\
-	IMASK_UNLOCK ;				/* exit critical reg */	\
-	sti	/* XXX _doreti repeats the cli/sti */
-
-
-/*
  * Macros for interrupt interrupt entry, call to handler, and exit.
  */
 
 #ifdef FAST_WITHOUTCPL
 
+/*
+ */
 #define	FAST_INTR(irq_num, vec_name)					\
 	.text ;								\
 	SUPERALIGN_TEXT ;						\
@@ -244,51 +148,122 @@ IDTVEC(vec_name) ;							\
 #endif /** FAST_WITHOUTCPL */
 
 
-#define	INTR(irq_num, vec_name)						\
-	.text ;								\
-	SUPERALIGN_TEXT ;						\
-IDTVEC(vec_name) ;							\
+/*
+ * 
+ */
+#define PUSH_FRAME							\
 	pushl	$0 ;		/* dummy error code */			\
 	pushl	$0 ;		/* dummy trap type */			\
 	pushal ;							\
 	pushl	%ds ;		/* save data and extra segments ... */	\
-	pushl	%es ;							\
-	movl	$KDSEL,%eax ;	/* ... and reload with kernel's ... */	\
-	movl	%ax,%ds ;	/* ... early for obsolete reasons */	\
-	movl	%ax,%es ;						\
-	MAYBE_MASK_IRQ(irq_num) ;					\
-	movl	$0, lapic_eoi ;						\
-	movl	_cpl,%eax ;						\
-	testl	$IRQ_BIT(irq_num), %eax ;				\
-	jne	3f ;							\
+	pushl	%es
+
+#define POP_FRAME							\
+	popl	%es ;							\
+	popl	%ds ;							\
+	popal ;								\
+	addl	$4+4,%esp
+
+/*
+ * Test to see whether we are handling an edge or level triggered INT.
+ *  Level-triggered INTs must still be masked as we don't clear the source,
+ *  and the EOI cycle would cause redundant INTs to occur.
+ */
+#define MASK_LEVEL_IRQ(irq_num)						\
+	IMASK_LOCK ;				/* into critical reg */	\
+	testl	$IRQ_BIT(irq_num), _apic_pin_trigger ;			\
+	jz	8f ;				/* edge, don't mask */	\
+	orl	$IRQ_BIT(irq_num), _apic_imen ;	/* set the mask bit */	\
+	movl	_ioapic, %ecx ;			/* ioapic[0] addr */	\
+	movl	$REDTBL_IDX(irq_num), (%ecx) ;	/* write the index */	\
+	movl	IOAPIC_WINDOW(%ecx), %eax ;	/* current value */	\
+	orl	$IOART_INTMASK, %eax ;		/* set the mask */	\
+	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
+8: ;									\
+	IMASK_UNLOCK
+
+/*
+ * Test to see if the source is currntly masked, clear if so.
+ */
+#define UNMASK_IRQ(irq_num)					\
+	IMASK_LOCK ;				/* into critical reg */	\
+	testl	$IRQ_BIT(irq_num), _apic_imen ;				\
+	je	9f ;							\
+	andl	$~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */	\
+	movl	_ioapic,%ecx ;			/* ioapic[0]addr */	\
+	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
+	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
+	andl	$~IOART_INTMASK,%eax ;		/* clear the mask */	\
+	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */		\
+9: ;									\
+	IMASK_UNLOCK
+
+#define	INTR(irq_num, vec_name)						\
+	.text ;								\
+	SUPERALIGN_TEXT ;						\
+IDTVEC(vec_name) ;							\
+	PUSH_FRAME ;							\
+	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
+	movl	%ax, %ds ;						\
+	movl	%ax, %es ;						\
+;									\
+	lock ;					/* MP-safe */		\
+	btsl	$(irq_num), iactive ;		/* lazy masking */	\
+	jc	1f ;				/* already active */	\
+;									\
+	ISR_TRYLOCK ;		/* XXX this is going away... */		\
+	testl	%eax, %eax ;			/* did we get it? */	\
+	jz	1f ;				/* no */		\
+;									\
+	CPL_LOCK ;				/* MP-safe */		\
+	testl	$IRQ_BIT(irq_num), _cpl ;				\
+	jne	2f ;							\
+	orl	$IRQ_BIT(irq_num), _cil ;				\
+	CPL_UNLOCK ;							\
+;									\
+	movl	$0, lapic_eoi ;			/* XXX too soon? */	\
 	incb	_intr_nesting_level ;					\
 __CONCAT(Xresume,irq_num): ;						\
-	FAKE_MCOUNT(12*4(%esp)) ;	/* XXX late to avoid dbl cnt */ \
-	incl	_cnt+V_INTR ;	/* tally interrupts */			\
-	movl	_intr_countp + (irq_num) * 4,%eax ;			\
-	incl	(%eax) ;						\
-	movl	_cpl,%eax ;						\
+	FAKE_MCOUNT(12*4(%esp)) ;		/* XXX avoid dbl cnt */ \
+	lock ;	incl	_cnt+V_INTR ;		/* tally interrupts */	\
+	movl	_intr_countp + (irq_num) * 4, %eax ;			\
+	lock ;	incl	(%eax) ;					\
+;									\
+	CPL_LOCK ;				/* MP-safe */		\
+	movl	_cpl, %eax ;						\
 	pushl	%eax ;							\
+	orl	_intr_mask + (irq_num) * 4, %eax ;			\
+	movl	%eax, _cpl ;						\
+	CPL_UNLOCK ;							\
+;									\
 	pushl	_intr_unit + (irq_num) * 4 ;				\
-	orl	_intr_mask + (irq_num) * 4,%eax ;			\
-	movl	%eax,_cpl ;						\
 	sti ;								\
 	call	*_intr_handler + (irq_num) * 4 ;			\
-	MAYBE_UNMASK_IRQ(irq_num) ;					\
+	cli ;								\
+;									\
+	lock ;	andl	$~IRQ_BIT(irq_num), iactive ;			\
+	UNMASK_IRQ(irq_num) ;						\
+	sti ;				/* doreti repeats cli/sti */	\
 	MEXITCOUNT ;							\
 	jmp	_doreti ;						\
 ;									\
 	ALIGN_TEXT ;							\
-3: ;									\
-	/* XXX skip mcounting here to avoid double count */		\
-	lock ;					/* MP-safe */		\
+1: ;						/* active or locked */	\
+	MASK_LEVEL_IRQ(irq_num) ;					\
+	movl	$0, lapic_eoi ;			/* do the EOI */	\
+;									\
+	CPL_LOCK ;				/* MP-safe */		\
 	orl	$IRQ_BIT(irq_num), _ipending ;				\
-	REL_ISR_LOCK ;							\
-	popl	%es ;							\
-	popl	%ds ;							\
-	popal ;								\
-	addl	$4+4,%esp ;						\
-	iret
+	CPL_UNLOCK ;							\
+;									\
+	POP_FRAME ;							\
+	iret ;								\
+;									\
+	ALIGN_TEXT ;							\
+2: ;						/* masked by cpl */	\
+	CPL_UNLOCK ;							\
+	ISR_RELLOCK ;		/* XXX this is going away... */		\
+	jmp	1b
 
 
 /*
@@ -302,10 +277,6 @@ __CONCAT(Xresume,irq_num): ;						\
 	SUPERALIGN_TEXT
 	.globl _Xspuriousint
 _Xspuriousint:
-#ifdef COUNT_SPURIOUS_INTS
-	ss
-	incl	_sihits
-#endif
 
 	/* No EOI cycle used here */
 
@@ -358,10 +329,6 @@ _Xcpustop:
 
 	movl	_cpuid, %eax
 
-#ifdef COUNT_CSHITS
-	incl	_cshits(,%eax,4)
-#endif /* COUNT_CSHITS */
-
 	ASMPOSTCODE_HI(0x1)
 
 	lock
@@ -470,12 +437,6 @@ _ivectors:
 iactive:
 	.long	0
 
-#ifdef COUNT_SPURIOUS_INTS
-	.globl	_sihits
-_sihits:
-	.long	0
-#endif /* COUNT_SPURIOUS_INTS */
-
 #ifdef COUNT_XINVLTLB_HITS
 	.globl	_xhits
 _xhits:
@@ -489,17 +450,9 @@ _stopped_cpus:
 _started_cpus:
 	.long	0
 
-#ifdef COUNT_CSHITS
-	.globl	_cshits
-_cshits:
-	.space	(NCPU * 4), 0
-#endif /* COUNT_CSHITS */
-
-#ifdef PEND_INTS
 	.globl	_apic_pin_trigger
 _apic_pin_trigger:
 	.space	(NAPIC * 4), 0
-#endif /* PEND_INTS */
 
 
 /*
diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s
index 09635fa..67bf676 100644
--- a/sys/i386/i386/exception.s
+++ b/sys/i386/i386/exception.s
@@ -30,7 +30,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: exception.s,v 1.38 1997/08/10 21:18:01 fsmp Exp $
+ *	$Id: exception.s,v 1.12 1997/08/21 04:53:27 smp Exp smp $
  */
 
 #include "npx.h"				/* NNPX */
@@ -39,48 +39,7 @@
 #include <machine/psl.h>			/* PSL_I */
 #include <machine/trap.h>			/* trap codes */
 #include <machine/asmacros.h>			/* miscellaneous macros */
-
-#ifdef SMP
-
-#define	MPLOCKED	lock ;
-
-#define FPU_LOCK	call	_get_fpu_lock
-#define ALIGN_LOCK	call	_get_align_lock
-#define SYSCALL_LOCK	call	_get_syscall_lock
-#define ALTSYSCALL_LOCK	call	_get_altsyscall_lock
-
-/* protects the IO APIC and apic_imen as a critical region */
-#define IMASK_LOCK							\
-	pushl	$_imen_lock ;			/* address of lock */	\
-	call	_s_lock ;			/* MP-safe */		\
-	addl	$4,%esp
-
-#define IMASK_UNLOCK							\
-	pushl	$_imen_lock ;			/* address of lock */	\
-	call	_s_unlock ;			/* MP-safe */		\
-	addl	$4,%esp
-
-/* protects cpl updates as a critical region */
-#define CPL_LOCK							\
-	pushl	$_cpl_lock ;			/* address of lock */	\
-	call	_s_lock ;			/* MP-safe */		\
-	addl	$4,%esp
-
-#define CPL_UNLOCK							\
-	pushl	$_cpl_lock ;			/* address of lock */	\
-	call	_s_unlock ;			/* MP-safe */		\
-	addl	$4,%esp
-
-#else  /* SMP */
-
-#define	MPLOCKED				/* NOP */
-
-#define FPU_LOCK				/* NOP */
-#define ALIGN_LOCK				/* NOP */
-#define SYSCALL_LOCK				/* NOP */
-#define ALTSYSCALL_LOCK				/* NOP */
-
-#endif  /* SMP */
+#include <machine/param.h>
 
 #define	KCSEL		0x08			/* kernel code selector */
 #define	KDSEL		0x10			/* kernel data selector */
diff --git a/sys/i386/i386/microtime.s b/sys/i386/i386/microtime.s
index 173f61a..7b4f93d 100644
--- a/sys/i386/i386/microtime.s
+++ b/sys/i386/i386/microtime.s
@@ -32,16 +32,13 @@
  * SUCH DAMAGE.
  *
  *	from: Steve McCanne's microtime code
- *	$Id: microtime.s,v 1.25 1997/07/19 04:00:32 fsmp Exp $
+ *	$Id: microtime.s,v 1.4 1997/08/21 04:53:27 smp Exp smp $
  */
 
 #include "opt_cpu.h"
 
 #include <machine/asmacros.h>
-
-#ifdef APIC_IO
-#include <machine/smptests.h>		/** NEW STRATEGY, APIC_PIN0_TIMER */
-#endif /* APIC_IO */
+#include <machine/param.h>
 
 #include <i386/isa/icu.h>
 #include <i386/isa/isa.h>
@@ -114,21 +111,10 @@ ENTRY(microtime)
 	movl	_timer0_max_count, %edx	/* prepare for 2 uses */
 
 #ifdef APIC_IO
-#ifdef NEW_STRATEGY
-
+	CPL_LOCK			/* MP-safe */
 	movl	_ipending, %eax
 	testl	%eax, _mask8254		/* is soft timer interrupt pending? */
-
-#else /** NEW_STRATEGY */
-
-#ifdef APIC_PIN0_TIMER
-	testl	$IRQ0, _ipending	/* is soft timer interrupt pending? */
-#else
-	movl	_ipending, %eax
-	testl	%eax, _mask8254		/* is soft timer interrupt pending? */
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
+	CPL_UNLOCK
 #else
 	testb	$IRQ0, _ipending	/* is soft timer interrupt pending? */
 #endif /* APIC_IO */
@@ -139,21 +125,8 @@ ENTRY(microtime)
 	jbe	1f
 
 #ifdef APIC_IO
-#ifdef NEW_STRATEGY
-
-	movl	lapic_irr1, %eax	/** XXX assumption: IRQ0-24 */
-	testl	%eax, _mask8254		/* is hard timer interrupt pending? */
-
-#else /** NEW_STRATEGY */
-
-#ifdef APIC_PIN0_TIMER
-	testl	$IRQ0, lapic_irr1
-#else
 	movl	lapic_irr1, %eax	/** XXX assumption: IRQ0-24 */
 	testl	%eax, _mask8254		/* is hard timer interrupt pending? */
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
 #else
 	inb	$IO_ICU1, %al		/* read IRR in ICU */
 	testb	$IRQ0, %al		/* is hard timer interrupt pending? */
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index e5fb310..b02fde7 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.41 1997/08/10 19:32:38 fsmp Exp $
+ *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -1419,6 +1419,18 @@ default_mp_table(int type)
  * initialize all the SMP locks
  */
 
+/* critical region around IO APIC, apic_imen */
+struct simplelock	imen_lock;
+
+/* critical region around splxx(), cpl, cil, ipending */
+struct simplelock	cpl_lock;
+
+/* Make FAST_INTR() routines sequential */
+struct simplelock	fast_intr_lock;
+
+/* critical region around INTR() routines */
+struct simplelock	intr_lock;
+
 /* lock the com (tty) data structures */
 struct simplelock	com_lock;
 
diff --git a/sys/i386/i386/mpapic.c b/sys/i386/i386/mpapic.c
index 699b735..db7b56b 100644
--- a/sys/i386/i386/mpapic.c
+++ b/sys/i386/i386/mpapic.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mpapic.c,v 1.26 1997/07/30 22:51:11 smp Exp smp $
+ *	$Id: mpapic.c,v 1.27 1997/08/21 04:53:27 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -30,7 +30,7 @@
 #include <sys/types.h>
 #include <sys/systm.h>
 
-#include <machine/smptests.h>	/** PEND_INTS, TEST_TEST1 */
+#include <machine/smptests.h>	/** TEST_TEST1 */
 #include <machine/smp.h>
 #include <machine/mpapic.h>
 #include <machine/segments.h>
@@ -146,9 +146,7 @@ static void polarity __P((int apic, int pin, u_int32_t * flags, int level));
 /*
  * Setup the IO APIC.
  */
-#if defined(PEND_INTS)
 extern int	apic_pin_trigger[];	/* 'opaque' */
-#endif /* PEND_INTS */
 int
 io_apic_setup(int apic)
 {
@@ -161,9 +159,7 @@ io_apic_setup(int apic)
 
 	target = IOART_DEST;
 
-#if defined(PEND_INTS)
 	apic_pin_trigger[apic] = 0;	/* default to edge-triggered */
-#endif /* PEND_INTS */
 
 	if (apic == 0) {
 		maxpin = REDIRCNT_IOAPIC(apic);		/* pins in APIC */
@@ -189,10 +185,8 @@ io_apic_setup(int apic)
 			else {
 				flags = DEFAULT_FLAGS;
 				level = trigger(apic, pin, &flags);
-#if defined(PEND_INTS)
 				if (level == 1)
 					apic_pin_trigger[apic] |= (1 << pin);
-#endif /* PEND_INTS */
 				polarity(apic, pin, &flags, level);
 			}
 
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index e5fb310..b02fde7 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.41 1997/08/10 19:32:38 fsmp Exp $
+ *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -1419,6 +1419,18 @@ default_mp_table(int type)
  * initialize all the SMP locks
  */
 
+/* critical region around IO APIC, apic_imen */
+struct simplelock	imen_lock;
+
+/* critical region around splxx(), cpl, cil, ipending */
+struct simplelock	cpl_lock;
+
+/* Make FAST_INTR() routines sequential */
+struct simplelock	fast_intr_lock;
+
+/* critical region around INTR() routines */
+struct simplelock	intr_lock;
+
 /* lock the com (tty) data structures */
 struct simplelock	com_lock;
 
diff --git a/sys/i386/i386/tsc.c b/sys/i386/i386/tsc.c
index 5b54178..7f89a7f 100644
--- a/sys/i386/i386/tsc.c
+++ b/sys/i386/i386/tsc.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.97 1997/07/22 20:12:04 fsmp Exp $
+ *	$Id: clock.c,v 1.8 1997/08/21 04:51:12 smp Exp smp $
  */
 
 /*
@@ -66,7 +66,6 @@
 #include <machine/ipl.h>
 #ifdef APIC_IO
 #include <machine/smp.h>
-#include <machine/smptests.h>		/** NEW_STRATEGY (,SMP_TIMER_NC) */
 #endif /* APIC_IO */
 
 #include <i386/isa/icu.h>
@@ -74,6 +73,9 @@
 #include <i386/isa/rtc.h>
 #include <i386/isa/timerreg.h>
 
+#include <i386/isa/intr_machdep.h>
+#include <sys/interrupt.h>
+
 /*
  * 32-bit time_t's can't reach leap years before 1904 or after 2036, so we
  * can use a simple formula for leap years.
@@ -859,11 +861,6 @@ cpu_initclocks()
 	/* Finish initializing 8253 timer 0. */
 #ifdef APIC_IO
 
-#ifdef NEW_STRATEGY
-#ifdef SMP_TIMER_NC
-#error 'options SMP_TIMER_NC' no longer used, remove & reconfig.
-#endif	/** XXX SMP_TIMER_NC */
-
 	/* 1st look for ExtInt on pin 0 */
 	if (apic_int_type(0, 0) == 3) {
 		/*
@@ -897,33 +894,6 @@ cpu_initclocks()
 	else 
 		panic("neither pin 0 or pin 2 works for 8254");
 
-#else /** NEW_STRATEGY */
-
-	/* 8254 is traditionally on ISA IRQ0 */
-#if defined(SMP_TIMER_NC)
-	x = -1;
-#else
-	x = isa_apic_pin(0);
-#endif	/** XXX SMP_TIMER_NC */
-
-	if (x < 0) {
-		/* bummer, attempt to redirect thru the 8259 */
-		if (bootverbose)
-			printf("APIC missing 8254 connection\n");
-
-		/* allow 8254 timer to INTerrupt 8259 */
-		x = inb(IO_ICU1 + 1);		/* current mask in 8259 */
-		x &= ~1;			/* clear 8254 timer mask */
-		outb(IO_ICU1 + 1, x);		/* write new mask */
-
-		/* program IO APIC for type 3 INT on INT0 */
-		if (ext_int_setup(0, 0) < 0)
-			panic("8254 redirect impossible!");
-		x = 0;				/* 8259 is on 0 */
-	}
-
-#endif /** NEW_STRATEGY */
-
 	/* setup the vectors */
 	vec[x] = (u_int)vec8254;
 	Xintr8254 = (u_int)ivectors[x];
@@ -966,9 +936,11 @@ cpu_initclocks()
 	if (isa_apic_pin(8) != 8)
 		panic("APIC RTC != 8");
 #endif /* APIC_IO */
+
 	register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
 		      /* unit */ 0);
+
 #ifdef APIC_IO
 	INTREN(APIC_IRQ8);
 #else
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index e5fb310..b02fde7 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.41 1997/08/10 19:32:38 fsmp Exp $
+ *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -1419,6 +1419,18 @@ default_mp_table(int type)
  * initialize all the SMP locks
  */
 
+/* critical region around IO APIC, apic_imen */
+struct simplelock	imen_lock;
+
+/* critical region around splxx(), cpl, cil, ipending */
+struct simplelock	cpl_lock;
+
+/* Make FAST_INTR() routines sequential */
+struct simplelock	fast_intr_lock;
+
+/* critical region around INTR() routines */
+struct simplelock	intr_lock;
+
 /* lock the com (tty) data structures */
 struct simplelock	com_lock;
 
diff --git a/sys/i386/include/param.h b/sys/i386/include/param.h
index 54b3b26..ec399c1 100644
--- a/sys/i386/include/param.h
+++ b/sys/i386/include/param.h
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)param.h	5.8 (Berkeley) 6/28/91
- *	$Id: param.h,v 1.33 1997/08/09 00:03:16 dyson Exp $
+ *	$Id: param.h,v 1.9 1997/08/21 04:48:45 smp Exp smp $
  */
 
 #ifndef _MACHINE_PARAM_H_
@@ -52,6 +52,9 @@
 #endif
 #define MID_MACHINE	MID_I386
 
+
+#ifndef LOCORE
+
 /*
  * Round p (pointer or byte index) up to a correctly-aligned value
  * for all data types (int, long, ...).   The result is unsigned int
@@ -136,8 +139,85 @@
 #define i386_btop(x)		((unsigned)(x) >> PAGE_SHIFT)
 #define i386_ptob(x)		((unsigned)(x) << PAGE_SHIFT)
 
+#endif /* !LOCORE */
+
+
 #ifndef _SIMPLELOCK_H_
 #define _SIMPLELOCK_H_
+
+#ifdef LOCORE
+
+#ifdef SMP
+
+#define	MPLOCKED	lock ;
+
+/*
+ * Some handy macros to allow logical organization and
+ * convenient reassignment of various locks.
+ */
+
+#define FPU_LOCK	call	_get_fpu_lock
+#define ALIGN_LOCK	call	_get_align_lock
+#define SYSCALL_LOCK	call	_get_syscall_lock
+#define ALTSYSCALL_LOCK	call	_get_altsyscall_lock
+
+/*
+ * Protects INTR() ISRs.
+ */
+#define ISR_TRYLOCK							\
+	pushl	$_mp_lock ;			/* GIANT_LOCK */	\
+	call	_MPtrylock ;			/* try to get lock */	\
+	add	$4, %esp
+
+#define ISR_RELLOCK							\
+	pushl	$_mp_lock ;			/* GIANT_LOCK */	\
+	call	_MPrellock ;						\
+	add	$4, %esp
+
+/*
+ * Protects the IO APIC and apic_imen as a critical region.
+ */
+#define IMASK_LOCK							\
+	pushl	$_imen_lock ;			/* address of lock */	\
+	call	_s_lock ;			/* MP-safe */		\
+	addl	$4, %esp
+
+#define IMASK_UNLOCK							\
+	pushl	$_imen_lock ;			/* address of lock */	\
+	call	_s_unlock ;			/* MP-safe */		\
+	addl	$4, %esp
+
+/*
+ * Protects spl updates as a critical region.
+ * Items within this 'region' include:
+ *  cpl
+ *  cil
+ *  ipending
+ *  ???
+ */
+#define CPL_LOCK							\
+	pushl	$_cpl_lock ;			/* address of lock */	\
+	call	_s_lock ;			/* MP-safe */		\
+	addl	$4, %esp
+
+#define CPL_UNLOCK							\
+	pushl	$_cpl_lock ;			/* address of lock */	\
+	call	_s_unlock ;			/* MP-safe */		\
+	addl	$4, %esp
+
+#else  /* SMP */
+
+#define	MPLOCKED				/* NOP */
+
+#define FPU_LOCK				/* NOP */
+#define ALIGN_LOCK				/* NOP */
+#define SYSCALL_LOCK				/* NOP */
+#define ALTSYSCALL_LOCK				/* NOP */
+
+#endif  /* SMP */
+
+#else /* LOCORE */
+
 /*
  * A simple spin lock.
  *
@@ -151,6 +231,19 @@ struct simplelock {
 	volatile int	lock_data;
 };
 
+/* functions in simplelock.s */
+void	s_lock_init		__P((struct simplelock *));
+void	s_lock			__P((struct simplelock *));
+int	s_lock_try		__P((struct simplelock *));
+void	s_unlock		__P((struct simplelock *));
+
+/* global data in mp_machdep.c */
+extern struct simplelock	imen_lock;
+extern struct simplelock	cpl_lock;
+extern struct simplelock	fast_intr_lock;
+extern struct simplelock	intr_lock;
+extern struct simplelock	com_lock;
+
 #if !defined(SIMPLELOCK_DEBUG) && NCPUS > 1
 /*
  * The simple-lock routines are the primitives out of which the lock
@@ -206,6 +299,7 @@ simple_unlock(struct simplelock *lkp)
 #endif /* the_original_code */
 
 #endif /* NCPUS > 1 */
+#endif /* LOCORE */
 #endif /* !_SIMPLELOCK_H_ */
 
 #endif /* !_MACHINE_PARAM_H_ */
diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h
index e80d91a..6b195f6f 100644
--- a/sys/i386/include/smp.h
+++ b/sys/i386/include/smp.h
@@ -6,7 +6,7 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
- * $Id: smp.h,v 1.26 1997/08/18 03:35:59 fsmp Exp $
+ * $Id: smp.h,v 1.23 1997/08/21 04:48:45 smp Exp smp $
  *
  */
 
@@ -75,10 +75,6 @@ extern volatile u_int		started_cpus;
 extern u_int			vec[];
 extern u_int			Xintr8254;
 extern u_int			mask8254;
-extern struct simplelock	imen_lock;
-extern struct simplelock	cpl_lock;
-extern struct simplelock	fast_intr_lock;
-extern struct simplelock	intr_lock;
 
 /* functions in apic_ipl.s */
 void	vec8254			__P((void));
@@ -88,13 +84,6 @@ void	apic_eoi		__P((void));
 u_int	io_apic_read		__P((int, int));
 void	io_apic_write		__P((int, int, u_int));
 
-/* functions in simplelock.s */
-#include <machine/param.h>
-void	s_lock_init		__P((struct simplelock *));
-void	s_lock			__P((struct simplelock *));
-int	s_lock_try		__P((struct simplelock *));
-void	s_unlock		__P((struct simplelock *));
-
 /* global data in mp_machdep.c */
 extern int			mp_ncpus;
 extern int			mp_naps;
@@ -113,8 +102,6 @@ extern u_int			all_cpus;
 extern u_int			SMP_prvpt[];
 extern u_char			SMP_ioapic[];
 
-extern struct simplelock	com_lock;
-
 /* functions in mp_machdep.c */
 u_int	mp_bootaddress		__P((u_int));
 int	mp_probe		__P((void));
diff --git a/sys/i386/include/smptests.h b/sys/i386/include/smptests.h
index 9a99169..e3f7e90 100644
--- a/sys/i386/include/smptests.h
+++ b/sys/i386/include/smptests.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: smptests.h,v 1.19 1997/08/04 17:31:28 fsmp Exp $
+ *	$Id: smptests.h,v 1.22 1997/08/21 04:48:45 smp Exp smp $
  */
 
 #ifndef _MACHINE_SMPTESTS_H_
@@ -36,55 +36,48 @@
 
 /*
  * Ignore the ipending bits when exiting FAST_INTR() routines.
+ *
+ ***
+ * according to Bruce:
+ *
+ * setsoft*() may set ipending.  setsofttty() is actually used in the
+ * FAST_INTR handler in some serial drivers.  This is necessary to get
+ * output completions and other urgent events handled as soon as possible.
+ * The flag(s) could be set in a variable other than ipending, but they
+ * needs to be checked against cpl to decide whether the software interrupt
+ * handler can/should run.
+ *
+ *  (FAST_INTR used to just return
+ * in all cases until rev.1.7 of vector.s.  This worked OK provided there
+ * were no user-mode CPU hogs.  CPU hogs caused an average latency of 1/2
+ * clock tick for output completions...)
+ ***
+ *
+ * So I need to restore cpl handling someday, but AFTER
+ *  I finish making spl/cpl MP-safe.
  */
 #define FAST_WITHOUTCPL
 
 
 /*
  * Use a simplelock to serialize FAST_INTR()s.
+ * sio.c, and probably other FAST_INTR() drivers, never expected several CPUs
+ * to be inside them at once.  Things such as global vars prevent more
+ * than 1 thread of execution from existing at once, so we serialize
+ * the access of FAST_INTR()s via a simple lock.
+ * One optimization on this would be a simple lock per DRIVER, but I'm
+ * not sure how to organize that yet...
  */
 #define FAST_SIMPLELOCK
 
 
 /*
- * Use the new INT passoff algorithm:
- *
- *	int_is_already_active = iactive & (1 << INT_NUMBER);
- *	iactive |= (1 << INT_NUMBER);
- *	if ( int_is_already_active || (try_mplock() == FAIL ) {
- *		mask_apic_int( INT_NUMBER );
- *		ipending |= (1 << INT_NUMBER);
- *		do_eoi();
- *		cleanup_and_iret();
- *	}
- */
-#define PEND_INTS
-
-
-/*
  * Portions of the old TEST_LOPRIO code, back from the grave!
  */
 #define GRAB_LOPRIO
 
 
 /*
- * 1st attempt to use the 'ExtInt' connected 8259 to attach 8254 timer.
- * failing that, attempt to attach 8254 timer via direct APIC pin.
- * failing that, panic!
- *
- */
-#define NEW_STRATEGY
-
-
-/*
- * For emergency fallback, define ONLY if 'NEW_STRATEGY' fails to work.
- * Formerly needed by Tyan Tomcat II and SuperMicro P6DNxxx motherboards.
- *
-#define SMP_TIMER_NC
- */
-
-
-/*
  * Send CPUSTOP IPI for stop/restart of other CPUs on DDB break.
  *
  */
@@ -101,22 +94,20 @@
 
 
 /*
- * deal with broken smp_idleloop()
+ * Deal with broken smp_idleloop().
  */
 #define IGNORE_IDLEPROCS
 
 
 /*
- * misc. counters
+ * Misc. counters.
  *
 #define COUNT_XINVLTLB_HITS
-#define COUNT_SPURIOUS_INTS
-#define COUNT_CSHITS
  */
 
 
 /**
- * hack to "fake-out" kernel into thinking it is running on a 'default config'
+ * Hack to "fake-out" kernel into thinking it is running on a 'default config'.
  *
  * value == default type
 #define TEST_DEFAULT_CONFIG	6
@@ -124,7 +115,7 @@
 
 
 /*
- * simple test code for IPI interaction, save for future...
+ * Simple test code for IPI interaction, save for future...
  *
 #define TEST_TEST1
 #define IPI_TARGET_TEST1	1
@@ -194,7 +185,7 @@
 
 
 /*
- * these are all temps for debugging...
+ * These are all temps for debugging...
  *
 #define GUARD_INTS
  */
diff --git a/sys/i386/isa/apic_ipl.s b/sys/i386/isa/apic_ipl.s
index 1412367..a36c4de 100644
--- a/sys/i386/isa/apic_ipl.s
+++ b/sys/i386/isa/apic_ipl.s
@@ -22,17 +22,13 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: apic_ipl.s,v 1.11 1997/07/31 17:28:56 fsmp Exp $
+ *	$Id: apic_ipl.s,v 1.23 1997/08/21 04:52:30 smp Exp smp $
  */
 
 
-#include <machine/smptests.h>		/** NEW_STRATEGY, APIC_PIN0_TIMER */
-
 	.data
 	ALIGN_DATA
 
-#ifdef NEW_STRATEGY
-
 /* this allows us to change the 8254 APIC pin# assignment */
 	.globl _Xintr8254
 _Xintr8254:
@@ -43,22 +39,6 @@ _Xintr8254:
 _mask8254:
 	.long	0
 
-#else /** NEW_STRATEGY */
-
-#ifndef APIC_PIN0_TIMER
-/* this allows us to change the 8254 APIC pin# assignment */
-	.globl _Xintr8254
-_Xintr8254:
-	.long	_Xintr7
-
-/* used by this file, microtime.s and clock.c */
-	.globl _mask8254
-_mask8254:
-	.long	0
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
-
 /*  */
 	.globl _vec
 _vec:
@@ -69,6 +49,7 @@ _vec:
 /* various simple locks */
 	.align 2				/* MUST be 32bit aligned */
 
+#if 0
 /* critical region around IO APIC */
 	.globl _imen_lock
 _imen_lock:
@@ -88,6 +69,7 @@ _fast_intr_lock:
 	.globl _intr_lock
 _intr_lock:
 	.long	0
+#endif
 
 /*
  * Note:
@@ -122,7 +104,6 @@ _apic_imen:
  * generic vector function for 8254 clock
  */
 	ALIGN_TEXT
-#ifdef NEW_STRATEGY
 
 	.globl	_vec8254
 _vec8254:
@@ -139,38 +120,6 @@ _vec8254:
 	movl	_Xintr8254, %eax
 	jmp	%eax			/* XXX might need _Xfastintr# */
 
-#else /** NEW_STRATEGY */
-
-#ifdef APIC_PIN0_TIMER
-vec0:
-	popl	%eax			/* return address */
-	pushfl
-	pushl	$KCSEL
-	pushl	%eax
-	cli
-	lock				/* MP-safe */
-	andl	$~IRQ_BIT(0), iactive	/* lazy masking */
-	MEXITCOUNT
-	jmp	_Xintr0			/* XXX might need _Xfastintr0 */
-#else
-	.globl	_vec8254
-_vec8254:
-	popl	%eax			/* return address */
-	pushfl
-	pushl	$KCSEL
-	pushl	%eax
-	movl	_mask8254, %eax		/* lazy masking */
-	notl	%eax
-	cli
-	lock				/* MP-safe */
-	andl	%eax, iactive
-	MEXITCOUNT
-	movl	_Xintr8254, %eax
-	jmp	%eax			/* XXX might need _Xfastintr# */
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
-
 
 /*
  * generic vector function for RTC clock
@@ -205,17 +154,7 @@ __CONCAT(vec,irq_num): ;						\
 	jmp	__CONCAT(_Xintr,irq_num)
 
 
-#ifdef NEW_STRATEGY
-
 	BUILD_VEC(0)
-
-#else /** NEW_STRATEGY */
-
-#ifndef APIC_PIN0_TIMER
-	BUILD_VEC(0)
-#endif /* APIC_PIN0_TIMER */
-
-#endif /** NEW_STRATEGY */
 	BUILD_VEC(1)
 	BUILD_VEC(2)
 	BUILD_VEC(3)
diff --git a/sys/i386/isa/apic_vector.s b/sys/i386/isa/apic_vector.s
index f2a8bce..9d1d4a0 100644
--- a/sys/i386/isa/apic_vector.s
+++ b/sys/i386/isa/apic_vector.s
@@ -1,12 +1,12 @@
 /*
  *	from: vector.s, 386BSD 0.1 unknown origin
- *	$Id: apic_vector.s,v 1.15 1997/08/10 20:58:57 fsmp Exp $
+ *	$Id: apic_vector.s,v 1.24 1997/08/21 04:52:30 smp Exp smp $
  */
 
 
 #include <machine/apic.h>
 #include <machine/smp.h>
-#include <machine/smptests.h>		/** PEND_INTS, various counters */
+#include <machine/smptests.h>			/** various things... */
 
 #include "i386/isa/intr_machdep.h"
 
@@ -27,6 +27,7 @@
 
 #define GET_FAST_INTR_LOCK						\
 	call	_get_isrlock
+
 #define REL_FAST_INTR_LOCK						\
 	pushl	$_mp_lock ;	/* GIANT_LOCK */			\
 	call	_MPrellock ;						\
@@ -34,11 +35,6 @@
 
 #endif /* FAST_SIMPLELOCK */
 
-#define REL_ISR_LOCK							\
-	pushl	$_mp_lock ;	/* GIANT_LOCK */			\
-	call	_MPrellock ;						\
-	add	$4, %esp
-
 /* convert an absolute IRQ# into a bitmask */
 #define IRQ_BIT(irq_num)	(1 << (irq_num))
 
@@ -47,105 +43,13 @@
 
 
 /*
- * 'lazy masking' code suggested by Bruce Evans <bde@zeta.org.au>
- */
-
-#ifdef PEND_INTS
-
-/*
- * the 1st version fails because masked edge-triggered INTs are lost
- * by the IO APIC.  This version tests to see whether we are handling
- * an edge or level triggered INT.  Level-triggered INTs must still be
- * masked as we don't clear the source, and the EOI cycle would allow
- * recursive INTs to occur.
- */
-#define MAYBE_MASK_IRQ(irq_num)						\
-	lock ;					/* MP-safe */		\
-	btsl	$(irq_num),iactive ;		/* lazy masking */	\
-	jc	6f ;				/* already active */	\
-	pushl	$_mp_lock ;			/* GIANT_LOCK */	\
-	call	_MPtrylock ;			/* try to get lock */	\
-	add $4,	%esp ;							\
-	testl	%eax, %eax ;			/* did we get it? */	\
-	jnz	8f ;				/* yes, enter kernel */	\
-6: ;						/* active or locked */	\
-	IMASK_LOCK ;				/* into critical reg */	\
-	testl	$IRQ_BIT(irq_num),_apic_pin_trigger ;			\
-	jz	7f ;				/* edge, don't mask */	\
-	orl	$IRQ_BIT(irq_num),_apic_imen ;	/* set the mask bit */	\
-	movl	_ioapic,%ecx ;			/* ioapic[0] addr */	\
-	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
-	orl	$IOART_INTMASK,%eax ;		/* set the mask */	\
-	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-7: ;									\
-	lock ;					/* MP-safe */		\
-	orl	$IRQ_BIT(irq_num), _ipending ;	/* set _ipending bit */	\
-	IMASK_UNLOCK ;				/* exit critical reg */	\
-	movl	$0, lapic_eoi ;			/* do the EOI */	\
-	popl	%es ;							\
-	popl	%ds ;							\
-	popal ;								\
-	addl	$4+4,%esp ;						\
-	iret ;								\
-;									\
-	ALIGN_TEXT ;							\
-8:
-
-#else /* PEND_INTS */
-
-#define MAYBE_MASK_IRQ(irq_num)						\
-	lock ;					/* MP-safe */		\
-	btsl	$(irq_num),iactive ;		/* lazy masking */	\
-	jnc	1f ;				/* NOT active */	\
-	IMASK_LOCK ;				/* enter critical reg */\
-	orl	$IRQ_BIT(irq_num),_apic_imen ;	/* set the mask bit */	\
-	movl	_ioapic,%ecx ;			/* ioapic[0]addr */	\
-	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
-	orl	$IOART_INTMASK,%eax ;		/* set the mask */	\
-	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-	lock ;					/* MP-safe */		\
-	orl	$IRQ_BIT(irq_num), _ipending ;	/* set _ipending bit */	\
-	movl	$0, lapic_eoi ;			/* do the EOI */	\
-	IMASK_UNLOCK ;				/* exit critical reg */	\
-	popl	%es ;							\
-	popl	%ds ;							\
-	popal ;								\
-	addl	$4+4,%esp ;						\
-	iret ;								\
-;									\
-	ALIGN_TEXT ;							\
-1: ;									\
-	call	_get_mplock			/* SMP Spin lock */
-
-#endif /* PEND_INTS */
-
-
-#define MAYBE_UNMASK_IRQ(irq_num)					\
-	cli ;	/* must unmask _apic_imen and IO APIC atomically */	\
-	lock ;					/* MP-safe */		\
-	andl	$~IRQ_BIT(irq_num),iactive ;				\
-	IMASK_LOCK ;				/* enter critical reg */\
-	testl	$IRQ_BIT(irq_num),_apic_imen ;				\
-	je	9f ;							\
-	andl	$~IRQ_BIT(irq_num),_apic_imen ;	/* clear mask bit */	\
-	movl	_ioapic,%ecx ;			/* ioapic[0]addr */	\
-	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
-	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
-	andl	$~IOART_INTMASK,%eax ;		/* clear the mask */	\
-	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */		\
-9: ;									\
-	IMASK_UNLOCK ;				/* exit critical reg */	\
-	sti	/* XXX _doreti repeats the cli/sti */
-
-
-/*
  * Macros for interrupt interrupt entry, call to handler, and exit.
  */
 
 #ifdef FAST_WITHOUTCPL
 
+/*
+ */
 #define	FAST_INTR(irq_num, vec_name)					\
 	.text ;								\
 	SUPERALIGN_TEXT ;						\
@@ -244,51 +148,122 @@ IDTVEC(vec_name) ;							\
 #endif /** FAST_WITHOUTCPL */
 
 
-#define	INTR(irq_num, vec_name)						\
-	.text ;								\
-	SUPERALIGN_TEXT ;						\
-IDTVEC(vec_name) ;							\
+/*
+ * 
+ */
+#define PUSH_FRAME							\
 	pushl	$0 ;		/* dummy error code */			\
 	pushl	$0 ;		/* dummy trap type */			\
 	pushal ;							\
 	pushl	%ds ;		/* save data and extra segments ... */	\
-	pushl	%es ;							\
-	movl	$KDSEL,%eax ;	/* ... and reload with kernel's ... */	\
-	movl	%ax,%ds ;	/* ... early for obsolete reasons */	\
-	movl	%ax,%es ;						\
-	MAYBE_MASK_IRQ(irq_num) ;					\
-	movl	$0, lapic_eoi ;						\
-	movl	_cpl,%eax ;						\
-	testl	$IRQ_BIT(irq_num), %eax ;				\
-	jne	3f ;							\
+	pushl	%es
+
+#define POP_FRAME							\
+	popl	%es ;							\
+	popl	%ds ;							\
+	popal ;								\
+	addl	$4+4,%esp
+
+/*
+ * Test to see whether we are handling an edge or level triggered INT.
+ *  Level-triggered INTs must still be masked as we don't clear the source,
+ *  and the EOI cycle would cause redundant INTs to occur.
+ */
+#define MASK_LEVEL_IRQ(irq_num)						\
+	IMASK_LOCK ;				/* into critical reg */	\
+	testl	$IRQ_BIT(irq_num), _apic_pin_trigger ;			\
+	jz	8f ;				/* edge, don't mask */	\
+	orl	$IRQ_BIT(irq_num), _apic_imen ;	/* set the mask bit */	\
+	movl	_ioapic, %ecx ;			/* ioapic[0] addr */	\
+	movl	$REDTBL_IDX(irq_num), (%ecx) ;	/* write the index */	\
+	movl	IOAPIC_WINDOW(%ecx), %eax ;	/* current value */	\
+	orl	$IOART_INTMASK, %eax ;		/* set the mask */	\
+	movl	%eax, IOAPIC_WINDOW(%ecx) ;	/* new value */		\
+8: ;									\
+	IMASK_UNLOCK
+
+/*
+ * Test to see if the source is currntly masked, clear if so.
+ */
+#define UNMASK_IRQ(irq_num)					\
+	IMASK_LOCK ;				/* into critical reg */	\
+	testl	$IRQ_BIT(irq_num), _apic_imen ;				\
+	je	9f ;							\
+	andl	$~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */	\
+	movl	_ioapic,%ecx ;			/* ioapic[0]addr */	\
+	movl	$REDTBL_IDX(irq_num),(%ecx) ;	/* write the index */	\
+	movl	IOAPIC_WINDOW(%ecx),%eax ;	/* current value */	\
+	andl	$~IOART_INTMASK,%eax ;		/* clear the mask */	\
+	movl	%eax,IOAPIC_WINDOW(%ecx) ;	/* new value */		\
+9: ;									\
+	IMASK_UNLOCK
+
+#define	INTR(irq_num, vec_name)						\
+	.text ;								\
+	SUPERALIGN_TEXT ;						\
+IDTVEC(vec_name) ;							\
+	PUSH_FRAME ;							\
+	movl	$KDSEL, %eax ;	/* reload with kernel's data segment */	\
+	movl	%ax, %ds ;						\
+	movl	%ax, %es ;						\
+;									\
+	lock ;					/* MP-safe */		\
+	btsl	$(irq_num), iactive ;		/* lazy masking */	\
+	jc	1f ;				/* already active */	\
+;									\
+	ISR_TRYLOCK ;		/* XXX this is going away... */		\
+	testl	%eax, %eax ;			/* did we get it? */	\
+	jz	1f ;				/* no */		\
+;									\
+	CPL_LOCK ;				/* MP-safe */		\
+	testl	$IRQ_BIT(irq_num), _cpl ;				\
+	jne	2f ;							\
+	orl	$IRQ_BIT(irq_num), _cil ;				\
+	CPL_UNLOCK ;							\
+;									\
+	movl	$0, lapic_eoi ;			/* XXX too soon? */	\
 	incb	_intr_nesting_level ;					\
 __CONCAT(Xresume,irq_num): ;						\
-	FAKE_MCOUNT(12*4(%esp)) ;	/* XXX late to avoid dbl cnt */ \
-	incl	_cnt+V_INTR ;	/* tally interrupts */			\
-	movl	_intr_countp + (irq_num) * 4,%eax ;			\
-	incl	(%eax) ;						\
-	movl	_cpl,%eax ;						\
+	FAKE_MCOUNT(12*4(%esp)) ;		/* XXX avoid dbl cnt */ \
+	lock ;	incl	_cnt+V_INTR ;		/* tally interrupts */	\
+	movl	_intr_countp + (irq_num) * 4, %eax ;			\
+	lock ;	incl	(%eax) ;					\
+;									\
+	CPL_LOCK ;				/* MP-safe */		\
+	movl	_cpl, %eax ;						\
 	pushl	%eax ;							\
+	orl	_intr_mask + (irq_num) * 4, %eax ;			\
+	movl	%eax, _cpl ;						\
+	CPL_UNLOCK ;							\
+;									\
 	pushl	_intr_unit + (irq_num) * 4 ;				\
-	orl	_intr_mask + (irq_num) * 4,%eax ;			\
-	movl	%eax,_cpl ;						\
 	sti ;								\
 	call	*_intr_handler + (irq_num) * 4 ;			\
-	MAYBE_UNMASK_IRQ(irq_num) ;					\
+	cli ;								\
+;									\
+	lock ;	andl	$~IRQ_BIT(irq_num), iactive ;			\
+	UNMASK_IRQ(irq_num) ;						\
+	sti ;				/* doreti repeats cli/sti */	\
 	MEXITCOUNT ;							\
 	jmp	_doreti ;						\
 ;									\
 	ALIGN_TEXT ;							\
-3: ;									\
-	/* XXX skip mcounting here to avoid double count */		\
-	lock ;					/* MP-safe */		\
+1: ;						/* active or locked */	\
+	MASK_LEVEL_IRQ(irq_num) ;					\
+	movl	$0, lapic_eoi ;			/* do the EOI */	\
+;									\
+	CPL_LOCK ;				/* MP-safe */		\
 	orl	$IRQ_BIT(irq_num), _ipending ;				\
-	REL_ISR_LOCK ;							\
-	popl	%es ;							\
-	popl	%ds ;							\
-	popal ;								\
-	addl	$4+4,%esp ;						\
-	iret
+	CPL_UNLOCK ;							\
+;									\
+	POP_FRAME ;							\
+	iret ;								\
+;									\
+	ALIGN_TEXT ;							\
+2: ;						/* masked by cpl */	\
+	CPL_UNLOCK ;							\
+	ISR_RELLOCK ;		/* XXX this is going away... */		\
+	jmp	1b
 
 
 /*
@@ -302,10 +277,6 @@ __CONCAT(Xresume,irq_num): ;						\
 	SUPERALIGN_TEXT
 	.globl _Xspuriousint
 _Xspuriousint:
-#ifdef COUNT_SPURIOUS_INTS
-	ss
-	incl	_sihits
-#endif
 
 	/* No EOI cycle used here */
 
@@ -358,10 +329,6 @@ _Xcpustop:
 
 	movl	_cpuid, %eax
 
-#ifdef COUNT_CSHITS
-	incl	_cshits(,%eax,4)
-#endif /* COUNT_CSHITS */
-
 	ASMPOSTCODE_HI(0x1)
 
 	lock
@@ -470,12 +437,6 @@ _ivectors:
 iactive:
 	.long	0
 
-#ifdef COUNT_SPURIOUS_INTS
-	.globl	_sihits
-_sihits:
-	.long	0
-#endif /* COUNT_SPURIOUS_INTS */
-
 #ifdef COUNT_XINVLTLB_HITS
 	.globl	_xhits
 _xhits:
@@ -489,17 +450,9 @@ _stopped_cpus:
 _started_cpus:
 	.long	0
 
-#ifdef COUNT_CSHITS
-	.globl	_cshits
-_cshits:
-	.space	(NCPU * 4), 0
-#endif /* COUNT_CSHITS */
-
-#ifdef PEND_INTS
 	.globl	_apic_pin_trigger
 _apic_pin_trigger:
 	.space	(NAPIC * 4), 0
-#endif /* PEND_INTS */
 
 
 /*
diff --git a/sys/i386/isa/bs/bsif.h b/sys/i386/isa/bs/bsif.h
index ac93bea..502bfe7 100644
--- a/sys/i386/isa/bs/bsif.h
+++ b/sys/i386/isa/bs/bsif.h
@@ -201,7 +201,17 @@ static BS_INLINE void memcopy __P((void *from, void *to, register size_t len));
 u_int32_t bs_adapter_info __P((int));
 #define delay(y) DELAY(y)
 extern int dma_init_flag;
+#ifdef SMP
+#error XXX see comments in i386/isa/bs/bsif.h for details
+/*
+ * ipending is 'opaque' in SMP, and can't be accessed this way.
+ * Since its my belief that this is PC98 code, and that PC98 and SMP
+ * are mutually exclusive, the above compile-time error is the "fix".
+ * Please inform smp@freebsd.org if this is NOT the case.
+ */
+#else
 #define softintr(y) ipending |= (y)
+#endif /* SMP */
 
 static BS_INLINE void
 memcopy(from, to, len)
diff --git a/sys/i386/isa/clock.c b/sys/i386/isa/clock.c
index 5b54178..7f89a7f 100644
--- a/sys/i386/isa/clock.c
+++ b/sys/i386/isa/clock.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)clock.c	7.2 (Berkeley) 5/12/91
- *	$Id: clock.c,v 1.97 1997/07/22 20:12:04 fsmp Exp $
+ *	$Id: clock.c,v 1.8 1997/08/21 04:51:12 smp Exp smp $
  */
 
 /*
@@ -66,7 +66,6 @@
 #include <machine/ipl.h>
 #ifdef APIC_IO
 #include <machine/smp.h>
-#include <machine/smptests.h>		/** NEW_STRATEGY (,SMP_TIMER_NC) */
 #endif /* APIC_IO */
 
 #include <i386/isa/icu.h>
@@ -74,6 +73,9 @@
 #include <i386/isa/rtc.h>
 #include <i386/isa/timerreg.h>
 
+#include <i386/isa/intr_machdep.h>
+#include <sys/interrupt.h>
+
 /*
  * 32-bit time_t's can't reach leap years before 1904 or after 2036, so we
  * can use a simple formula for leap years.
@@ -859,11 +861,6 @@ cpu_initclocks()
 	/* Finish initializing 8253 timer 0. */
 #ifdef APIC_IO
 
-#ifdef NEW_STRATEGY
-#ifdef SMP_TIMER_NC
-#error 'options SMP_TIMER_NC' no longer used, remove & reconfig.
-#endif	/** XXX SMP_TIMER_NC */
-
 	/* 1st look for ExtInt on pin 0 */
 	if (apic_int_type(0, 0) == 3) {
 		/*
@@ -897,33 +894,6 @@ cpu_initclocks()
 	else 
 		panic("neither pin 0 or pin 2 works for 8254");
 
-#else /** NEW_STRATEGY */
-
-	/* 8254 is traditionally on ISA IRQ0 */
-#if defined(SMP_TIMER_NC)
-	x = -1;
-#else
-	x = isa_apic_pin(0);
-#endif	/** XXX SMP_TIMER_NC */
-
-	if (x < 0) {
-		/* bummer, attempt to redirect thru the 8259 */
-		if (bootverbose)
-			printf("APIC missing 8254 connection\n");
-
-		/* allow 8254 timer to INTerrupt 8259 */
-		x = inb(IO_ICU1 + 1);		/* current mask in 8259 */
-		x &= ~1;			/* clear 8254 timer mask */
-		outb(IO_ICU1 + 1, x);		/* write new mask */
-
-		/* program IO APIC for type 3 INT on INT0 */
-		if (ext_int_setup(0, 0) < 0)
-			panic("8254 redirect impossible!");
-		x = 0;				/* 8259 is on 0 */
-	}
-
-#endif /** NEW_STRATEGY */
-
 	/* setup the vectors */
 	vec[x] = (u_int)vec8254;
 	Xintr8254 = (u_int)ivectors[x];
@@ -966,9 +936,11 @@ cpu_initclocks()
 	if (isa_apic_pin(8) != 8)
 		panic("APIC RTC != 8");
 #endif /* APIC_IO */
+
 	register_intr(/* irq */ 8, /* XXX id */ 1, /* flags */ 0,
 		      /* XXX */ (inthand2_t *)rtcintr, &stat_imask,
 		      /* unit */ 0);
+
 #ifdef APIC_IO
 	INTREN(APIC_IRQ8);
 #else
diff --git a/sys/i386/isa/isa.c b/sys/i386/isa/isa.c
index 4d14fff..55066a5 100644
--- a/sys/i386/isa/isa.c
+++ b/sys/i386/isa/isa.c
@@ -34,7 +34,7 @@
  * SUCH DAMAGE.
  *
  *	from: @(#)isa.c	7.2 (Berkeley) 5/13/91
- *	$Id: isa.c,v 1.99 1997/07/29 05:24:36 msmith Exp $
+ *	$Id: isa.c,v 1.2 1997/08/21 04:51:00 smp Exp smp $
  */
 
 /*
@@ -55,7 +55,6 @@
 #include <machine/md_var.h>
 #ifdef APIC_IO
 #include <machine/smp.h>
-#include <machine/apic.h>
 #endif /* APIC_IO */
 #include <vm/vm.h>
 #include <vm/vm_param.h>