Eliminate frequent silo overflows by restoring the TEST_LOPRIO code.

This code was eliminated when the PEND_INTS algorithm was added.  But it was
discovered that PEND_INTS only worsen latency for FAST_INTR() routines,
which can't be marked pending.

Noticed & debugged by:	dave adkins <adkin003@gold.tc.umn.edu>

8 files changed, 127 insertions, 23 deletions

diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
index 0a3bf40..14d0a8a 100644
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -33,7 +33,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: swtch.s,v 1.4 1997/07/30 22:51:11 smp Exp smp $
+ *	$Id: swtch.s,v 1.5 1997/08/04 17:17:29 smp Exp smp $
  */
 
 #include "npx.h"
@@ -46,6 +46,7 @@
 #ifdef SMP
 #include <machine/pmap.h>
 #include <machine/apic.h>
+#include <machine/smptests.h>		/** GRAB_LOPRIO */
 #endif /* SMP */
 
 #include "assym.s"
@@ -509,6 +510,13 @@ swtch_com:
 	movl	%eax,(%esp)
 
 #ifdef SMP
+#ifdef GRAB_LOPRIO				/* hold LOPRIO for INTs */
+#ifdef CHEAP_TPR
+	movl	$0, lapic_tpr
+#else
+	andl	$~APIC_TPR_PRIO, lapic_tpr
+#endif /** CHEAP_TPR */
+#endif /** GRAB_LOPRIO */
 	movl	_cpuid,%eax
 	movb	%al, P_ONCPU(%ecx)
 #endif /* SMP */
diff --git a/sys/amd64/amd64/swtch.s b/sys/amd64/amd64/swtch.s
index 0a3bf40..14d0a8a 100644
--- a/sys/amd64/amd64/swtch.s
+++ b/sys/amd64/amd64/swtch.s
@@ -33,7 +33,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: swtch.s,v 1.4 1997/07/30 22:51:11 smp Exp smp $
+ *	$Id: swtch.s,v 1.5 1997/08/04 17:17:29 smp Exp smp $
  */
 
 #include "npx.h"
@@ -46,6 +46,7 @@
 #ifdef SMP
 #include <machine/pmap.h>
 #include <machine/apic.h>
+#include <machine/smptests.h>		/** GRAB_LOPRIO */
 #endif /* SMP */
 
 #include "assym.s"
@@ -509,6 +510,13 @@ swtch_com:
 	movl	%eax,(%esp)
 
 #ifdef SMP
+#ifdef GRAB_LOPRIO				/* hold LOPRIO for INTs */
+#ifdef CHEAP_TPR
+	movl	$0, lapic_tpr
+#else
+	andl	$~APIC_TPR_PRIO, lapic_tpr
+#endif /** CHEAP_TPR */
+#endif /** GRAB_LOPRIO */
 	movl	_cpuid,%eax
 	movb	%al, P_ONCPU(%ecx)
 #endif /* SMP */
diff --git a/sys/amd64/include/apicreg.h b/sys/amd64/include/apicreg.h
index 8df09cd..50f8416 100644
--- a/sys/amd64/include/apicreg.h
+++ b/sys/amd64/include/apicreg.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: apic.h,v 1.6 1997/07/30 22:44:20 smp Exp smp $
+ *	$Id: apic.h,v 1.7 1997/08/04 17:20:39 smp Exp smp $
  */
 
 #ifndef _MACHINE_APIC_H_
@@ -221,7 +221,11 @@ typedef struct IOAPIC ioapic_t;
  */
 
 /* default level for TPR */
-#define LOPRIO_LEVEL		0x00000010	/* TPR of CPUs accepting INT */
+#define LOPRIO_LEVEL		0x00000010	/* TPR of CPU accepting INTs */
+
+#ifdef GRAB_LOPRIO
+#define ALLHWI_LEVEL		0x00000000	/* TPR of CPU grabbing INTs */
+#endif /** GRAB_LOPRIO */
 
 /* XXX these 2 don't really belong here... */
 #define COUNT_FIELD		0x00ffffff	/* count portion of the lock */
diff --git a/sys/i386/i386/mplock.s b/sys/i386/i386/mplock.s
index b5ad875..30f7c93 100644
--- a/sys/i386/i386/mplock.s
+++ b/sys/i386/i386/mplock.s
@@ -6,7 +6,7 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
- * $Id: mplock.s,v 1.12 1997/07/30 22:51:11 smp Exp smp $
+ * $Id: mplock.s,v 1.14 1997/08/04 17:19:17 smp Exp smp $
  *
  * Functions for locking between CPUs in a SMP system.
  *
@@ -22,7 +22,7 @@
  */
 
 #include <machine/asmacros.h>		/* miscellaneous asm macros */
-#include <machine/smptests.h>		/** TEST_LOPRIO */
+#include <machine/smptests.h>		/** GRAB_LOPRIO */
 #include <machine/apic.h>
 
 #include <i386/isa/intr_machdep.h>
@@ -40,8 +40,52 @@ _tryhits:        9938            2196d           44cc
 #define FREE_FIRST
 #define GLPROFILE
 
-#define	MAYBE_PUSHL_EAX	pushl	%eax
-#define	MAYBE_POPL_EAX	popl	%eax
+#ifdef GRAB_LOPRIO
+/*
+ * Claim LOWest PRIOrity, ie. attempt to grab ALL INTerrupts.
+ */
+
+/* location of saved TPR on stack */
+#define TPR_TARGET		12(%esp)
+
+/* we assumme that the 'reserved bits' can be written with zeros */
+#ifdef CHEAP_TPR
+
+/* after 1st acquire of lock we attempt to grab all hardware INTs */
+#define GRAB_HWI \
+	movl	$ALLHWI_LEVEL, TPR_TARGET	/* task prio to 'all HWI' */
+
+#define GRAB_HWI_2 \
+	movl	$ALLHWI_LEVEL, lapic_tpr	/* task prio to 'all HWI' */
+
+/* after last release of lock give up LOW PRIO (ie, arbitrate INTerrupts) */
+#define ARB_HWI \
+	movl	$LOPRIO_LEVEL, lapic_tpr	/* task prio to 'arbitrate' */
+
+#else /** CHEAP_TPR */
+
+#define GRAB_HWI \
+	andl	$~APIC_TPR_PRIO, TPR_TARGET	/* task prio to 'all HWI' */
+
+#define GRAB_HWI_2 \
+	andl	$~APIC_TPR_PRIO, lapic_tpr	/* task prio to 'all HWI' */
+
+#define ARB_HWI								\
+	movl	lapic_tpr, %eax ;		/* TPR */		\
+	andl	$~APIC_TPR_PRIO, %eax ;		/* clear TPR field */	\
+	orl	$LOPRIO_LEVEL, %eax ;		/* prio to arbitrate */	\
+	movl	%eax, lapic_tpr ;		/* set it */		\
+  	movl	(%edx), %eax			/* reload %eax with lock */
+
+#endif /** CHEAP_TPR */
+
+#else /** GRAB_LOPRIO */
+
+#define GRAB_HWI				/* nop */
+#define GRAB_HWI_2				/* nop */
+#define ARB_HWI					/* nop */
+
+#endif /** GRAB_LOPRIO */
 
 
 	.text
@@ -66,6 +110,7 @@ NON_GPROF_ENTRY(MPgetlock)
 #ifdef GLPROFILE
 	incl	_gethits2
 #endif /* GLPROFILE */
+	GRAB_HWI			/* 1st acquire, grab hw INTs */
 	ret
 2:
   	movl	(%edx), %eax		/* Try to see if we have it already */
@@ -120,6 +165,7 @@ NON_GPROF_ENTRY(MPgetlock)
 #else
 	jne	3f			/* ...do not collect $200 */
 #endif /* GLPROFILE */
+	GRAB_HWI			/* 1st acquire, grab hw INTs */
 	ret
 #ifdef GLPROFILE
 4:
@@ -154,6 +200,7 @@ NON_GPROF_ENTRY(MPtrylock)
 #ifdef GLPROFILE
 	incl	_tryhits2
 #endif /* GLPROFILE */
+	GRAB_HWI_2			/* 1st acquire, grab hw INTs */
 	movl	$1, %eax
 	ret
 1:
@@ -204,6 +251,7 @@ NON_GPROF_ENTRY(MPtrylock)
 #ifdef GLPROFILE
 	incl	_tryhits2
 #endif /* GLPROFILE */
+	GRAB_HWI_2			/* 1st acquire, grab hw INTs */
 	movl	$1, %eax
 	ret
 2:
@@ -230,6 +278,7 @@ NON_GPROF_ENTRY(MPrellock)
 	decl	%ecx			/* - new count is one less */
 	testl	$COUNT_FIELD, %ecx	/* - Unless it's zero... */
 	jnz	2f
+	ARB_HWI				/* last release, arbitrate hw INTs */
 	movl	$FREE_LOCK, %ecx	/* - In which case we release it */
 2:
 	lock
@@ -243,10 +292,20 @@ NON_GPROF_ENTRY(MPrellock)
  *  -----------------
  *  All registers preserved
  *
+ *  Stack (after call to _MPgetlock):
+ *	
+ *	&mp_lock	 4(%esp)
+ *	EFLAGS		 8(%esp)
+ *	local APIC TPR	12(%esp)
+ *	edx		16(%esp)
+ *	ecx		20(%esp)
+ *	eax		24(%esp)
  */
 
 NON_GPROF_ENTRY(get_mplock)
-	MAYBE_PUSHL_EAX
+	pushl	%eax
+	pushl	%ecx
+	pushl	%edx
 
 	/* block all HW INTs via Task Priority Register */
 #ifdef CHEAP_TPR
@@ -267,17 +326,15 @@ NON_GPROF_ENTRY(get_mplock)
 #endif /** CHEAP_TPR */
 	sti				/* allow IPI (and only IPI) INTs */
 1:
-	pushl	%ecx
-	pushl	%edx
 	pushl	$_mp_lock
 	call	_MPgetlock
 	add	$4, %esp
-	popl	%edx
-	popl	%ecx
 
 	popfl				/* restore original EFLAGS */
 	popl	lapic_tpr		/* restore TPR */
-	MAYBE_POPL_EAX
+	popl	%edx
+	popl	%ecx
+	popl	%eax
 	ret
 
 /***********************************************************************
@@ -285,6 +342,11 @@ NON_GPROF_ENTRY(get_mplock)
  *  -----------------
  *  no registers preserved, assummed the calling ISR does!
  *
+ *  Stack (after call to _MPgetlock):
+ *	
+ *	&mp_lock	 4(%esp)
+ *	EFLAGS		 8(%esp)
+ *	local APIC TPR	12(%esp)
  */
 
 NON_GPROF_ENTRY(get_isrlock)
@@ -350,7 +412,7 @@ NON_GPROF_ENTRY(try_isrlock)
  */
 
 NON_GPROF_ENTRY(rel_mplock)
-	MAYBE_PUSHL_EAX
+	pushl	%eax
 	pushl	%ecx
 	pushl	%edx
 	pushl	$_mp_lock
@@ -358,7 +420,7 @@ NON_GPROF_ENTRY(rel_mplock)
 	add	$4, %esp
 	popl	%edx
 	popl	%ecx
-	MAYBE_POPL_EAX
+	popl	%eax
 	ret
 
 /***********************************************************************
@@ -380,7 +442,7 @@ NON_GPROF_ENTRY(rel_isrlock)
 
 	.data
 	.globl _mp_lock
-	.align  4	/* mp_lock aligned on int boundary */
+	.align  2			/* mp_lock aligned on int boundary */
 _mp_lock:	.long	0		
 
 #ifdef GLPROFILE
diff --git a/sys/i386/i386/swtch.s b/sys/i386/i386/swtch.s
index 0a3bf40..14d0a8a 100644
--- a/sys/i386/i386/swtch.s
+++ b/sys/i386/i386/swtch.s
@@ -33,7 +33,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: swtch.s,v 1.4 1997/07/30 22:51:11 smp Exp smp $
+ *	$Id: swtch.s,v 1.5 1997/08/04 17:17:29 smp Exp smp $
  */
 
 #include "npx.h"
@@ -46,6 +46,7 @@
 #ifdef SMP
 #include <machine/pmap.h>
 #include <machine/apic.h>
+#include <machine/smptests.h>		/** GRAB_LOPRIO */
 #endif /* SMP */
 
 #include "assym.s"
@@ -509,6 +510,13 @@ swtch_com:
 	movl	%eax,(%esp)
 
 #ifdef SMP
+#ifdef GRAB_LOPRIO				/* hold LOPRIO for INTs */
+#ifdef CHEAP_TPR
+	movl	$0, lapic_tpr
+#else
+	andl	$~APIC_TPR_PRIO, lapic_tpr
+#endif /** CHEAP_TPR */
+#endif /** GRAB_LOPRIO */
 	movl	_cpuid,%eax
 	movb	%al, P_ONCPU(%ecx)
 #endif /* SMP */
diff --git a/sys/i386/include/apic.h b/sys/i386/include/apic.h
index 8df09cd..50f8416 100644
--- a/sys/i386/include/apic.h
+++ b/sys/i386/include/apic.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: apic.h,v 1.6 1997/07/30 22:44:20 smp Exp smp $
+ *	$Id: apic.h,v 1.7 1997/08/04 17:20:39 smp Exp smp $
  */
 
 #ifndef _MACHINE_APIC_H_
@@ -221,7 +221,11 @@ typedef struct IOAPIC ioapic_t;
  */
 
 /* default level for TPR */
-#define LOPRIO_LEVEL		0x00000010	/* TPR of CPUs accepting INT */
+#define LOPRIO_LEVEL		0x00000010	/* TPR of CPU accepting INTs */
+
+#ifdef GRAB_LOPRIO
+#define ALLHWI_LEVEL		0x00000000	/* TPR of CPU grabbing INTs */
+#endif /** GRAB_LOPRIO */
 
 /* XXX these 2 don't really belong here... */
 #define COUNT_FIELD		0x00ffffff	/* count portion of the lock */
diff --git a/sys/i386/include/apicreg.h b/sys/i386/include/apicreg.h
index 8df09cd..50f8416 100644
--- a/sys/i386/include/apicreg.h
+++ b/sys/i386/include/apicreg.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: apic.h,v 1.6 1997/07/30 22:44:20 smp Exp smp $
+ *	$Id: apic.h,v 1.7 1997/08/04 17:20:39 smp Exp smp $
  */
 
 #ifndef _MACHINE_APIC_H_
@@ -221,7 +221,11 @@ typedef struct IOAPIC ioapic_t;
  */
 
 /* default level for TPR */
-#define LOPRIO_LEVEL		0x00000010	/* TPR of CPUs accepting INT */
+#define LOPRIO_LEVEL		0x00000010	/* TPR of CPU accepting INTs */
+
+#ifdef GRAB_LOPRIO
+#define ALLHWI_LEVEL		0x00000000	/* TPR of CPU grabbing INTs */
+#endif /** GRAB_LOPRIO */
 
 /* XXX these 2 don't really belong here... */
 #define COUNT_FIELD		0x00ffffff	/* count portion of the lock */
diff --git a/sys/i386/include/smptests.h b/sys/i386/include/smptests.h
index bb44570..09607e2 100644
--- a/sys/i386/include/smptests.h
+++ b/sys/i386/include/smptests.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: smptests.h,v 1.19 1997/07/30 22:44:20 smp Exp smp $
+ *	$Id: smptests.h,v 1.20 1997/08/04 17:20:39 smp Exp smp $
  */
 
 #ifndef _MACHINE_SMPTESTS_H_
@@ -66,6 +66,12 @@
 
 
 /*
+ * Portions of the old TEST_LOPRIO code, back from the grave!
+ */
+#define GRAB_LOPRIO
+
+
+/*
  * 1st attempt to use the 'ExtInt' connected 8259 to attach 8254 timer.
  * failing that, attempt to attach 8254 timer via direct APIC pin.
  * failing that, panic!