Convert the interrupt queue from an array to a linked list. Implement

intr_dequeue in asm so that it can easily be modified to do light weight
context switching.

8 files changed, 383 insertions, 200 deletions

diff --git a/sys/sparc64/include/intr_machdep.h b/sys/sparc64/include/intr_machdep.h
index c8e36ca..a08a9fc 100644
--- a/sys/sparc64/include/intr_machdep.h
+++ b/sys/sparc64/include/intr_machdep.h
@@ -29,11 +29,12 @@
 #ifndef	_MACHINE_INTR_MACHDEP_H_
 #define	_MACHINE_INTR_MACHDEP_H_
 
-#define	NPIL		(1 << 4)
-#define	NIV		(1 << 11)
+#define	IRSR_BUSY	(1 << 5)
 
-#define	IQ_SIZE		(NPIL * 2)
-#define	IQ_MASK		(IQ_SIZE - 1)
+#define	PIL_MAX		(1 << 4)
+#define	IV_MAX		(1 << 11)
+
+#define	IR_FREE		(PIL_MAX * 2)
 
 #define	IH_SHIFT	PTR_SHIFT
 #define	IQE_SHIFT	5
@@ -47,27 +48,19 @@
 #define	PIL_FAST	13	/* fast interrupts */
 #define	PIL_TICK	14
 
-struct trapframe;
-
 typedef	void ih_func_t(struct trapframe *);
 typedef	void iv_func_t(void *);
 
-struct iqe {
-	u_int	iqe_tag;
-	u_int	iqe_pri;
-	u_long	iqe_vec;
-	iv_func_t *iqe_func;
-	void	*iqe_arg;
-};
+struct ithd;
 
-struct intr_queue {
-	struct	iqe iq_queue[IQ_SIZE];	/* must be first */
-	u_long	iq_head;
-	u_long	iq_tail;
+struct intr_request {
+	struct	intr_request *ir_next;
+	iv_func_t *ir_func;
+	void	*ir_arg;
+	u_int	ir_vec;
+	u_int	ir_pri;
 };
 
-struct ithd;
-
 struct intr_vector {
 	iv_func_t *iv_func;
 	void	*iv_arg;
diff --git a/sys/sparc64/include/pcpu.h b/sys/sparc64/include/pcpu.h
index 6f3d646..24d16cf 100644
--- a/sys/sparc64/include/pcpu.h
+++ b/sys/sparc64/include/pcpu.h
@@ -45,7 +45,10 @@ struct vmspace;
  * point at the globaldata structure.
  */
 #define	PCPU_MD_FIELDS							\
-	struct	intr_queue pc_iq;		/* interrupt queue */	\
+	struct	intr_request pc_irpool[IR_FREE];			\
+	struct	intr_request *pc_irhead;				\
+	struct	intr_request **pc_irtail;				\
+	struct	intr_request *pc_irfree;				\
 	struct	vmspace *pc_vmspace;					\
 	vm_offset_t pc_addr;						\
 	u_int 	pc_mid;							\
diff --git a/sys/sparc64/sparc64/exception.S b/sys/sparc64/sparc64/exception.S
index eb18495..1822aa4 100644
--- a/sys/sparc64/sparc64/exception.S
+++ b/sys/sparc64/sparc64/exception.S
@@ -358,10 +358,10 @@ ENTRY(rsf_fatal)
 	sir
 END(rsf_fatal)
 
-	.comm	intrnames, NIV * 8
+	.comm	intrnames, IV_MAX * 8
 	.comm	eintrnames, 0
 
-	.comm	intrcnt, NIV * 8
+	.comm	intrcnt, IV_MAX * 8
 	.comm	eintrcnt, 0
 
 /*
@@ -538,8 +538,12 @@ END(tl0_sfsr_trap)
 	INTR_LEVEL(0)
 	.endm
 
-	.macro	tl0_intr_vector
-	b,a	%xcc, intr_enqueue
+	.macro	intr_vector
+	ldxa	[%g0] ASI_INTR_RECEIVE, %g1
+	andcc	%g1, IRSR_BUSY, %g0
+	bnz,a,pt %xcc, intr_enqueue
+	 nop
+	sir
 	.align	32
 	.endm
 
@@ -1136,109 +1140,209 @@ END(tl1_sfsr_trap)
 	INTR_LEVEL(1)
 	.endm
 
-	.macro	tl1_intr_vector
-	b,a	intr_enqueue
-	.align	32
-	.endm
+ENTRY(intr_dequeue)
+	save	%sp, -CCFSZ, %sp
+
+1:	ldx	[PCPU(IRHEAD)], %l0
+	brnz,a,pt %l0, 2f
+	 nop
+
+	ret
+	 restore
+
+2:	wrpr	%g0, PSTATE_NORMAL, %pstate
+
+	ldx	[%l0 + IR_NEXT], %l1
+	brnz,pt	%l1, 3f
+	 stx	%l1, [PCPU(IRHEAD)]
+	PCPU_ADDR(IRHEAD, %l1)
+	stx	%l1, [PCPU(IRTAIL)]
 
+3:	ldx	[%l0 + IR_FUNC], %o0
+	ldx	[%l0 + IR_ARG], %o1
+	ldx	[%l0 + IR_VEC], %o2
+
+	ldx	[PCPU(IRFREE)], %l1
+	stx	%l1, [%l0 + IR_NEXT]
+	stx	%l0, [PCPU(IRFREE)]
+
+	wrpr	%g0, PSTATE_KERNEL, %pstate
+
+	call	%o0
+	 mov	%o1, %o0
+	ba,a	%xcc, 1b
+	 nop
+END(intr_dequeue)
+
+/*
+ * Handle a vectored interrupt.
+ *
+ * This is either a data bearing mondo vector interrupt, or a cross trap
+ * request from another cpu.  In either case the hardware supplies an
+ * interrupt packet, in the form of 3 data words which are read from internal
+ * registers.  A data bearing mondo vector packet consists of an interrupt
+ * number in the first data word, and zero in 2nd and 3rd.  We use the
+ * interrupt number to find the function, argument and priority from the
+ * intr_vector table, allocate and fill in an intr_request from the per-cpu
+ * free list, link it onto the per-cpu active list and finally post a softint
+ * at the desired priority.  Cross trap requests come in 2 forms, direct
+ * and queued.  Direct requests are distinguished by the first data word
+ * being zero.  The 2nd data word carries a function to call and the 3rd
+ * an argument to pass.  The function is jumped to directly.  It executes
+ * in nucleus context on interrupt globals and with all interrupts disabled,
+ * therefore it must be fast, and the things that it can do are limited.
+ * Queued cross trap requests are handled much like mondo vectors, except
+ * that the function, argument and priority are contained in the interrupt
+ * packet itself.  They are distinguished by the upper 4 bits of the data
+ * word being non-zero, which specifies the priority of the softint to
+ * deliver.
+ *
+ * Register usage:
+ *	%g1 - pointer to intr_request
+ *	%g2 - pointer to intr_vector, temp once required data is loaded
+ *	%g3 - interrupt number for mondo vectors, unused otherwise
+ *	%g4 - function, from the interrupt packet for cross traps, or
+ *	      loaded from the interrupt registers for mondo vecors
+ *	%g5 - argument, as above for %g4
+ *	%g6 - softint priority
+ */
 ENTRY(intr_enqueue)
 	/*
 	 * Load the interrupt packet from the hardware.
 	 */
 	wr	%g0, ASI_SDB_INTR_R, %asi
-	ldxa	[%g0] ASI_INTR_RECEIVE, %g2
 	ldxa	[%g0 + AA_SDB_INTR_D0] %asi, %g3
 	ldxa	[%g0 + AA_SDB_INTR_D1] %asi, %g4
 	ldxa	[%g0 + AA_SDB_INTR_D2] %asi, %g5
 	stxa	%g0, [%g0] ASI_INTR_RECEIVE
 	membar	#Sync
 
+#if KTR_COMPILE & KTR_INTR
+	CATR(KTR_INTR, "intr_enqueue: data=%#lx %#lx %#lx"
+	    , %g1, %g2, %g6, 7, 8, 9)
+	stx	%g3, [%g1 + KTR_PARM1]
+	stx	%g4, [%g1 + KTR_PARM2]
+	stx	%g5, [%g1 + KTR_PARM3]
+9:
+#endif
+
 	/*
-	 * If the second data word is present it points to code to execute
-	 * directly.  Jump to it.
+	 * If the first data word is zero this is a direct cross trap request.
+	 * The 2nd word points to code to execute and the 3rd is an argument
+	 * to pass.  Jump to it.
 	 */
-	brz,a,pt %g4, 1f
+	brnz,a,pt %g3, 1f
 	 nop
+
+#if KTR_COMPILE & KTR_INTR
+	CATR(KTR_INTR, "intr_enqueue: direct ipi func=%#lx arg=%#lx"
+	    , %g1, %g2, %g6, 7, 8, 9)
+	stx	%g4, [%g1 + KTR_PARM1]
+	stx	%g5, [%g1 + KTR_PARM2]
+9:
+#endif
+
 	jmpl	%g4, %g0
 	 nop
+	/* NOTREACHED */
 
 	/*
-	 * Find the head of the queue and advance it.
+	 * If the high 4 bits of the 1st data word are non-zero, this is a
+	 * queued cross trap request to be delivered as a softint.  The high
+	 * 4 bits of the 1st data word specify a priority, and the 2nd and
+	 * 3rd a function and argument.
 	 */
-1:	ldx	[PCPU(IQ) + IQ_HEAD], %g1
-	add	%g1, 1, %g6
-	and	%g6, IQ_MASK, %g6
-	stx	%g6, [PCPU(IQ) + IQ_HEAD]
+1:	srlx	%g3, 60, %g6
+	brnz,a,pn %g6, 2f
+	 clr	%g3
 
 	/*
-	 * Find the iqe.
+	 * Find the function, argument and desired priority from the
+	 * intr_vector table.
 	 */
-	sllx	%g1, IQE_SHIFT, %g1
-	add	%g1, PCPU_REG, %g1
-	add	%g1, PC_IQ, %g1
+	SET(intr_vectors, %g4, %g2)
+	sllx	%g3, IV_SHIFT, %g4
+	add	%g2, %g4, %g2
 
-	/*
-	 * Store the tag and first data word in the iqe.  These are always
-	 * valid.
-	 */
-	stw	%g2, [%g1 + IQE_TAG]
-	stx	%g3, [%g1 + IQE_VEC]
-
-#ifdef INVARIANTS
-	/*
-	 * If the new head is the same as the tail, the next interrupt will
-	 * overwrite unserviced packets.  This is bad.
-	 */
-	ldx	[PCPU(IQ) + IQ_TAIL], %g2
-	cmp	%g2, %g6
-	be	%xcc, 2f
-	 nop
+#if KTR_COMPILE & KTR_INTR
+	CATR(KTR_INTR, "intr_enqueue: mondo vector func=%#lx arg=%#lx pri=%#lx"
+	    , %g4, %g5, %g6, 7, 8, 9)
+	ldx	[%g2 + IV_FUNC], %g5
+	stx	%g5, [%g4 + KTR_PARM1]
+	ldx	[%g2 + IV_ARG], %g5
+	stx	%g5, [%g4 + KTR_PARM2]
+	ldx	[%g2 + IV_PRI], %g5
+	stx	%g5, [%g4 + KTR_PARM3]
+9:
 #endif
 
-	/*
-	 * Load the function, argument and priority and store them in the iqe.
-	 */
-	sllx	%g3, IV_SHIFT, %g3
-	SET(intr_vectors, %g6, %g2)
-	add	%g2, %g3, %g2
 	ldx	[%g2 + IV_FUNC], %g4
 	ldx	[%g2 + IV_ARG], %g5
 	lduw	[%g2 + IV_PRI], %g6
-	stx	%g4, [%g1 + IQE_FUNC]
-	stx	%g5, [%g1 + IQE_ARG]
-	stw	%g6, [%g1 + IQE_PRI]
 
+	ba,a	%xcc, 3f
+	 nop
+
+	/*
+	 * Get a intr_request from the free list.  There should always be one
+	 * unless we are getting an interrupt storm from stray interrupts, in
+	 * which case the we will deference a NULL pointer and panic.
+	 */
+2:
 #if KTR_COMPILE & KTR_INTR
-	CATR(KTR_INTR, "intr_enqueue: head=%d tail=%d pri=%p tag=%#x vec=%#x"
-	    , %g2, %g3, %g4, 7, 8, 9)
-	ldx	[PCPU(IQ) + IQ_HEAD], %g3
-	stx	%g3, [%g2 + KTR_PARM1]
-	ldx	[PCPU(IQ) + IQ_TAIL], %g3
-	stx	%g3, [%g2 + KTR_PARM2]
-	lduw	[%g1 + IQE_PRI], %g3
-	stx	%g3, [%g2 + KTR_PARM3]
-	lduw	[%g1 + IQE_TAG], %g3
-	stx	%g3, [%g2 + KTR_PARM4]
-	ldx	[%g1 + IQE_VEC], %g3
-	stx	%g3, [%g2 + KTR_PARM5]
+	CATR(KTR_INTR, "intr_enqueue: queued ipi func=%#lx arg=%#lx pri=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	stx	%g4, [%g1 + KTR_PARM1]
+	stx	%g5, [%g1 + KTR_PARM2]
+	stx	%g6, [%g1 + KTR_PARM3]
 9:
+	clr	%g3
 #endif
 
+3:
+	ldx	[PCPU(IRFREE)], %g1
+	ldx	[%g1 + IR_NEXT], %g2
+	stx	%g2, [PCPU(IRFREE)]
+
+	/*
+	 * Store the vector number, function, argument and priority.
+	 */
+	stw	%g3, [%g1 + IR_VEC]
+	stx	%g4, [%g1 + IR_FUNC]
+	stx	%g5, [%g1 + IR_ARG]
+	stw	%g6, [%g1 + IR_PRI]
+
+	/*
+	 * Link it onto the end of the active list.
+	 */
+	stx	%g0, [%g1 + IR_NEXT]
+	ldx	[PCPU(IRTAIL)], %g4
+	stx	%g1, [%g4]
+	add	%g1, IR_NEXT, %g1
+	stx	%g1, [PCPU(IRTAIL)]
+
 	/*
 	 * Trigger a softint at the level indicated by the priority.
 	 */
 	mov	1, %g1
 	sllx	%g1, %g6, %g1
-	wr	%g1, 0, %asr20
 
-	retry
+#if KTR_COMPILE & KTR_INTR
+	CATR(KTR_INTR, "intr_enqueue: softint pil=%#lx pri=%#lx mask=%#lx"
+	    , %g2, %g3, %g4, 7, 8, 9)
+	rdpr	%pil, %g3
+	stx	%g3, [%g2 + KTR_PARM1]
+	stx	%g6, [%g2 + KTR_PARM2]
+	stx	%g1, [%g2 + KTR_PARM3]
+9:
+#endif
+
+	wr	%g1, 0, %asr20
 
-#ifdef INVARIANTS
 	/*
-	 * The interrupt queue is about to overflow.  We are in big trouble.
+	 * Done, retry the instruction.
 	 */
-2:	sir
-#endif
+	retry
 END(intr_enqueue)
 
 	.macro	tl1_immu_miss
@@ -1759,7 +1863,7 @@ tl0_intr_level:
 	tl0_intr_level					! 0x41-0x4f
 	tl0_reserved	16				! 0x50-0x5f
 tl0_intr_vector:
-	tl0_intr_vector					! 0x60
+	intr_vector					! 0x60
 tl0_watch_phys:
 	tl0_gen		T_PA_WATCHPOINT			! 0x61
 tl0_watch_virt:
@@ -1859,7 +1963,7 @@ tl1_intr_level:
 	tl1_intr_level					! 0x241-0x24f
 	tl1_reserved	16				! 0x250-0x25f
 tl1_intr_vector:
-	tl1_intr_vector					! 0x260
+	intr_vector					! 0x260
 tl1_watch_phys:
 	tl1_gen		T_PA_WATCHPOINT			! 0x261
 tl1_watch_virt:
@@ -2432,8 +2536,7 @@ ENTRY(tl1_trap)
 	    , %g1, %g2, %g3, 7, 8, 9)
 	ldx	[PCPU(CURTHREAD)], %g2
 	stx	%g2, [%g1 + KTR_PARM1]
-	andn	%o0, T_KERNEL, %g2
-	stx	%g2, [%g1 + KTR_PARM2]
+	stx	%o0, [%g1 + KTR_PARM2]
 	stx	%l3, [%g1 + KTR_PARM3]
 	stx	%l1, [%g1 + KTR_PARM4]
 	stx	%i6, [%g1 + KTR_PARM5]
diff --git a/sys/sparc64/sparc64/exception.s b/sys/sparc64/sparc64/exception.s
index eb18495..1822aa4 100644
--- a/sys/sparc64/sparc64/exception.s
+++ b/sys/sparc64/sparc64/exception.s
@@ -358,10 +358,10 @@ ENTRY(rsf_fatal)
 	sir
 END(rsf_fatal)
 
-	.comm	intrnames, NIV * 8
+	.comm	intrnames, IV_MAX * 8
 	.comm	eintrnames, 0
 
-	.comm	intrcnt, NIV * 8
+	.comm	intrcnt, IV_MAX * 8
 	.comm	eintrcnt, 0
 
 /*
@@ -538,8 +538,12 @@ END(tl0_sfsr_trap)
 	INTR_LEVEL(0)
 	.endm
 
-	.macro	tl0_intr_vector
-	b,a	%xcc, intr_enqueue
+	.macro	intr_vector
+	ldxa	[%g0] ASI_INTR_RECEIVE, %g1
+	andcc	%g1, IRSR_BUSY, %g0
+	bnz,a,pt %xcc, intr_enqueue
+	 nop
+	sir
 	.align	32
 	.endm
 
@@ -1136,109 +1140,209 @@ END(tl1_sfsr_trap)
 	INTR_LEVEL(1)
 	.endm
 
-	.macro	tl1_intr_vector
-	b,a	intr_enqueue
-	.align	32
-	.endm
+ENTRY(intr_dequeue)
+	save	%sp, -CCFSZ, %sp
+
+1:	ldx	[PCPU(IRHEAD)], %l0
+	brnz,a,pt %l0, 2f
+	 nop
+
+	ret
+	 restore
+
+2:	wrpr	%g0, PSTATE_NORMAL, %pstate
+
+	ldx	[%l0 + IR_NEXT], %l1
+	brnz,pt	%l1, 3f
+	 stx	%l1, [PCPU(IRHEAD)]
+	PCPU_ADDR(IRHEAD, %l1)
+	stx	%l1, [PCPU(IRTAIL)]
 
+3:	ldx	[%l0 + IR_FUNC], %o0
+	ldx	[%l0 + IR_ARG], %o1
+	ldx	[%l0 + IR_VEC], %o2
+
+	ldx	[PCPU(IRFREE)], %l1
+	stx	%l1, [%l0 + IR_NEXT]
+	stx	%l0, [PCPU(IRFREE)]
+
+	wrpr	%g0, PSTATE_KERNEL, %pstate
+
+	call	%o0
+	 mov	%o1, %o0
+	ba,a	%xcc, 1b
+	 nop
+END(intr_dequeue)
+
+/*
+ * Handle a vectored interrupt.
+ *
+ * This is either a data bearing mondo vector interrupt, or a cross trap
+ * request from another cpu.  In either case the hardware supplies an
+ * interrupt packet, in the form of 3 data words which are read from internal
+ * registers.  A data bearing mondo vector packet consists of an interrupt
+ * number in the first data word, and zero in 2nd and 3rd.  We use the
+ * interrupt number to find the function, argument and priority from the
+ * intr_vector table, allocate and fill in an intr_request from the per-cpu
+ * free list, link it onto the per-cpu active list and finally post a softint
+ * at the desired priority.  Cross trap requests come in 2 forms, direct
+ * and queued.  Direct requests are distinguished by the first data word
+ * being zero.  The 2nd data word carries a function to call and the 3rd
+ * an argument to pass.  The function is jumped to directly.  It executes
+ * in nucleus context on interrupt globals and with all interrupts disabled,
+ * therefore it must be fast, and the things that it can do are limited.
+ * Queued cross trap requests are handled much like mondo vectors, except
+ * that the function, argument and priority are contained in the interrupt
+ * packet itself.  They are distinguished by the upper 4 bits of the data
+ * word being non-zero, which specifies the priority of the softint to
+ * deliver.
+ *
+ * Register usage:
+ *	%g1 - pointer to intr_request
+ *	%g2 - pointer to intr_vector, temp once required data is loaded
+ *	%g3 - interrupt number for mondo vectors, unused otherwise
+ *	%g4 - function, from the interrupt packet for cross traps, or
+ *	      loaded from the interrupt registers for mondo vecors
+ *	%g5 - argument, as above for %g4
+ *	%g6 - softint priority
+ */
 ENTRY(intr_enqueue)
 	/*
 	 * Load the interrupt packet from the hardware.
 	 */
 	wr	%g0, ASI_SDB_INTR_R, %asi
-	ldxa	[%g0] ASI_INTR_RECEIVE, %g2
 	ldxa	[%g0 + AA_SDB_INTR_D0] %asi, %g3
 	ldxa	[%g0 + AA_SDB_INTR_D1] %asi, %g4
 	ldxa	[%g0 + AA_SDB_INTR_D2] %asi, %g5
 	stxa	%g0, [%g0] ASI_INTR_RECEIVE
 	membar	#Sync
 
+#if KTR_COMPILE & KTR_INTR
+	CATR(KTR_INTR, "intr_enqueue: data=%#lx %#lx %#lx"
+	    , %g1, %g2, %g6, 7, 8, 9)
+	stx	%g3, [%g1 + KTR_PARM1]
+	stx	%g4, [%g1 + KTR_PARM2]
+	stx	%g5, [%g1 + KTR_PARM3]
+9:
+#endif
+
 	/*
-	 * If the second data word is present it points to code to execute
-	 * directly.  Jump to it.
+	 * If the first data word is zero this is a direct cross trap request.
+	 * The 2nd word points to code to execute and the 3rd is an argument
+	 * to pass.  Jump to it.
 	 */
-	brz,a,pt %g4, 1f
+	brnz,a,pt %g3, 1f
 	 nop
+
+#if KTR_COMPILE & KTR_INTR
+	CATR(KTR_INTR, "intr_enqueue: direct ipi func=%#lx arg=%#lx"
+	    , %g1, %g2, %g6, 7, 8, 9)
+	stx	%g4, [%g1 + KTR_PARM1]
+	stx	%g5, [%g1 + KTR_PARM2]
+9:
+#endif
+
 	jmpl	%g4, %g0
 	 nop
+	/* NOTREACHED */
 
 	/*
-	 * Find the head of the queue and advance it.
+	 * If the high 4 bits of the 1st data word are non-zero, this is a
+	 * queued cross trap request to be delivered as a softint.  The high
+	 * 4 bits of the 1st data word specify a priority, and the 2nd and
+	 * 3rd a function and argument.
 	 */
-1:	ldx	[PCPU(IQ) + IQ_HEAD], %g1
-	add	%g1, 1, %g6
-	and	%g6, IQ_MASK, %g6
-	stx	%g6, [PCPU(IQ) + IQ_HEAD]
+1:	srlx	%g3, 60, %g6
+	brnz,a,pn %g6, 2f
+	 clr	%g3
 
 	/*
-	 * Find the iqe.
+	 * Find the function, argument and desired priority from the
+	 * intr_vector table.
 	 */
-	sllx	%g1, IQE_SHIFT, %g1
-	add	%g1, PCPU_REG, %g1
-	add	%g1, PC_IQ, %g1
+	SET(intr_vectors, %g4, %g2)
+	sllx	%g3, IV_SHIFT, %g4
+	add	%g2, %g4, %g2
 
-	/*
-	 * Store the tag and first data word in the iqe.  These are always
-	 * valid.
-	 */
-	stw	%g2, [%g1 + IQE_TAG]
-	stx	%g3, [%g1 + IQE_VEC]
-
-#ifdef INVARIANTS
-	/*
-	 * If the new head is the same as the tail, the next interrupt will
-	 * overwrite unserviced packets.  This is bad.
-	 */
-	ldx	[PCPU(IQ) + IQ_TAIL], %g2
-	cmp	%g2, %g6
-	be	%xcc, 2f
-	 nop
+#if KTR_COMPILE & KTR_INTR
+	CATR(KTR_INTR, "intr_enqueue: mondo vector func=%#lx arg=%#lx pri=%#lx"
+	    , %g4, %g5, %g6, 7, 8, 9)
+	ldx	[%g2 + IV_FUNC], %g5
+	stx	%g5, [%g4 + KTR_PARM1]
+	ldx	[%g2 + IV_ARG], %g5
+	stx	%g5, [%g4 + KTR_PARM2]
+	ldx	[%g2 + IV_PRI], %g5
+	stx	%g5, [%g4 + KTR_PARM3]
+9:
 #endif
 
-	/*
-	 * Load the function, argument and priority and store them in the iqe.
-	 */
-	sllx	%g3, IV_SHIFT, %g3
-	SET(intr_vectors, %g6, %g2)
-	add	%g2, %g3, %g2
 	ldx	[%g2 + IV_FUNC], %g4
 	ldx	[%g2 + IV_ARG], %g5
 	lduw	[%g2 + IV_PRI], %g6
-	stx	%g4, [%g1 + IQE_FUNC]
-	stx	%g5, [%g1 + IQE_ARG]
-	stw	%g6, [%g1 + IQE_PRI]
 
+	ba,a	%xcc, 3f
+	 nop
+
+	/*
+	 * Get a intr_request from the free list.  There should always be one
+	 * unless we are getting an interrupt storm from stray interrupts, in
+	 * which case the we will deference a NULL pointer and panic.
+	 */
+2:
 #if KTR_COMPILE & KTR_INTR
-	CATR(KTR_INTR, "intr_enqueue: head=%d tail=%d pri=%p tag=%#x vec=%#x"
-	    , %g2, %g3, %g4, 7, 8, 9)
-	ldx	[PCPU(IQ) + IQ_HEAD], %g3
-	stx	%g3, [%g2 + KTR_PARM1]
-	ldx	[PCPU(IQ) + IQ_TAIL], %g3
-	stx	%g3, [%g2 + KTR_PARM2]
-	lduw	[%g1 + IQE_PRI], %g3
-	stx	%g3, [%g2 + KTR_PARM3]
-	lduw	[%g1 + IQE_TAG], %g3
-	stx	%g3, [%g2 + KTR_PARM4]
-	ldx	[%g1 + IQE_VEC], %g3
-	stx	%g3, [%g2 + KTR_PARM5]
+	CATR(KTR_INTR, "intr_enqueue: queued ipi func=%#lx arg=%#lx pri=%#lx"
+	    , %g1, %g2, %g3, 7, 8, 9)
+	stx	%g4, [%g1 + KTR_PARM1]
+	stx	%g5, [%g1 + KTR_PARM2]
+	stx	%g6, [%g1 + KTR_PARM3]
 9:
+	clr	%g3
 #endif
 
+3:
+	ldx	[PCPU(IRFREE)], %g1
+	ldx	[%g1 + IR_NEXT], %g2
+	stx	%g2, [PCPU(IRFREE)]
+
+	/*
+	 * Store the vector number, function, argument and priority.
+	 */
+	stw	%g3, [%g1 + IR_VEC]
+	stx	%g4, [%g1 + IR_FUNC]
+	stx	%g5, [%g1 + IR_ARG]
+	stw	%g6, [%g1 + IR_PRI]
+
+	/*
+	 * Link it onto the end of the active list.
+	 */
+	stx	%g0, [%g1 + IR_NEXT]
+	ldx	[PCPU(IRTAIL)], %g4
+	stx	%g1, [%g4]
+	add	%g1, IR_NEXT, %g1
+	stx	%g1, [PCPU(IRTAIL)]
+
 	/*
 	 * Trigger a softint at the level indicated by the priority.
 	 */
 	mov	1, %g1
 	sllx	%g1, %g6, %g1
-	wr	%g1, 0, %asr20
 
-	retry
+#if KTR_COMPILE & KTR_INTR
+	CATR(KTR_INTR, "intr_enqueue: softint pil=%#lx pri=%#lx mask=%#lx"
+	    , %g2, %g3, %g4, 7, 8, 9)
+	rdpr	%pil, %g3
+	stx	%g3, [%g2 + KTR_PARM1]
+	stx	%g6, [%g2 + KTR_PARM2]
+	stx	%g1, [%g2 + KTR_PARM3]
+9:
+#endif
+
+	wr	%g1, 0, %asr20
 
-#ifdef INVARIANTS
 	/*
-	 * The interrupt queue is about to overflow.  We are in big trouble.
+	 * Done, retry the instruction.
 	 */
-2:	sir
-#endif
+	retry
 END(intr_enqueue)
 
 	.macro	tl1_immu_miss
@@ -1759,7 +1863,7 @@ tl0_intr_level:
 	tl0_intr_level					! 0x41-0x4f
 	tl0_reserved	16				! 0x50-0x5f
 tl0_intr_vector:
-	tl0_intr_vector					! 0x60
+	intr_vector					! 0x60
 tl0_watch_phys:
 	tl0_gen		T_PA_WATCHPOINT			! 0x61
 tl0_watch_virt:
@@ -1859,7 +1963,7 @@ tl1_intr_level:
 	tl1_intr_level					! 0x241-0x24f
 	tl1_reserved	16				! 0x250-0x25f
 tl1_intr_vector:
-	tl1_intr_vector					! 0x260
+	intr_vector					! 0x260
 tl1_watch_phys:
 	tl1_gen		T_PA_WATCHPOINT			! 0x261
 tl1_watch_virt:
@@ -2432,8 +2536,7 @@ ENTRY(tl1_trap)
 	    , %g1, %g2, %g3, 7, 8, 9)
 	ldx	[PCPU(CURTHREAD)], %g2
 	stx	%g2, [%g1 + KTR_PARM1]
-	andn	%o0, T_KERNEL, %g2
-	stx	%g2, [%g1 + KTR_PARM2]
+	stx	%o0, [%g1 + KTR_PARM2]
 	stx	%l3, [%g1 + KTR_PARM3]
 	stx	%l1, [%g1 + KTR_PARM4]
 	stx	%i6, [%g1 + KTR_PARM5]
diff --git a/sys/sparc64/sparc64/genassym.c b/sys/sparc64/sparc64/genassym.c
index 7c975f6..d618a48 100644
--- a/sys/sparc64/sparc64/genassym.c
+++ b/sys/sparc64/sparc64/genassym.c
@@ -177,7 +177,9 @@ ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
 ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
 ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid));
 ASSYM(PC_CPUMASK, offsetof(struct pcpu, pc_cpumask));
-ASSYM(PC_IQ, offsetof(struct pcpu, pc_iq));
+ASSYM(PC_IRHEAD, offsetof(struct pcpu, pc_irhead));
+ASSYM(PC_IRTAIL, offsetof(struct pcpu, pc_irtail));
+ASSYM(PC_IRFREE, offsetof(struct pcpu, pc_irfree));
 ASSYM(PC_MID, offsetof(struct pcpu, pc_mid));
 ASSYM(PC_TLB_CTX, offsetof(struct pcpu, pc_tlb_ctx));
 ASSYM(PC_TLB_CTX_MAX, offsetof(struct pcpu, pc_tlb_ctx_max));
@@ -187,16 +189,13 @@ ASSYM(PC_SIZEOF, sizeof(struct pcpu));
 
 ASSYM(IH_SHIFT, IH_SHIFT);
 
-ASSYM(IQ_MASK, IQ_MASK);
-ASSYM(IQ_HEAD, offsetof(struct intr_queue, iq_head));
-ASSYM(IQ_TAIL, offsetof(struct intr_queue, iq_tail));
+ASSYM(IRSR_BUSY, IRSR_BUSY);
 
-ASSYM(IQE_SHIFT, IQE_SHIFT);
-ASSYM(IQE_TAG, offsetof(struct iqe, iqe_tag));
-ASSYM(IQE_PRI, offsetof(struct iqe, iqe_pri));
-ASSYM(IQE_VEC, offsetof(struct iqe, iqe_vec));
-ASSYM(IQE_FUNC, offsetof(struct iqe, iqe_func));
-ASSYM(IQE_ARG, offsetof(struct iqe, iqe_arg));
+ASSYM(IR_NEXT, offsetof(struct intr_request, ir_next));
+ASSYM(IR_FUNC, offsetof(struct intr_request, ir_func));
+ASSYM(IR_ARG, offsetof(struct intr_request, ir_arg));
+ASSYM(IR_PRI, offsetof(struct intr_request, ir_pri));
+ASSYM(IR_VEC, offsetof(struct intr_request, ir_vec));
 
 ASSYM(ITA_MASK, offsetof(struct ipi_tlb_args, ita_mask));
 ASSYM(ITA_TLB, offsetof(struct ipi_tlb_args, ita_tlb));
@@ -210,7 +209,7 @@ ASSYM(IV_FUNC, offsetof(struct intr_vector, iv_func));
 ASSYM(IV_ARG, offsetof(struct intr_vector, iv_arg));
 ASSYM(IV_PRI, offsetof(struct intr_vector, iv_pri));
 
-ASSYM(NIV, NIV);
+ASSYM(IV_MAX, IV_MAX);
 
 ASSYM(KEF_ASTPENDING, KEF_ASTPENDING);
 ASSYM(KEF_NEEDRESCHED, KEF_NEEDRESCHED);
diff --git a/sys/sparc64/sparc64/intr_machdep.c b/sys/sparc64/sparc64/intr_machdep.c
index 46b55b4..43f9bb9 100644
--- a/sys/sparc64/sparc64/intr_machdep.c
+++ b/sys/sparc64/sparc64/intr_machdep.c
@@ -80,12 +80,11 @@
 #define	MAX_STRAY_LOG	5
 
 CTASSERT((1 << IV_SHIFT) == sizeof(struct intr_vector));
-CTASSERT((1 << IQE_SHIFT) == sizeof(struct iqe));
 
-ih_func_t *intr_handlers[NPIL];
-struct	intr_vector intr_vectors[NIV];
+ih_func_t *intr_handlers[PIL_MAX];
+struct	intr_vector intr_vectors[IV_MAX];
 
-u_long	intr_stray_count[NIV];
+u_long	intr_stray_count[IV_MAX];
 
 /* protect the intr_vectors table */
 static struct	mtx intr_table_lock;
@@ -94,32 +93,6 @@ static void intr_stray_level(struct trapframe *tf);
 static void intr_stray_vector(void *cookie);
 
 void
-intr_dequeue(struct trapframe *tf)
-{
-	struct intr_queue *iq;
-	struct iqe *iqe;
-	u_long head;
-	u_long next;
-	u_long tail;
-
-	iq = PCPU_PTR(iq);
-	for (head = iq->iq_head;; head = next) {
-		for (tail = iq->iq_tail; tail != head;) {
-			iqe = &iq->iq_queue[tail];
-			atomic_add_long(&intrcnt[iqe->iqe_vec], 1);
-			KASSERT(iqe->iqe_func != NULL,
-			    ("intr_dequeue: iqe->iqe_func NULL"));
-			iqe->iqe_func(iqe->iqe_arg);
-			tail = (tail + 1) & IQ_MASK;
-		}
-		iq->iq_tail = tail;
-		next = iq->iq_head;
-		if (head == next)
-			break;
-	}
-}
-
-void
 intr_setup(int pri, ih_func_t *ihf, int vec, iv_func_t *ivf, void *iva)
 {
 	u_long ps;
@@ -162,9 +135,9 @@ intr_init1()
 	int i;
 
 	/* Mark all interrupts as being stray. */
-	for (i = 0; i < NPIL; i++)
+	for (i = 0; i < PIL_MAX; i++)
 		intr_handlers[i] = intr_stray_level;
-	for (i = 0; i < NIV; i++) {
+	for (i = 0; i < IV_MAX; i++) {
 		intr_vectors[i].iv_func = intr_stray_vector;
 		intr_vectors[i].iv_arg = &intr_vectors[i];
 		intr_vectors[i].iv_pri = PIL_LOW;
diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c
index d68d18c..962929b 100644
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@@ -183,6 +183,15 @@ cpu_startup(void *arg)
 void
 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size)
 {
+	struct intr_request *ir;
+	int i;
+
+	pcpu->pc_irtail = &pcpu->pc_irhead;
+	for (i = 0; i < IR_FREE; i++) {
+		ir = &pcpu->pc_irpool[i];
+		ir->ir_next = pcpu->pc_irfree;
+		pcpu->pc_irfree = ir;
+	}
 }
 
 unsigned
diff --git a/sys/sparc64/sparc64/nexus.c b/sys/sparc64/sparc64/nexus.c
index d86cc71..cf70ced 100644
--- a/sys/sparc64/sparc64/nexus.c
+++ b/sys/sparc64/sparc64/nexus.c
@@ -217,7 +217,7 @@ nexus_probe(device_t dev)
 	sc->sc_mem_rman.rm_descr = "UPA Device Memory";
 	if (rman_init(&sc->sc_intr_rman) != 0 ||
 	    rman_init(&sc->sc_mem_rman) != 0 ||
-	    rman_manage_region(&sc->sc_intr_rman, 0, NIV - 1) != 0 ||
+	    rman_manage_region(&sc->sc_intr_rman, 0, IV_MAX - 1) != 0 ||
 	    rman_manage_region(&sc->sc_mem_rman, UPA_MEMSTART, UPA_MEMEND) != 0)
 		panic("nexus_probe: failed to set up rmans");
 	for (child = OF_child(root); child != 0; child = OF_peer(child)) {