Optimize intr_execute_handlers() by combining the pic_disable_source() and

pic_eoi_source() into one call.  This halves the number of spinlock operations
and indirect function calls in the normal case of handling a normal (ithread)
interrupt.  Optimize the atpic and ioapic drivers to use inlines where
appropriate in supporting the intr_execute_handlers() change.

This knocks 900ns, or roughly 1350 cycles, off of the time spent servicing an
interrupt in the common case on my 1.5GHz P4 uniprocessor system.  SMP systems
likely won't see as much of a gain due to the ioapic being more efficient than
the atpic.  I'll investigate porting this to amd64 soon.

Reviewed by:	jhb

4 files changed, 71 insertions, 27 deletions

diff --git a/sys/i386/i386/intr_machdep.c b/sys/i386/i386/intr_machdep.c
index 8a8968b..c544251 100644
--- a/sys/i386/i386/intr_machdep.c
+++ b/sys/i386/i386/intr_machdep.c
@@ -212,8 +212,7 @@ intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe)
 		 * For stray and threaded interrupts, we mask and EOI the
 		 * source.
 		 */
-		isrc->is_pic->pic_disable_source(isrc);
-		isrc->is_pic->pic_eoi_source(isrc);
+		isrc->is_pic->pic_disable_source(isrc, PIC_EOI);
 		if (ih == NULL)
 			error = EINVAL;
 		else
diff --git a/sys/i386/i386/io_apic.c b/sys/i386/i386/io_apic.c
index 99cf908..aca7d78 100644
--- a/sys/i386/i386/io_apic.c
+++ b/sys/i386/i386/io_apic.c
@@ -118,7 +118,7 @@ static void	ioapic_write(volatile ioapic_t *apic, int reg, u_int val);
 static const char *ioapic_bus_string(int bus_type);
 static void	ioapic_print_vector(struct ioapic_intsrc *intpin);
 static void	ioapic_enable_source(struct intsrc *isrc);
-static void	ioapic_disable_source(struct intsrc *isrc);
+static void	ioapic_disable_source(struct intsrc *isrc, int eoi);
 static void	ioapic_eoi_source(struct intsrc *isrc);
 static void	ioapic_enable_intr(struct intsrc *isrc);
 static int	ioapic_vector(struct intsrc *isrc);
@@ -147,6 +147,12 @@ static int mixed_mode_active = 1;
 #endif
 TUNABLE_INT("hw.apic.mixed_mode", &mixed_mode_active);
 
+static __inline void
+_ioapic_eoi_source(struct intsrc *isrc)
+{
+	lapic_eoi();
+}
+
 static u_int
 ioapic_read(volatile ioapic_t *apic, int reg)
 {
@@ -224,7 +230,7 @@ ioapic_enable_source(struct intsrc *isrc)
 }
 
 static void
-ioapic_disable_source(struct intsrc *isrc)
+ioapic_disable_source(struct intsrc *isrc, int eoi)
 {
 	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
 	struct ioapic *io = (struct ioapic *)isrc->is_pic;
@@ -239,6 +245,10 @@ ioapic_disable_source(struct intsrc *isrc)
 		    flags);
 		intpin->io_masked = 1;
 	}
+
+	if (eoi == PIC_EOI)
+		_ioapic_eoi_source(isrc);
+
 	mtx_unlock_spin(&icu_lock);
 }
 
@@ -246,7 +256,7 @@ static void
 ioapic_eoi_source(struct intsrc *isrc)
 {
 
-	lapic_eoi();
+	_ioapic_eoi_source(isrc);
 }
 
 /*
diff --git a/sys/i386/include/intr_machdep.h b/sys/i386/include/intr_machdep.h
index d8256c9..af9e672 100644
--- a/sys/i386/include/intr_machdep.h
+++ b/sys/i386/include/intr_machdep.h
@@ -50,7 +50,7 @@ struct intsrc;
  */
 struct pic {
 	void (*pic_enable_source)(struct intsrc *);
-	void (*pic_disable_source)(struct intsrc *);
+	void (*pic_disable_source)(struct intsrc *, int);
 	void (*pic_eoi_source)(struct intsrc *);
 	void (*pic_enable_intr)(struct intsrc *);
 	int (*pic_vector)(struct intsrc *);
@@ -61,6 +61,12 @@ struct pic {
 	    enum intr_polarity);
 };
 
+/* Flags for pic_disable_source() */
+enum {
+	PIC_EOI,
+	PIC_NO_EOI,
+};
+
 /*
  * An interrupt source.  The upper-layer code uses the PIC methods to
  * control a given source.  The lower-layer PIC drivers can store additional
diff --git a/sys/i386/isa/atpic.c b/sys/i386/isa/atpic.c
index 9a56f08..8f456ac 100644
--- a/sys/i386/isa/atpic.c
+++ b/sys/i386/isa/atpic.c
@@ -154,7 +154,7 @@ struct atpic_intsrc {
 };
 
 static void atpic_enable_source(struct intsrc *isrc);
-static void atpic_disable_source(struct intsrc *isrc);
+static void atpic_disable_source(struct intsrc *isrc, int eoi);
 static void atpic_eoi_master(struct intsrc *isrc);
 static void atpic_eoi_slave(struct intsrc *isrc);
 static void atpic_enable_intr(struct intsrc *isrc);
@@ -191,6 +191,35 @@ static struct atpic_intsrc atintrs[] = {
 
 CTASSERT(sizeof(atintrs) / sizeof(atintrs[0]) == NUM_ISA_IRQS);
 
+static __inline void
+_atpic_eoi_master(struct intsrc *isrc)
+{
+
+	KASSERT(isrc->is_pic == &atpics[MASTER].at_pic,
+	    ("%s: mismatched pic", __func__));
+#ifndef AUTO_EOI_1
+	outb(atpics[MASTER].at_ioaddr, OCW2_EOI);
+#endif
+}
+
+/*
+ * The data sheet says no auto-EOI on slave, but it sometimes works.
+ * So, if AUTO_EOI_2 is enabled, we use it.
+ */
+static __inline void
+_atpic_eoi_slave(struct intsrc *isrc)
+{
+
+	KASSERT(isrc->is_pic == &atpics[SLAVE].at_pic,
+	    ("%s: mismatched pic", __func__));
+#ifndef AUTO_EOI_2
+	outb(atpics[SLAVE].at_ioaddr, OCW2_EOI);
+#ifndef AUTO_EOI_1
+	outb(atpics[MASTER].at_ioaddr, OCW2_EOI);
+#endif
+#endif
+}
+
 static void
 atpic_enable_source(struct intsrc *isrc)
 {
@@ -206,48 +235,48 @@ atpic_enable_source(struct intsrc *isrc)
 }
 
 static void
-atpic_disable_source(struct intsrc *isrc)
+atpic_disable_source(struct intsrc *isrc, int eoi)
 {
 	struct atpic_intsrc *ai = (struct atpic_intsrc *)isrc;
 	struct atpic *ap = (struct atpic *)isrc->is_pic;
 
-	if (ai->at_trigger == INTR_TRIGGER_EDGE)
-		return;
 	mtx_lock_spin(&icu_lock);
-	*ap->at_imen |= IMEN_MASK(ai);
-	outb(ap->at_ioaddr + ICU_IMR_OFFSET, *ap->at_imen);
+	if (ai->at_trigger != INTR_TRIGGER_EDGE) {
+		*ap->at_imen |= IMEN_MASK(ai);
+		outb(ap->at_ioaddr + ICU_IMR_OFFSET, *ap->at_imen);
+	}
+
+	/*
+	 * Take care to call these functions directly instead of through
+	 * a function pointer.  All of the referenced variables should
+	 * still be hot in the cache.
+	 */
+	if (eoi == PIC_EOI) {
+		if (isrc->is_pic == &atpics[MASTER].at_pic)
+			_atpic_eoi_master(isrc);
+		else
+			_atpic_eoi_slave(isrc);
+	}
+
 	mtx_unlock_spin(&icu_lock);
 }
 
 static void
 atpic_eoi_master(struct intsrc *isrc)
 {
-
-	KASSERT(isrc->is_pic == &atpics[MASTER].at_pic,
-	    ("%s: mismatched pic", __func__));
 #ifndef AUTO_EOI_1
 	mtx_lock_spin(&icu_lock);
-	outb(atpics[MASTER].at_ioaddr, OCW2_EOI);
+	_atpic_eoi_master(isrc);
 	mtx_unlock_spin(&icu_lock);
 #endif
 }
 
-/*
- * The data sheet says no auto-EOI on slave, but it sometimes works.
- * So, if AUTO_EOI_2 is enabled, we use it.
- */
 static void
 atpic_eoi_slave(struct intsrc *isrc)
 {
-
-	KASSERT(isrc->is_pic == &atpics[SLAVE].at_pic,
-	    ("%s: mismatched pic", __func__));
 #ifndef AUTO_EOI_2
 	mtx_lock_spin(&icu_lock);
-	outb(atpics[SLAVE].at_ioaddr, OCW2_EOI);
-#ifndef AUTO_EOI_1
-	outb(atpics[MASTER].at_ioaddr, OCW2_EOI);
-#endif
+	_atpic_eoi_slave(isrc);
 	mtx_unlock_spin(&icu_lock);
 #endif
 }