Sync with i386 - Optimize intr_execute_handlers a bit etc.

4 files changed, 71 insertions, 27 deletions

diff --git a/sys/amd64/amd64/intr_machdep.c b/sys/amd64/amd64/intr_machdep.c
index 8a8968b..c544251 100644
--- a/sys/amd64/amd64/intr_machdep.c
+++ b/sys/amd64/amd64/intr_machdep.c
@@ -212,8 +212,7 @@ intr_execute_handlers(struct intsrc *isrc, struct intrframe *iframe)
 		 * For stray and threaded interrupts, we mask and EOI the
 		 * source.
 		 */
-		isrc->is_pic->pic_disable_source(isrc);
-		isrc->is_pic->pic_eoi_source(isrc);
+		isrc->is_pic->pic_disable_source(isrc, PIC_EOI);
 		if (ih == NULL)
 			error = EINVAL;
 		else
diff --git a/sys/amd64/amd64/io_apic.c b/sys/amd64/amd64/io_apic.c
index 4b8568d..8cc14d3 100644
--- a/sys/amd64/amd64/io_apic.c
+++ b/sys/amd64/amd64/io_apic.c
@@ -119,7 +119,7 @@ static void	ioapic_write(volatile ioapic_t *apic, int reg, u_int val);
 static const char *ioapic_bus_string(int bus_type);
 static void	ioapic_print_vector(struct ioapic_intsrc *intpin);
 static void	ioapic_enable_source(struct intsrc *isrc);
-static void	ioapic_disable_source(struct intsrc *isrc);
+static void	ioapic_disable_source(struct intsrc *isrc, int eoi);
 static void	ioapic_eoi_source(struct intsrc *isrc);
 static void	ioapic_enable_intr(struct intsrc *isrc);
 static int	ioapic_vector(struct intsrc *isrc);
@@ -148,6 +148,12 @@ static int mixed_mode_active = 1;
 #endif
 TUNABLE_INT("hw.apic.mixed_mode", &mixed_mode_active);
 
+static __inline void
+_ioapic_eoi_source(struct intsrc *isrc)
+{
+	lapic_eoi();
+}
+
 static u_int
 ioapic_read(volatile ioapic_t *apic, int reg)
 {
@@ -225,7 +231,7 @@ ioapic_enable_source(struct intsrc *isrc)
 }
 
 static void
-ioapic_disable_source(struct intsrc *isrc)
+ioapic_disable_source(struct intsrc *isrc, int eoi)
 {
 	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
 	struct ioapic *io = (struct ioapic *)isrc->is_pic;
@@ -240,6 +246,10 @@ ioapic_disable_source(struct intsrc *isrc)
 		    flags);
 		intpin->io_masked = 1;
 	}
+
+	if (eoi == PIC_EOI)
+		_ioapic_eoi_source(isrc);
+
 	mtx_unlock_spin(&icu_lock);
 }
 
@@ -247,7 +257,7 @@ static void
 ioapic_eoi_source(struct intsrc *isrc)
 {
 
-	lapic_eoi();
+	_ioapic_eoi_source(isrc);
 }
 
 /*
diff --git a/sys/amd64/include/intr_machdep.h b/sys/amd64/include/intr_machdep.h
index d8256c9..af9e672 100644
--- a/sys/amd64/include/intr_machdep.h
+++ b/sys/amd64/include/intr_machdep.h
@@ -50,7 +50,7 @@ struct intsrc;
  */
 struct pic {
 	void (*pic_enable_source)(struct intsrc *);
-	void (*pic_disable_source)(struct intsrc *);
+	void (*pic_disable_source)(struct intsrc *, int);
 	void (*pic_eoi_source)(struct intsrc *);
 	void (*pic_enable_intr)(struct intsrc *);
 	int (*pic_vector)(struct intsrc *);
@@ -61,6 +61,12 @@ struct pic {
 	    enum intr_polarity);
 };
 
+/* Flags for pic_disable_source() */
+enum {
+	PIC_EOI,
+	PIC_NO_EOI,
+};
+
 /*
  * An interrupt source.  The upper-layer code uses the PIC methods to
  * control a given source.  The lower-layer PIC drivers can store additional
diff --git a/sys/amd64/isa/atpic.c b/sys/amd64/isa/atpic.c
index c1f0178..3719a9b 100644
--- a/sys/amd64/isa/atpic.c
+++ b/sys/amd64/isa/atpic.c
@@ -140,7 +140,7 @@ struct atpic_intsrc {
 };
 
 static void atpic_enable_source(struct intsrc *isrc);
-static void atpic_disable_source(struct intsrc *isrc);
+static void atpic_disable_source(struct intsrc *isrc, int eoi);
 static void atpic_eoi_master(struct intsrc *isrc);
 static void atpic_eoi_slave(struct intsrc *isrc);
 static void atpic_enable_intr(struct intsrc *isrc);
@@ -177,6 +177,35 @@ static struct atpic_intsrc atintrs[] = {
 
 CTASSERT(sizeof(atintrs) / sizeof(atintrs[0]) == NUM_ISA_IRQS);
 
+static __inline void
+_atpic_eoi_master(struct intsrc *isrc)
+{
+
+	KASSERT(isrc->is_pic == &atpics[MASTER].at_pic,
+	    ("%s: mismatched pic", __func__));
+#ifndef AUTO_EOI_1
+	outb(atpics[MASTER].at_ioaddr, OCW2_EOI);
+#endif
+}
+
+/*
+ * The data sheet says no auto-EOI on slave, but it sometimes works.
+ * So, if AUTO_EOI_2 is enabled, we use it.
+ */
+static __inline void
+_atpic_eoi_slave(struct intsrc *isrc)
+{
+
+	KASSERT(isrc->is_pic == &atpics[SLAVE].at_pic,
+	    ("%s: mismatched pic", __func__));
+#ifndef AUTO_EOI_2
+	outb(atpics[SLAVE].at_ioaddr, OCW2_EOI);
+#ifndef AUTO_EOI_1
+	outb(atpics[MASTER].at_ioaddr, OCW2_EOI);
+#endif
+#endif
+}
+
 static void
 atpic_enable_source(struct intsrc *isrc)
 {
@@ -192,48 +221,48 @@ atpic_enable_source(struct intsrc *isrc)
 }
 
 static void
-atpic_disable_source(struct intsrc *isrc)
+atpic_disable_source(struct intsrc *isrc, int eoi)
 {
 	struct atpic_intsrc *ai = (struct atpic_intsrc *)isrc;
 	struct atpic *ap = (struct atpic *)isrc->is_pic;
 
-	if (ai->at_trigger == INTR_TRIGGER_EDGE)
-		return;
 	mtx_lock_spin(&icu_lock);
-	*ap->at_imen |= IMEN_MASK(ai);
-	outb(ap->at_ioaddr + ICU_IMR_OFFSET, *ap->at_imen);
+	if (ai->at_trigger != INTR_TRIGGER_EDGE) {
+		*ap->at_imen |= IMEN_MASK(ai);
+		outb(ap->at_ioaddr + ICU_IMR_OFFSET, *ap->at_imen);
+	}
+
+	/*
+	 * Take care to call these functions directly instead of through
+	 * a function pointer.  All of the referenced variables should
+	 * still be hot in the cache.
+	 */
+	if (eoi == PIC_EOI) {
+		if (isrc->is_pic == &atpics[MASTER].at_pic)
+			_atpic_eoi_master(isrc);
+		else
+			_atpic_eoi_slave(isrc);
+	}
+
 	mtx_unlock_spin(&icu_lock);
 }
 
 static void
 atpic_eoi_master(struct intsrc *isrc)
 {
-
-	KASSERT(isrc->is_pic == &atpics[MASTER].at_pic,
-	    ("%s: mismatched pic", __func__));
 #ifndef AUTO_EOI_1
 	mtx_lock_spin(&icu_lock);
-	outb(atpics[MASTER].at_ioaddr, OCW2_EOI);
+	_atpic_eoi_master(isrc);
 	mtx_unlock_spin(&icu_lock);
 #endif
 }
 
-/*
- * The data sheet says no auto-EOI on slave, but it sometimes works.
- * So, if AUTO_EOI_2 is enabled, we use it.
- */
 static void
 atpic_eoi_slave(struct intsrc *isrc)
 {
-
-	KASSERT(isrc->is_pic == &atpics[SLAVE].at_pic,
-	    ("%s: mismatched pic", __func__));
 #ifndef AUTO_EOI_2
 	mtx_lock_spin(&icu_lock);
-	outb(atpics[SLAVE].at_ioaddr, OCW2_EOI);
-#ifndef AUTO_EOI_1
-	outb(atpics[MASTER].at_ioaddr, OCW2_EOI);
-#endif
+	_atpic_eoi_slave(isrc);
 	mtx_unlock_spin(&icu_lock);
 #endif
 }