Further fixes for multiple-IO-APIC systems from Tor Egge:

Further experimentation showed that some Dell 2450 machines with the
prevention kludge installed still got T_RESERVED traps.  CPU interrupt
vector 0x7A was observed to be triggered.  This might have been the
bitwise OR of two different vectors sent from each of the IOAPICs at
the same time.

	IOAPIC #0: 0x68 --> irq 8: RTC timer interrupt
	IOAPIC #1: 0x32 --> irq 18: scsi host adapter or network interface
		   ----
		   0x7a --> T_RESERVED

Both IOAPICs had ID 0.

Appendix B.3 in the MP spec indicates that the operating system is
responsible for assigning unique IDs to the IOAPICs.

The enclosed patch programs the IOAPIC IDs according to the IOAPIC
entries in the MP table.

Submitted by:	tegge

11 files changed, 35 insertions, 49 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index b5bc1fd..1155e67 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -572,6 +572,7 @@ mp_enable(u_int boot_addr)
 	for (apic = 0; apic < mp_napics; ++apic) {
 		ux = io_apic_read(apic, IOAPIC_VER);
 		io_apic_versions[apic] = ux;
+		io_apic_set_id(apic, IO_TO_ID(apic));
 	}
 
 	/* program each IO APIC in the system */
@@ -1661,13 +1662,7 @@ default_mp_table(int type)
 #else
 	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
 #endif	/* REALLY_ANAL_IOAPICID_VALUE */
-		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
-		ux &= ~APIC_ID_MASK;	/* clear the ID field */
-		ux |= 0x02000000;	/* set it to '2' */
-		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
-		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
-		if ((ux & APIC_ID_MASK) != 0x02000000)
-			panic("can't control IO APIC ID, reg: 0x%08x", ux);
+		io_apic_set_id(0, 2);
 		io_apic_id = 2;
 	}
 	IO_TO_ID(0) = io_apic_id;
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index b5bc1fd..1155e67 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -572,6 +572,7 @@ mp_enable(u_int boot_addr)
 	for (apic = 0; apic < mp_napics; ++apic) {
 		ux = io_apic_read(apic, IOAPIC_VER);
 		io_apic_versions[apic] = ux;
+		io_apic_set_id(apic, IO_TO_ID(apic));
 	}
 
 	/* program each IO APIC in the system */
@@ -1661,13 +1662,7 @@ default_mp_table(int type)
 #else
 	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
 #endif	/* REALLY_ANAL_IOAPICID_VALUE */
-		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
-		ux &= ~APIC_ID_MASK;	/* clear the ID field */
-		ux |= 0x02000000;	/* set it to '2' */
-		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
-		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
-		if ((ux & APIC_ID_MASK) != 0x02000000)
-			panic("can't control IO APIC ID, reg: 0x%08x", ux);
+		io_apic_set_id(0, 2);
 		io_apic_id = 2;
 	}
 	IO_TO_ID(0) = io_apic_id;
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index b5bc1fd..1155e67 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -572,6 +572,7 @@ mp_enable(u_int boot_addr)
 	for (apic = 0; apic < mp_napics; ++apic) {
 		ux = io_apic_read(apic, IOAPIC_VER);
 		io_apic_versions[apic] = ux;
+		io_apic_set_id(apic, IO_TO_ID(apic));
 	}
 
 	/* program each IO APIC in the system */
@@ -1661,13 +1662,7 @@ default_mp_table(int type)
 #else
 	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
 #endif	/* REALLY_ANAL_IOAPICID_VALUE */
-		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
-		ux &= ~APIC_ID_MASK;	/* clear the ID field */
-		ux |= 0x02000000;	/* set it to '2' */
-		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
-		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
-		if ((ux & APIC_ID_MASK) != 0x02000000)
-			panic("can't control IO APIC ID, reg: 0x%08x", ux);
+		io_apic_set_id(0, 2);
 		io_apic_id = 2;
 	}
 	IO_TO_ID(0) = io_apic_id;
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index c11bcb2..17b7a3a 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -165,6 +165,7 @@ void	imen_dump		__P((void));
 int	apic_ipi		__P((int, int, int));
 int	selected_apic_ipi	__P((u_int, int, int));
 int	io_apic_setup		__P((int));
+void	io_apic_set_id		__P((int, int));
 int	ext_int_setup		__P((int, int));
 
 #if defined(READY)
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index b5bc1fd..1155e67 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -572,6 +572,7 @@ mp_enable(u_int boot_addr)
 	for (apic = 0; apic < mp_napics; ++apic) {
 		ux = io_apic_read(apic, IOAPIC_VER);
 		io_apic_versions[apic] = ux;
+		io_apic_set_id(apic, IO_TO_ID(apic));
 	}
 
 	/* program each IO APIC in the system */
@@ -1661,13 +1662,7 @@ default_mp_table(int type)
 #else
 	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
 #endif	/* REALLY_ANAL_IOAPICID_VALUE */
-		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
-		ux &= ~APIC_ID_MASK;	/* clear the ID field */
-		ux |= 0x02000000;	/* set it to '2' */
-		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
-		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
-		if ((ux & APIC_ID_MASK) != 0x02000000)
-			panic("can't control IO APIC ID, reg: 0x%08x", ux);
+		io_apic_set_id(0, 2);
 		io_apic_id = 2;
 	}
 	IO_TO_ID(0) = io_apic_id;
diff --git a/sys/i386/i386/mpapic.c b/sys/i386/i386/mpapic.c
index 975706a..7e0f197 100644
--- a/sys/i386/i386/mpapic.c
+++ b/sys/i386/i386/mpapic.c
@@ -143,6 +143,24 @@ static void polarity __P((int apic, int pin, u_int32_t * flags, int level));
 	  IOART_DESTPHY |	\
 	  IOART_DELLOPRI))
 
+void
+io_apic_set_id(int apic, int id)
+{
+	u_int32_t ux;
+	
+	ux = io_apic_read(apic, IOAPIC_ID);	/* get current contents */
+	if (((ux & APIC_ID_MASK) >> 24) != id) {
+		ux &= ~APIC_ID_MASK;	/* clear the ID field */
+		ux |= (id << 24);
+		io_apic_write(apic, IOAPIC_ID, ux);	/* write new value */
+		ux = io_apic_read(apic, IOAPIC_ID);	/* re-read && test */
+		if (((ux & APIC_ID_MASK) >> 24) != id)
+			panic("can't control IO APIC #%d ID, reg: 0x%08x",
+			      apic, ux);
+	}
+}
+
+
 /*
  * Setup the IO APIC.
  */
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index b5bc1fd..1155e67 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -572,6 +572,7 @@ mp_enable(u_int boot_addr)
 	for (apic = 0; apic < mp_napics; ++apic) {
 		ux = io_apic_read(apic, IOAPIC_VER);
 		io_apic_versions[apic] = ux;
+		io_apic_set_id(apic, IO_TO_ID(apic));
 	}
 
 	/* program each IO APIC in the system */
@@ -1661,13 +1662,7 @@ default_mp_table(int type)
 #else
 	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
 #endif	/* REALLY_ANAL_IOAPICID_VALUE */
-		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
-		ux &= ~APIC_ID_MASK;	/* clear the ID field */
-		ux |= 0x02000000;	/* set it to '2' */
-		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
-		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
-		if ((ux & APIC_ID_MASK) != 0x02000000)
-			panic("can't control IO APIC ID, reg: 0x%08x", ux);
+		io_apic_set_id(0, 2);
 		io_apic_id = 2;
 	}
 	IO_TO_ID(0) = io_apic_id;
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index b5bc1fd..1155e67 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -572,6 +572,7 @@ mp_enable(u_int boot_addr)
 	for (apic = 0; apic < mp_napics; ++apic) {
 		ux = io_apic_read(apic, IOAPIC_VER);
 		io_apic_versions[apic] = ux;
+		io_apic_set_id(apic, IO_TO_ID(apic));
 	}
 
 	/* program each IO APIC in the system */
@@ -1661,13 +1662,7 @@ default_mp_table(int type)
 #else
 	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
 #endif	/* REALLY_ANAL_IOAPICID_VALUE */
-		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
-		ux &= ~APIC_ID_MASK;	/* clear the ID field */
-		ux |= 0x02000000;	/* set it to '2' */
-		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
-		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
-		if ((ux & APIC_ID_MASK) != 0x02000000)
-			panic("can't control IO APIC ID, reg: 0x%08x", ux);
+		io_apic_set_id(0, 2);
 		io_apic_id = 2;
 	}
 	IO_TO_ID(0) = io_apic_id;
diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h
index c11bcb2..17b7a3a 100644
--- a/sys/i386/include/smp.h
+++ b/sys/i386/include/smp.h
@@ -165,6 +165,7 @@ void	imen_dump		__P((void));
 int	apic_ipi		__P((int, int, int));
 int	selected_apic_ipi	__P((u_int, int, int));
 int	io_apic_setup		__P((int));
+void	io_apic_set_id		__P((int, int));
 int	ext_int_setup		__P((int, int));
 
 #if defined(READY)
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index b5bc1fd..1155e67 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -572,6 +572,7 @@ mp_enable(u_int boot_addr)
 	for (apic = 0; apic < mp_napics; ++apic) {
 		ux = io_apic_read(apic, IOAPIC_VER);
 		io_apic_versions[apic] = ux;
+		io_apic_set_id(apic, IO_TO_ID(apic));
 	}
 
 	/* program each IO APIC in the system */
@@ -1661,13 +1662,7 @@ default_mp_table(int type)
 #else
 	if ((io_apic_id == 0) || (io_apic_id == 1) || (io_apic_id == 15)) {
 #endif	/* REALLY_ANAL_IOAPICID_VALUE */
-		ux = io_apic_read(0, IOAPIC_ID);	/* get current contents */
-		ux &= ~APIC_ID_MASK;	/* clear the ID field */
-		ux |= 0x02000000;	/* set it to '2' */
-		io_apic_write(0, IOAPIC_ID, ux);	/* write new value */
-		ux = io_apic_read(0, IOAPIC_ID);	/* re-read && test */
-		if ((ux & APIC_ID_MASK) != 0x02000000)
-			panic("can't control IO APIC ID, reg: 0x%08x", ux);
+		io_apic_set_id(0, 2);
 		io_apic_id = 2;
 	}
 	IO_TO_ID(0) = io_apic_id;
diff --git a/sys/sys/smp.h b/sys/sys/smp.h
index c11bcb2..17b7a3a 100644
--- a/sys/sys/smp.h
+++ b/sys/sys/smp.h
@@ -165,6 +165,7 @@ void	imen_dump		__P((void));
 int	apic_ipi		__P((int, int, int));
 int	selected_apic_ipi	__P((u_int, int, int));
 int	io_apic_setup		__P((int));
+void	io_apic_set_id		__P((int, int));
 int	ext_int_setup		__P((int, int));
 
 #if defined(READY)