- Allocate apic vectors on a per-cpu basis. This allows us to allocate

   more irqs as we have more cpus.  This is principally useful on systems
   with msi devices which may want many irqs per-cpu.

Discussed with:	jhb
Sponsored by:	Nokia

6 files changed, 153 insertions, 77 deletions

diff --git a/sys/i386/i386/io_apic.c b/sys/i386/i386/io_apic.c
index b038724..f63f28f 100644
--- a/sys/i386/i386/io_apic.c
+++ b/sys/i386/i386/io_apic.c
@@ -327,39 +327,56 @@ ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id)
 {
 	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
 	struct ioapic *io = (struct ioapic *)isrc->is_pic;
+	u_int old_vector;
+	u_int old_id;
 
+	/*
+	 * keep 1st core as the destination for NMI
+	 */
+	if (intpin->io_irq == IRQ_NMI)
+		apic_id = 0;
+
+	/*
+	 * Set us up to free the old irq.
+	 */
+	old_vector = intpin->io_vector;
+	old_id = intpin->io_cpu;
+	if (old_vector && apic_id == old_id)
+		return;
+
+	/*
+	 * Allocate an APIC vector for this interrupt pin.  Once
+	 * we have a vector we program the interrupt pin.
+	 */
 	intpin->io_cpu = apic_id;
+	intpin->io_vector = apic_alloc_vector(apic_id, intpin->io_irq);
 	if (bootverbose) {
-		printf("ioapic%u: Assigning ", io->io_id);
+		printf("ioapic%u: routing intpin %u (", io->io_id,
+		    intpin->io_intpin);
 		ioapic_print_irq(intpin);
-		printf(" to local APIC %u\n", intpin->io_cpu);
+		printf(") to lapic %u vector %u\n", intpin->io_cpu,
+		    intpin->io_vector);
 	}
 	ioapic_program_intpin(intpin);
+	/*
+	 * Free the old vector after the new one is established.  This is done
+	 * to prevent races where we could miss an interrupt.
+	 */
+	if (old_vector)
+		apic_free_vector(old_id, old_vector, intpin->io_irq);
 }
 
 static void
 ioapic_enable_intr(struct intsrc *isrc)
 {
 	struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
-	struct ioapic *io = (struct ioapic *)isrc->is_pic;
 
-	if (intpin->io_vector == 0) {
-		/*
-		 * Allocate an APIC vector for this interrupt pin.  Once
-		 * we have a vector we program the interrupt pin.
-		 */
-		intpin->io_vector = apic_alloc_vector(intpin->io_irq);
-		if (bootverbose) {
-			printf("ioapic%u: routing intpin %u (", io->io_id,
-			    intpin->io_intpin);
-			ioapic_print_irq(intpin);
-			printf(") to vector %u\n", intpin->io_vector);
-		}
-		ioapic_program_intpin(intpin);
-		apic_enable_vector(intpin->io_vector);
-	}
+	if (intpin->io_vector == 0)
+		ioapic_assign_cpu(isrc, pcpu_find(0)->pc_apic_id);
+	apic_enable_vector(intpin->io_cpu, intpin->io_vector);
 }
 
+
 static void
 ioapic_disable_intr(struct intsrc *isrc)
 {
@@ -369,11 +386,11 @@ ioapic_disable_intr(struct intsrc *isrc)
 	if (intpin->io_vector != 0) {
 		/* Mask this interrupt pin and free its APIC vector. */
 		vector = intpin->io_vector;
-		apic_disable_vector(vector);
+		apic_disable_vector(intpin->io_cpu, vector);
 		intpin->io_masked = 1;
 		intpin->io_vector = 0;
 		ioapic_program_intpin(intpin);
-		apic_free_vector(vector, intpin->io_irq);
+		apic_free_vector(intpin->io_cpu, vector, intpin->io_irq);
 	}
 }
 
diff --git a/sys/i386/i386/local_apic.c b/sys/i386/i386/local_apic.c
index 2394e20..f543e68 100644
--- a/sys/i386/i386/local_apic.c
+++ b/sys/i386/i386/local_apic.c
@@ -46,6 +46,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/pcpu.h>
+#include <sys/proc.h>
+#include <sys/sched.h>
 #include <sys/smp.h>
 
 #include <vm/vm.h>
@@ -109,6 +111,8 @@ struct lapic {
 	u_long la_hard_ticks;
 	u_long la_stat_ticks;
 	u_long la_prof_ticks;
+	/* Include IDT_SYSCALL to make indexing easier. */
+	u_int la_ioint_irqs[APIC_NUM_IOINTS + 1];
 } static lapics[MAX_APIC_ID + 1];
 
 /* XXX: should thermal be an NMI? */
@@ -134,8 +138,6 @@ static inthand_t *ioint_handlers[] = {
 	IDTVEC(apic_isr7),	/* 224 - 255 */
 };
 
-/* Include IDT_SYSCALL to make indexing easier. */
-static u_int ioint_irqs[APIC_NUM_IOINTS + 1];
 
 static u_int32_t lapic_timer_divisors[] = { 
 	APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16,
@@ -216,7 +218,6 @@ lapic_init(vm_paddr_t addr)
 
 	/* Perform basic initialization of the BSP's local APIC. */
 	lapic_enable();
-	ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
 
 	/* Set BSP's per-CPU local APIC ID. */
 	PCPU_SET(apic_id, lapic_id());
@@ -224,7 +225,6 @@ lapic_init(vm_paddr_t addr)
 	/* Local APIC timer interrupt. */
 	setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_SYS386IGT, SEL_KPL,
 	    GSEL(GCODE_SEL, SEL_KPL));
-	ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] = IRQ_TIMER;
 
 	/* XXX: error/thermal interrupts */
 }
@@ -256,6 +256,9 @@ lapic_create(u_int apic_id, int boot_cpu)
 		lapics[apic_id].la_lvts[i] = lvts[i];
 		lapics[apic_id].la_lvts[i].lvt_active = 0;
 	}
+	lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL;
+	lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] =
+	    IRQ_TIMER;
 
 #ifdef SMP
 	cpu_add(apic_id, boot_cpu);
@@ -666,7 +669,8 @@ lapic_handle_intr(int vector, struct trapframe *frame)
 
 	if (vector == -1)
 		panic("Couldn't get vector from ISR!");
-	isrc = intr_lookup_source(apic_idt_to_irq(vector));
+	isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id),
+	    vector));
 	intr_execute_handlers(isrc, frame);
 }
 
@@ -781,9 +785,19 @@ lapic_timer_enable_intr(void)
 	lapic->lvt_timer = value;
 }
 
+u_int
+apic_cpuid(u_int apic_id)
+{
+#ifdef SMP
+	return apic_cpuids[apic_id];
+#else
+	return 0;
+#endif
+}
+
 /* Request a free IDT vector to be used by the specified IRQ. */
 u_int
-apic_alloc_vector(u_int irq)
+apic_alloc_vector(u_int apic_id, u_int irq)
 {
 	u_int vector;
 
@@ -795,9 +809,9 @@ apic_alloc_vector(u_int irq)
 	 */
 	mtx_lock_spin(&icu_lock);
 	for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
-		if (ioint_irqs[vector] != 0)
+		if (lapics[apic_id].la_ioint_irqs[vector] != 0)
 			continue;
-		ioint_irqs[vector] = irq;
+		lapics[apic_id].la_ioint_irqs[vector] = irq;
 		mtx_unlock_spin(&icu_lock);
 		return (vector + APIC_IO_INTS);
 	}
@@ -812,7 +826,7 @@ apic_alloc_vector(u_int irq)
  * satisfied, 0 is returned.
  */
 u_int
-apic_alloc_vectors(u_int *irqs, u_int count, u_int align)
+apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
 {
 	u_int first, run, vector;
 
@@ -835,7 +849,7 @@ apic_alloc_vectors(u_int *irqs, u_int count, u_int align)
 	for (vector = 0; vector < APIC_NUM_IOINTS; vector++) {
 
 		/* Vector is in use, end run. */
-		if (ioint_irqs[vector] != 0) {
+		if (lapics[apic_id].la_ioint_irqs[vector] != 0) {
 			run = 0;
 			first = 0;
 			continue;
@@ -855,7 +869,8 @@ apic_alloc_vectors(u_int *irqs, u_int count, u_int align)
 
 		/* Found a run, assign IRQs and return the first vector. */
 		for (vector = 0; vector < count; vector++)
-			ioint_irqs[first + vector] = irqs[vector];
+			lapics[apic_id].la_ioint_irqs[first + vector] =
+			    irqs[vector];
 		mtx_unlock_spin(&icu_lock);
 		return (first + APIC_IO_INTS);
 	}
@@ -864,8 +879,14 @@ apic_alloc_vectors(u_int *irqs, u_int count, u_int align)
 	return (0);
 }
 
+/*
+ * Enable a vector for a particular apic_id.  Since all lapics share idt
+ * entries and ioint_handlers this enables the vector on all lapics.  lapics
+ * which do not have the vector configured would report spurious interrupts
+ * should it fire.
+ */
 void
-apic_enable_vector(u_int vector)
+apic_enable_vector(u_int apic_id, u_int vector)
 {
 
 	KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
@@ -876,7 +897,7 @@ apic_enable_vector(u_int vector)
 }
 
 void
-apic_disable_vector(u_int vector)
+apic_disable_vector(u_int apic_id, u_int vector)
 {
 
 	KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
@@ -888,27 +909,42 @@ apic_disable_vector(u_int vector)
 
 /* Release an APIC vector when it's no longer in use. */
 void
-apic_free_vector(u_int vector, u_int irq)
+apic_free_vector(u_int apic_id, u_int vector, u_int irq)
 {
+	struct thread *td;
 	KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
 	    vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
 	    ("Vector %u does not map to an IRQ line", vector));
 	KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq));
-	KASSERT(ioint_irqs[vector - APIC_IO_INTS] == irq, ("IRQ mismatch"));
+	KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] ==
+	    irq, ("IRQ mismatch"));
+
+	/*
+	 * Bind us to the cpu that owned the vector before freeing it so
+	 * we don't lose an interrupt delivery race.
+	 */
+	td = curthread;
+	thread_lock(td);
+	if (sched_is_bound(td))
+		panic("apic_free_vector: Thread already bound.\n");
+	sched_bind(td, apic_cpuid(apic_id));
 	mtx_lock_spin(&icu_lock);
-	ioint_irqs[vector - APIC_IO_INTS] = 0;
+	lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = 0;
 	mtx_unlock_spin(&icu_lock);
+	sched_unbind(td);
+	thread_unlock(td);
+
 }
 
 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */
 u_int
-apic_idt_to_irq(u_int vector)
+apic_idt_to_irq(u_int apic_id, u_int vector)
 {
 
 	KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL &&
 	    vector <= APIC_IO_INTS + APIC_NUM_IOINTS,
 	    ("Vector %u does not map to an IRQ line", vector));
-	return (ioint_irqs[vector - APIC_IO_INTS]);
+	return (lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS]);
 }
 
 #ifdef DDB
@@ -919,6 +955,7 @@ DB_SHOW_COMMAND(apic, db_show_apic)
 {
 	struct intsrc *isrc;
 	int i, verbose;
+	u_int apic_id;
 	u_int irq;
 
 	if (strcmp(modif, "vv") == 0)
@@ -927,9 +964,14 @@ DB_SHOW_COMMAND(apic, db_show_apic)
 		verbose = 1;
 	else
 		verbose = 0;
-	for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
-		irq = ioint_irqs[i];
-		if (irq != 0 && irq != IRQ_SYSCALL) {
+	for (apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) {
+		if (lapics[apic_id].la_present == 0)
+			continue;
+		db_printf("Interrupts bound to lapic %u\n", apic_id);
+		for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) {
+			irq = lapics[apic_id].la_ioint_irqs[i];
+			if (irq == 0 || irq == IRQ_SYSCALL)
+				continue;
 			db_printf("vec 0x%2x -> ", i + APIC_IO_INTS);
 			if (irq == IRQ_TIMER)
 				db_printf("lapic timer\n");
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index bf8a2c6..d871017 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -206,6 +206,7 @@ struct cpu_info {
 	int	cpu_disabled:1;
 } static cpu_info[MAX_APIC_ID + 1];
 int cpu_apic_ids[MAXCPU];
+int apic_cpuids[MAX_APIC_ID + 1];
 
 /* Holds pending bitmap based IPIs per CPU */
 static volatile u_int cpu_ipi_pending[MAXCPU];
@@ -397,6 +398,7 @@ cpu_mp_start(void)
 		KASSERT(boot_cpu_id == PCPU_GET(apic_id),
 		    ("BSP's APIC ID doesn't match boot_cpu_id"));
 	cpu_apic_ids[0] = boot_cpu_id;
+	apic_cpuids[boot_cpu_id] = 0;
 
 	assign_cpu_ids();
 
@@ -705,6 +707,7 @@ assign_cpu_ids(void)
 
 		if (mp_ncpus < MAXCPU) {
 			cpu_apic_ids[mp_ncpus] = i;
+			apic_cpuids[i] = mp_ncpus;
 			mp_ncpus++;
 		} else
 			cpu_info[i].cpu_disabled = 1;
diff --git a/sys/i386/i386/msi.c b/sys/i386/i386/msi.c
index 437d859..8b18a10 100644
--- a/sys/i386/i386/msi.c
+++ b/sys/i386/i386/msi.c
@@ -161,7 +161,9 @@ msi_enable_intr(struct intsrc *isrc)
 {
 	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
 
-	apic_enable_vector(msi->msi_vector);
+	if (msi->msi_vector == 0)
+		msi_assign_cpu(isrc, 0);
+	apic_enable_vector(msi->msi_cpu, msi->msi_vector);
 }
 
 static void
@@ -169,7 +171,7 @@ msi_disable_intr(struct intsrc *isrc)
 {
 	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
 
-	apic_disable_vector(msi->msi_vector);
+	apic_disable_vector(msi->msi_cpu, msi->msi_vector);
 }
 
 static int
@@ -199,15 +201,35 @@ static void
 msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
 {
 	struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
-
+	int old_vector;
+	u_int old_id;
+	int vector;
+
+	/* Store information to free existing irq. */
+	old_vector = msi->msi_vector;
+	old_id = msi->msi_cpu;
+	if (old_vector && old_id == apic_id)
+		return;
+	/* Allocate IDT vector on this cpu. */
+	vector = apic_alloc_vector(apic_id, msi->msi_irq);
+	if (vector == 0)
+		return; /* XXX alloc_vector panics on failure. */
 	msi->msi_cpu = apic_id;
+	msi->msi_vector = vector;
 	if (bootverbose)
-		printf("msi: Assigning %s IRQ %d to local APIC %u\n",
+		printf("msi: Assigning %s IRQ %d to local APIC %u vector %u\n",
 		    msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
-		    msi->msi_cpu);	
+		    msi->msi_cpu, msi->msi_vector);
 	pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
+	/*
+	 * Free the old vector after the new one is established.  This is done
+	 * to prevent races where we could miss an interrupt.
+	 */
+	if (old_vector)
+		apic_free_vector(old_id, old_vector, msi->msi_irq);
 }
 
+
 void
 msi_init(void)
 {
@@ -263,7 +285,7 @@ int
 msi_alloc(device_t dev, int count, int maxcount, int *irqs)
 {
 	struct msi_intsrc *msi, *fsrc;
-	int cnt, i, vector;
+	int cnt, i;
 
 	if (!msi_enabled)
 		return (ENXIO);
@@ -309,22 +331,12 @@ again:
 	/* Ok, we now have the IRQs allocated. */
 	KASSERT(cnt == count, ("count mismatch"));
 
-	/* Allocate 'count' IDT vectors. */
-	vector = apic_alloc_vectors(irqs, count, maxcount);
-	if (vector == 0) {
-		mtx_unlock(&msi_lock);
-		return (ENOSPC);
-	}
-
 	/* Assign IDT vectors and make these messages owned by 'dev'. */
 	fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
 	for (i = 0; i < count; i++) {
 		msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
 		msi->msi_dev = dev;
-		msi->msi_vector = vector + i;
-		if (bootverbose)
-			printf("msi: routing MSI IRQ %d to vector %u\n",
-			    msi->msi_irq, msi->msi_vector);
+		msi->msi_vector = 0;
 		msi->msi_first = fsrc;
 		KASSERT(msi->msi_intsrc.is_handlers == 0,
 		    ("dead MSI has handlers"));
@@ -377,14 +389,18 @@ msi_release(int *irqs, int count)
 		KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
 		msi->msi_first = NULL;
 		msi->msi_dev = NULL;
-		apic_free_vector(msi->msi_vector, msi->msi_irq);
+		if (msi->msi_vector)
+			apic_free_vector(msi->msi_cpu, msi->msi_vector,
+			    msi->msi_irq);
 		msi->msi_vector = 0;
 	}
 
 	/* Clear out the first message. */
 	first->msi_first = NULL;
 	first->msi_dev = NULL;
-	apic_free_vector(first->msi_vector, first->msi_irq);
+	if (first->msi_vector)
+		apic_free_vector(first->msi_cpu, first->msi_vector,
+		    first->msi_irq);
 	first->msi_vector = 0;
 	first->msi_count = 0;
 
@@ -433,7 +449,7 @@ int
 msix_alloc(device_t dev, int *irq)
 {
 	struct msi_intsrc *msi;
-	int i, vector;
+	int i;
 
 	if (!msi_enabled)
 		return (ENXIO);
@@ -468,15 +484,9 @@ again:
 		goto again;
 	}
 
-	/* Allocate an IDT vector. */
-	vector = apic_alloc_vector(i);
-	if (bootverbose)
-		printf("msi: routing MSI-X IRQ %d to vector %u\n", msi->msi_irq,
-		    vector);
-
 	/* Setup source. */
 	msi->msi_dev = dev;
-	msi->msi_vector = vector;
+	msi->msi_vector = 0;
 	msi->msi_msix = 1;
 
 	KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
@@ -508,7 +518,8 @@ msix_release(int irq)
 
 	/* Clear out the message. */
 	msi->msi_dev = NULL;
-	apic_free_vector(msi->msi_vector, msi->msi_irq);
+	if (msi->msi_vector)
+		apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
 	msi->msi_vector = 0;
 	msi->msi_msix = 0;
 
diff --git a/sys/i386/include/apicvar.h b/sys/i386/include/apicvar.h
index c33c0a9..c64dae3 100644
--- a/sys/i386/include/apicvar.h
+++ b/sys/i386/include/apicvar.h
@@ -187,14 +187,17 @@ inthand_t
 	IDTVEC(apic_isr7), IDTVEC(spuriousint), IDTVEC(timerint);
 
 extern vm_paddr_t lapic_paddr;
-
-u_int	apic_alloc_vector(u_int irq);
-u_int	apic_alloc_vectors(u_int *irqs, u_int count, u_int align);
-void	apic_disable_vector(u_int vector);
-void	apic_enable_vector(u_int vector);
-void	apic_free_vector(u_int vector, u_int irq);
-u_int	apic_idt_to_irq(u_int vector);
+extern int apic_cpuids[];
+
+u_int	apic_alloc_vector(u_int apic_id, u_int irq);
+u_int	apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count,
+	    u_int align);
+void	apic_disable_vector(u_int apic_id, u_int vector);
+void	apic_enable_vector(u_int apic_id, u_int vector);
+void	apic_free_vector(u_int apic_id, u_int vector, u_int irq);
+u_int	apic_idt_to_irq(u_int apic_id, u_int vector);
 void	apic_register_enumerator(struct apic_enumerator *enumerator);
+u_int	apic_cpuid(u_int apic_id);
 void	*ioapic_create(vm_paddr_t addr, int32_t apic_id, int intbase);
 int	ioapic_disable_pin(void *cookie, u_int pin);
 int	ioapic_get_vector(void *cookie, u_int pin);
diff --git a/sys/i386/include/intr_machdep.h b/sys/i386/include/intr_machdep.h
index 2aeb0c9..4593077 100644
--- a/sys/i386/include/intr_machdep.h
+++ b/sys/i386/include/intr_machdep.h
@@ -47,7 +47,7 @@
  * IRQ values beyond 256 are used by MSI.  We leave 255 unused to avoid
  * confusion since 255 is used in PCI to indicate an invalid IRQ.
  */
-#define	NUM_MSI_INTS	128
+#define	NUM_MSI_INTS	512
 #define	FIRST_MSI_INT	256
 #define	NUM_IO_INTS	(FIRST_MSI_INT + NUM_MSI_INTS)