MFC 260239,261268,265058:

Expand the support for PCI INTx interrupts including providing interrupt
routing information for INTx interrupts to I/O APIC pins and enabling
INTx interrupts in the virtio and AHCI backends.

13 files changed, 387 insertions, 126 deletions

diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile
index c16a759..d9950a7 100644
--- a/usr.sbin/bhyve/Makefile
+++ b/usr.sbin/bhyve/Makefile
@@ -7,11 +7,37 @@ PROG=	bhyve
 DEBUG_FLAGS= -g -O0 
 
 MAN=	bhyve.8
-SRCS=	acpi.c atpic.c bhyverun.c block_if.c consport.c dbgport.c elcr.c
-SRCS+=  inout.c legacy_irq.c mem.c mevent.c mptbl.c pci_ahci.c
-SRCS+=	pci_emul.c pci_hostbridge.c pci_lpc.c pci_passthru.c pci_virtio_block.c
-SRCS+=	pci_virtio_net.c pci_uart.c pit_8254.c pm.c pmtmr.c post.c rtc.c
-SRCS+=	uart_emul.c virtio.c xmsr.c spinup_ap.c
+
+SRCS=	\
+	acpi.c			\
+	atpic.c			\
+	bhyverun.c		\
+	block_if.c		\
+	consport.c		\
+	dbgport.c		\
+	elcr.c			\
+	inout.c			\
+	ioapic.c		\
+	mem.c			\
+	mevent.c		\
+	mptbl.c			\
+	pci_ahci.c		\
+	pci_emul.c		\
+	pci_hostbridge.c	\
+	pci_lpc.c		\
+	pci_passthru.c		\
+	pci_virtio_block.c	\
+	pci_virtio_net.c	\
+	pci_uart.c		\
+	pit_8254.c		\
+	pm.c			\
+	pmtmr.c			\
+	post.c			\
+	rtc.c			\
+	uart_emul.c		\
+	virtio.c		\
+	xmsr.c			\
+	spinup_ap.c
 
 .PATH:	${.CURDIR}/../../sys/amd64/vmm
 SRCS+=	vmm_instruction_emul.c
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index dfc7d29..c587aef 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$");
 #include "acpi.h"
 #include "inout.h"
 #include "dbgport.h"
-#include "legacy_irq.h"
+#include "ioapic.h"
 #include "mem.h"
 #include "mevent.h"
 #include "mptbl.h"
@@ -695,7 +695,7 @@ main(int argc, char *argv[])
 
 	init_mem();
 	init_inout();
-	legacy_irq_init();
+	ioapic_init(ctx);
 
 	rtc_init(ctx);
 
diff --git a/usr.sbin/bhyve/legacy_irq.c b/usr.sbin/bhyve/ioapic.c
index 9e86efd..2950d9a 100644
--- a/usr.sbin/bhyve/legacy_irq.c
+++ b/usr.sbin/bhyve/ioapic.c
@@ -1,5 +1,6 @@
 /*-
- * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
+ * Copyright (c) 2014 Advanced Computing Technologies LLC
+ * Written by: John H. Baldwin <jhb@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -11,10 +12,10 @@
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
- * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
@@ -22,59 +23,52 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
- *
- * $FreeBSD$
  */
 
-#include <stdbool.h>
-#include <assert.h>
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+
+#include <machine/vmm.h>
+#include <vmmapi.h>
+
+#include "ioapic.h"
 
 /*
- * Used to keep track of legacy interrupt owners/requestors
+ * Assign PCI INTx interrupts to I/O APIC pins in a round-robin
+ * fashion.  Note that we have no idea what the HPET is using, but the
+ * HPET is also programmable whereas this is intended for hardwired
+ * PCI interrupts.
+ *
+ * This assumes a single I/O APIC where pins >= 16 are permitted for
+ * PCI devices.
  */
-#define NLIRQ		16
-
-static struct lirqinfo {
-	bool	li_generic;
-	bool	li_allocated;
-} lirq[NLIRQ];
+static int pci_pins;
 
 void
-legacy_irq_init(void)
+ioapic_init(struct vmctx *ctx)
 {
 
-	/*
-	 * Allow ISA IRQs 5,10,11,12, and 15 to be available for generic use.
-	 */
-	lirq[5].li_generic = true;
-	lirq[10].li_generic = true;
-	lirq[11].li_generic = true;
-	lirq[12].li_generic = true;
-	lirq[15].li_generic = true;
+	if (vm_ioapic_pincount(ctx, &pci_pins) < 0) {
+		pci_pins = 0;
+		return;
+	}
+
+	/* Ignore the first 16 pins. */
+	if (pci_pins <= 16) {
+		pci_pins = 0;
+		return;
+	}
+	pci_pins -= 16;
 }
 
 int
-legacy_irq_alloc(int irq)
+ioapic_pci_alloc_irq(void)
 {
-	int i;
-
-	assert(irq < NLIRQ);
-
-	if (irq < 0) {
-		for (i = 0; i < NLIRQ; i++) {
-			if (lirq[i].li_generic && !lirq[i].li_allocated) {
-				irq = i;
-				break;
-			}
-		}
-	} else {
-		if (lirq[irq].li_allocated)
-			irq = -1;
-	}
+	static int last_pin;
 
-	if (irq >= 0) {
-		lirq[irq].li_allocated = true;
-		return (irq);
-	} else
+	if (pci_pins == 0)
 		return (-1);
+	return (16 + (last_pin++ % pci_pins));
 }
diff --git a/usr.sbin/bhyve/legacy_irq.h b/usr.sbin/bhyve/ioapic.h
index 3531e11..3cfca4f 100644
--- a/usr.sbin/bhyve/legacy_irq.h
+++ b/usr.sbin/bhyve/ioapic.h
@@ -1,5 +1,6 @@
 /*-
- * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
+ * Copyright (c) 2014 Advanced Computing Technologies LLC
+ * Written by: John H. Baldwin <jhb@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -11,10 +12,10 @@
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
- * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
@@ -26,16 +27,13 @@
  * $FreeBSD$
  */
 
-#ifndef _LEGACY_IRQ_H_
-#define	_LEGACY_IRQ_H_
+#ifndef _IOAPIC_H_
+#define	_IOAPIC_H_
 
 /*
- * Allocate a legacy irq. The argument 'irq' can be set to -1 to allocate any
- * available irq.
- *
- * Returns -1 on failure or the allocated irq number on success.
+ * Allocate a PCI IRQ from the I/O APIC.
  */
-int	legacy_irq_alloc(int irq);
-void	legacy_irq_init(void);
+void	ioapic_init(struct vmctx *ctx);
+int	ioapic_pci_alloc_irq(void);
 
 #endif
diff --git a/usr.sbin/bhyve/mptbl.c b/usr.sbin/bhyve/mptbl.c
index ea332d4..3608efa 100644
--- a/usr.sbin/bhyve/mptbl.c
+++ b/usr.sbin/bhyve/mptbl.c
@@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$");
 #include "acpi.h"
 #include "bhyverun.h"
 #include "mptbl.h"
+#include "pci_emul.h"
 
 #define MPTABLE_BASE		0xF0000
 
@@ -75,9 +76,6 @@ __FBSDID("$FreeBSD$");
 /* Number of local intr entries */
 #define	MPEII_NUM_LOCAL_IRQ	2
 
-/* Number of i/o intr entries */
-#define	MPEII_MAX_IRQ		24
-
 /* Bus entry defines */
 #define MPE_NUM_BUSES		2
 #define MPE_BUSNAME_LEN		6
@@ -195,8 +193,42 @@ mpt_build_ioapic_entries(io_apic_entry_ptr mpei, int id)
 	mpei->apic_address = IOAPIC_PADDR;
 }
 
+static int
+mpt_count_ioint_entries(void)
+{
+
+	/*
+	 * Always include entries for the first 16 pins along with a entry
+	 * for each active PCI INTx pin.
+	 */
+	return (16 + pci_count_lintr());
+}
+
+static void
+mpt_generate_pci_int(int slot, int pin, int ioapic_irq, void *arg)
+{
+	int_entry_ptr *mpiep, mpie;
+
+	mpiep = arg;
+	mpie = *mpiep;
+	memset(mpie, 0, sizeof(*mpie));
+
+	/*
+	 * This is always after another I/O interrupt entry, so cheat
+	 * and fetch the I/O APIC ID from the prior entry.
+	 */
+	mpie->type = MPCT_ENTRY_INT;
+	mpie->int_type = INTENTRY_TYPE_INT;
+	mpie->src_bus_id = 0;
+	mpie->src_bus_irq = slot << 2 | (pin - 1);
+	mpie->dst_apic_id = mpie[-1].dst_apic_id;
+	mpie->dst_apic_int = ioapic_irq;
+
+	*mpiep = mpie + 1;
+}
+
 static void
-mpt_build_ioint_entries(int_entry_ptr mpie, int num_pins, int id)
+mpt_build_ioint_entries(int_entry_ptr mpie, int id)
 {
 	int pin;
 
@@ -206,8 +238,8 @@ mpt_build_ioint_entries(int_entry_ptr mpie, int num_pins, int id)
 	 * just use the default config, tweek later if needed.
 	 */
 
-	/* Run through all 16 pins. */
-	for (pin = 0; pin < num_pins; pin++) {
+	/* First, generate the first 16 pins. */
+	for (pin = 0; pin < 16; pin++) {
 		memset(mpie, 0, sizeof(*mpie));
 		mpie->type = MPCT_ENTRY_INT;
 		mpie->src_bus_id = 1;
@@ -235,16 +267,6 @@ mpt_build_ioint_entries(int_entry_ptr mpie, int num_pins, int id)
 			mpie->int_type = INTENTRY_TYPE_INT;
 			mpie->src_bus_irq = SCI_INT;
 			break;
-		case 5:
-		case 10:
-		case 11:
-			/*
-			 * PCI Irqs set to level triggered and active-lo.
-			 */
-			mpie->int_flags = INTENTRY_FLAGS_POLARITY_ACTIVELO |
-			    INTENTRY_FLAGS_TRIGGER_LEVEL;
-			mpie->src_bus_id = 0;
-			/* fall through.. */
 		default:
 			/* All other pins are identity mapped. */
 			mpie->int_type = INTENTRY_TYPE_INT;
@@ -254,6 +276,8 @@ mpt_build_ioint_entries(int_entry_ptr mpie, int num_pins, int id)
 		mpie++;
 	}
 
+	/* Next, generate entries for any PCI INTx interrupts. */
+	pci_walk_lintr(mpt_generate_pci_int, &mpie); 
 }
 
 void
@@ -273,6 +297,7 @@ mptable_build(struct vmctx *ctx, int ncpu)
 	proc_entry_ptr		mpep;
 	mpfps_t			mpfp;
 	int_entry_ptr		mpie;
+	int			ioints;
 	char 			*curraddr;
 	char 			*startaddr;
 
@@ -307,9 +332,10 @@ mptable_build(struct vmctx *ctx, int ncpu)
 	mpch->entry_count++;
 
 	mpie = (int_entry_ptr) curraddr;
-	mpt_build_ioint_entries(mpie, MPEII_MAX_IRQ, 0);
-	curraddr += sizeof(*mpie) * MPEII_MAX_IRQ;
-	mpch->entry_count += MPEII_MAX_IRQ;
+	ioints = mpt_count_ioint_entries();
+	mpt_build_ioint_entries(mpie, 0);
+	curraddr += sizeof(*mpie) * ioints;
+	mpch->entry_count += ioints;
 
 	mpie = (int_entry_ptr)curraddr;
 	mpt_build_localint_entries(mpie);
diff --git a/usr.sbin/bhyve/pci_ahci.c b/usr.sbin/bhyve/pci_ahci.c
index fb32bad..dd9bfb9 100644
--- a/usr.sbin/bhyve/pci_ahci.c
+++ b/usr.sbin/bhyve/pci_ahci.c
@@ -192,6 +192,7 @@ struct pci_ahci_softc {
 	uint32_t em_ctl;
 	uint32_t cap2;
 	uint32_t bohc;
+	uint32_t lintr;
 	struct ahci_port port[MAX_PORTS];
 };
 #define	ahci_ctx(sc)	((sc)->asc_pi->pi_vmctx)
@@ -211,8 +212,11 @@ static inline void lba_to_msf(uint8_t *buf, int lba)
 static void
 ahci_generate_intr(struct pci_ahci_softc *sc)
 {
+	struct pci_devinst *pi;
 	int i;
 
+	pi = sc->asc_pi;
+
 	for (i = 0; i < sc->ports; i++) {
 		struct ahci_port *pr;
 		pr = &sc->port[i];
@@ -222,8 +226,28 @@ ahci_generate_intr(struct pci_ahci_softc *sc)
 
 	DPRINTF("%s %x\n", __func__, sc->is);
 
-	if (sc->is && (sc->ghc & AHCI_GHC_IE))
-		pci_generate_msi(sc->asc_pi, 0);
+	if (sc->is && (sc->ghc & AHCI_GHC_IE)) {		
+		if (pci_msi_enabled(pi)) {
+			/*
+			 * Generate an MSI interrupt on every edge
+			 */
+			pci_generate_msi(pi, 0);
+		} else if (!sc->lintr) {
+			/*
+			 * Only generate a pin-based interrupt if one wasn't
+			 * in progress
+			 */
+			sc->lintr = 1;
+			pci_lintr_assert(pi);
+		}
+	} else if (sc->lintr) {
+		/*
+		 * No interrupts: deassert pin-based signal if it had
+		 * been asserted
+		 */
+		pci_lintr_deassert(pi);
+		sc->lintr = 0;
+	}
 }
 
 static void
@@ -367,6 +391,12 @@ ahci_reset(struct pci_ahci_softc *sc)
 
 	sc->ghc = AHCI_GHC_AE;
 	sc->is = 0;
+
+	if (sc->lintr) {
+		pci_lintr_deassert(sc->asc_pi);
+		sc->lintr = 0;
+	}
+
 	for (i = 0; i < sc->ports; i++) {
 		sc->port[i].ie = 0;
 		sc->port[i].is = 0;
@@ -1815,6 +1845,8 @@ pci_ahci_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts, int atapi)
 	pci_emul_alloc_bar(pi, 5, PCIBAR_MEM32,
 	    AHCI_OFFSET + sc->ports * AHCI_STEP);
 
+	pci_lintr_request(pi);
+
 open_fail:
 	if (ret) {
 		blockif_close(sc->port[0].bctx);
diff --git a/usr.sbin/bhyve/pci_emul.c b/usr.sbin/bhyve/pci_emul.c
index 96ba5d7..0b50135 100644
--- a/usr.sbin/bhyve/pci_emul.c
+++ b/usr.sbin/bhyve/pci_emul.c
@@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/errno.h>
 
 #include <ctype.h>
+#include <pthread.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -47,7 +48,7 @@ __FBSDID("$FreeBSD$");
 #include "acpi.h"
 #include "bhyverun.h"
 #include "inout.h"
-#include "legacy_irq.h"
+#include "ioapic.h"
 #include "mem.h"
 #include "pci_emul.h"
 #include "pci_lpc.h"
@@ -71,11 +72,21 @@ do {									\
 #define MAXSLOTS	(PCI_SLOTMAX + 1)
 #define	MAXFUNCS	(PCI_FUNCMAX + 1)
 
-static struct slotinfo {
-	char	*si_name;
-	char	*si_param;
-	struct pci_devinst *si_devi;
-} pci_slotinfo[MAXSLOTS][MAXFUNCS];
+struct funcinfo {
+	char	*fi_name;
+	char	*fi_param;
+	struct pci_devinst *fi_devi;
+};
+
+struct intxinfo {
+	int	ii_count;
+	int	ii_ioapic_irq;
+};
+
+struct slotinfo {
+	struct intxinfo si_intpins[4];
+	struct funcinfo si_funcs[MAXFUNCS];
+} pci_slotinfo[MAXSLOTS];
 
 SET_DECLARE(pci_devemu_set, struct pci_devemu);
 
@@ -92,6 +103,7 @@ static uint64_t pci_emul_membase64;
 #define	PCI_EMUL_MEMLIMIT64	0xFD00000000UL
 
 static struct pci_devemu *pci_emul_finddev(char *name);
+static void	pci_lintr_update(struct pci_devinst *pi);
 
 static int pci_emul_devices;
 static struct mem_range pci_mem_hole;
@@ -154,7 +166,7 @@ pci_parse_slot(char *opt)
 		goto done;
 	}
 
-	if (pci_slotinfo[snum][fnum].si_name != NULL) {
+	if (pci_slotinfo[snum].si_funcs[fnum].fi_name != NULL) {
 		fprintf(stderr, "pci slot %d:%d already occupied!\n",
 			snum, fnum);
 		goto done;
@@ -167,8 +179,8 @@ pci_parse_slot(char *opt)
 	}
 
 	error = 0;
-	pci_slotinfo[snum][fnum].si_name = emul;
-	pci_slotinfo[snum][fnum].si_param = config;
+	pci_slotinfo[snum].si_funcs[fnum].fi_name = emul;
+	pci_slotinfo[snum].si_funcs[fnum].fi_param = config;
 
 done:
 	if (error)
@@ -666,7 +678,10 @@ pci_emul_init(struct vmctx *ctx, struct pci_devemu *pde, int slot, int func,
 	pdi->pi_bus = 0;
 	pdi->pi_slot = slot;
 	pdi->pi_func = func;
-	pdi->pi_lintr_pin = -1;
+	pthread_mutex_init(&pdi->pi_lintr.lock, NULL);
+	pdi->pi_lintr.pin = 0;
+	pdi->pi_lintr.state = IDLE;
+	pdi->pi_lintr.ioapic_irq = 0;
 	pdi->pi_d = pde;
 	snprintf(pdi->pi_name, PI_NAMESZ, "%s-pci-%d", pde->pe_emu, slot);
 
@@ -682,7 +697,7 @@ pci_emul_init(struct vmctx *ctx, struct pci_devemu *pde, int slot, int func,
 		free(pdi);
 	} else {
 		pci_emul_devices++;
-		pci_slotinfo[slot][func].si_devi = pdi;
+		pci_slotinfo[slot].si_funcs[func].fi_devi = pdi;
 	}
 
 	return (err);
@@ -816,6 +831,7 @@ msixcap_cfgwrite(struct pci_devinst *pi, int capoff, int offset,
 
 		pi->pi_msix.enabled = val & PCIM_MSIXCTRL_MSIX_ENABLE;
 		pi->pi_msix.function_mask = val & PCIM_MSIXCTRL_FUNCTION_MASK;
+		pci_lintr_update(pi);
 	} 
 	
 	CFGWRITE(pi, offset, val, bytes);
@@ -854,6 +870,7 @@ msicap_cfgwrite(struct pci_devinst *pi, int capoff, int offset,
 		} else {
 			pi->pi_msi.maxmsgnum = 0;
 		}
+		pci_lintr_update(pi);
 	}
 
 	CFGWRITE(pi, offset, val, bytes);
@@ -993,7 +1010,7 @@ int
 init_pci(struct vmctx *ctx)
 {
 	struct pci_devemu *pde;
-	struct slotinfo *si;
+	struct funcinfo *fi;
 	size_t lowmem;
 	int slot, func;
 	int error;
@@ -1004,12 +1021,12 @@ init_pci(struct vmctx *ctx)
 
 	for (slot = 0; slot < MAXSLOTS; slot++) {
 		for (func = 0; func < MAXFUNCS; func++) {
-			si = &pci_slotinfo[slot][func];
-			if (si->si_name != NULL) {
-				pde = pci_emul_finddev(si->si_name);
+			fi = &pci_slotinfo[slot].si_funcs[func];
+			if (fi->fi_name != NULL) {
+				pde = pci_emul_finddev(fi->fi_name);
 				assert(pde != NULL);
 				error = pci_emul_init(ctx, pde, slot, func,
-					    si->si_param);
+					    fi->fi_param);
 				if (error)
 					return (error);
 			}
@@ -1042,11 +1059,27 @@ init_pci(struct vmctx *ctx)
 	return (0);
 }
 
+static void
+pci_prt_entry(int slot, int pin, int ioapic_irq, void *arg)
+{
+	int *count;
+
+	count = arg;
+	dsdt_line("  Package (0x04)");
+	dsdt_line("  {");
+	dsdt_line("    0x%X,", slot << 16 | 0xffff);
+	dsdt_line("    0x%02X,", pin - 1);
+	dsdt_line("    Zero,");
+	dsdt_line("    0x%X", ioapic_irq);
+	dsdt_line("  }%s", *count == 1 ? "" : ",");
+	(*count)--;
+}
+
 void
 pci_write_dsdt(void)
 {
 	struct pci_devinst *pi;
-	int slot, func;
+	int count, slot, func;
 
 	dsdt_indent(1);
 	dsdt_line("Scope (_SB)");
@@ -1107,11 +1140,20 @@ pci_write_dsdt(void)
 	    PCI_EMUL_MEMLIMIT64 - PCI_EMUL_MEMBASE64);
 	dsdt_line("        ,, , AddressRangeMemory, TypeStatic)");
 	dsdt_line("    })");
+	count = pci_count_lintr();
+	if (count != 0) {
+		dsdt_indent(2);
+		dsdt_line("Name (_PRT, Package (0x%02X)", count);
+		dsdt_line("{");
+		pci_walk_lintr(pci_prt_entry, &count);
+		dsdt_line("})");
+		dsdt_unindent(2);
+	}
 
 	dsdt_indent(2);
 	for (slot = 0; slot < MAXSLOTS; slot++) {
 		for (func = 0; func < MAXFUNCS; func++) {
-			pi = pci_slotinfo[slot][func].si_devi;
+			pi = pci_slotinfo[slot].si_funcs[func].fi_devi;
 			if (pi != NULL && pi->pi_d->pe_write_dsdt != NULL)
 				pi->pi_d->pe_write_dsdt(pi);
 		}
@@ -1176,18 +1218,54 @@ pci_generate_msi(struct pci_devinst *pi, int index)
 	}
 }
 
+static bool
+pci_lintr_permitted(struct pci_devinst *pi)
+{
+	uint16_t cmd;
+
+	cmd = pci_get_cfgdata16(pi, PCIR_COMMAND);
+	return (!(pi->pi_msi.enabled || pi->pi_msix.enabled ||
+		(cmd & PCIM_CMD_INTxDIS)));
+}
+
 int
-pci_lintr_request(struct pci_devinst *pi, int req)
+pci_lintr_request(struct pci_devinst *pi)
 {
-	int irq;
+	struct slotinfo *si;
+	int bestpin, bestcount, irq, pin;
 
-	irq = legacy_irq_alloc(req);
-	if (irq < 0)
-		return (-1);
+	/*
+	 * First, allocate a pin from our slot.
+	 */
+	si = &pci_slotinfo[pi->pi_slot];
+	bestpin = 0;
+	bestcount = si->si_intpins[0].ii_count;
+	for (pin = 1; pin < 4; pin++) {
+		if (si->si_intpins[pin].ii_count < bestcount) {
+			bestpin = pin;
+			bestcount = si->si_intpins[pin].ii_count;
+		}
+	}
+
+	/*
+	 * Attempt to allocate an I/O APIC pin for this intpin.  If
+	 * 8259A support is added we will need a separate field to
+	 * assign the intpin to an input pin on the PCI interrupt
+	 * router.
+	 */
+	if (si->si_intpins[bestpin].ii_count == 0) {
+		irq = ioapic_pci_alloc_irq();
+		if (irq < 0)
+			return (-1);		
+		si->si_intpins[bestpin].ii_ioapic_irq = irq;
+	} else
+		irq = si->si_intpins[bestpin].ii_ioapic_irq;
+	si->si_intpins[bestpin].ii_count++;
 
-	pi->pi_lintr_pin = irq;
+	pi->pi_lintr.pin = bestpin + 1;
+	pi->pi_lintr.ioapic_irq = irq;
 	pci_set_cfgdata8(pi, PCIR_INTLINE, irq);
-	pci_set_cfgdata8(pi, PCIR_INTPIN, 1);
+	pci_set_cfgdata8(pi, PCIR_INTPIN, bestpin + 1);
 	return (0);
 }
 
@@ -1195,23 +1273,77 @@ void
 pci_lintr_assert(struct pci_devinst *pi)
 {
 
-	assert(pi->pi_lintr_pin >= 0);
+	assert(pi->pi_lintr.pin > 0);
 
-	if (pi->pi_lintr_state == 0) {
-		pi->pi_lintr_state = 1;
-		vm_ioapic_assert_irq(pi->pi_vmctx, pi->pi_lintr_pin);
+	pthread_mutex_lock(&pi->pi_lintr.lock);
+	if (pi->pi_lintr.state == IDLE) {
+		if (pci_lintr_permitted(pi)) {
+			pi->pi_lintr.state = ASSERTED;
+			vm_ioapic_assert_irq(pi->pi_vmctx,
+			    pi->pi_lintr.ioapic_irq);
+		} else
+			pi->pi_lintr.state = PENDING;
 	}
+	pthread_mutex_unlock(&pi->pi_lintr.lock);
 }
 
 void
 pci_lintr_deassert(struct pci_devinst *pi)
 {
 
-	assert(pi->pi_lintr_pin >= 0);
+	assert(pi->pi_lintr.pin > 0);
+
+	pthread_mutex_lock(&pi->pi_lintr.lock);
+	if (pi->pi_lintr.state == ASSERTED) {
+		pi->pi_lintr.state = IDLE;
+		vm_ioapic_deassert_irq(pi->pi_vmctx, pi->pi_lintr.ioapic_irq);
+	} else if (pi->pi_lintr.state == PENDING)
+		pi->pi_lintr.state = IDLE;
+	pthread_mutex_unlock(&pi->pi_lintr.lock);
+}
+
+static void
+pci_lintr_update(struct pci_devinst *pi)
+{
 
-	if (pi->pi_lintr_state == 1) {
-		pi->pi_lintr_state = 0;
-		vm_ioapic_deassert_irq(pi->pi_vmctx, pi->pi_lintr_pin);
+	pthread_mutex_lock(&pi->pi_lintr.lock);
+	if (pi->pi_lintr.state == ASSERTED && !pci_lintr_permitted(pi)) {
+		vm_ioapic_deassert_irq(pi->pi_vmctx, pi->pi_lintr.ioapic_irq);
+		pi->pi_lintr.state = PENDING;
+	} else if (pi->pi_lintr.state == PENDING && pci_lintr_permitted(pi)) {
+		pi->pi_lintr.state = ASSERTED;
+		vm_ioapic_assert_irq(pi->pi_vmctx, pi->pi_lintr.ioapic_irq);
+	}
+	pthread_mutex_unlock(&pi->pi_lintr.lock);
+}
+
+int
+pci_count_lintr(void)
+{
+	int count, slot, pin;
+
+	count = 0;
+	for (slot = 0; slot < MAXSLOTS; slot++) {
+		for (pin = 0; pin < 4; pin++) {
+			if (pci_slotinfo[slot].si_intpins[pin].ii_count != 0)
+				count++;
+		}
+	}
+	return (count);
+}
+
+void
+pci_walk_lintr(pci_lintr_cb cb, void *arg)
+{
+	struct intxinfo *ii;
+	int slot, pin;
+
+	for (slot = 0; slot < MAXSLOTS; slot++) {
+		for (pin = 0; pin < 4; pin++) {
+			ii = &pci_slotinfo[slot].si_intpins[pin];
+			if (ii->ii_count != 0)
+				cb(slot, pin + 1, ii->ii_ioapic_irq, arg);
+		}
 	}
 }
 
@@ -1226,7 +1358,7 @@ pci_emul_is_mfdev(int slot)
 
 	numfuncs = 0;
 	for (f = 0; f < MAXFUNCS; f++) {
-		if (pci_slotinfo[slot][f].si_devi != NULL) {
+		if (pci_slotinfo[slot].si_funcs[f].fi_devi != NULL) {
 			numfuncs++;
 		}
 	}
@@ -1348,6 +1480,12 @@ pci_emul_cmdwrite(struct pci_devinst *pi, uint32_t new, int bytes)
 				assert(0); 
 		}
 	}
+
+	/*
+	 * If INTx has been unmasked and is pending, assert the
+	 * interrupt.
+	 */
+	pci_lintr_update(pi);
 }	
 
 static int
@@ -1362,7 +1500,7 @@ pci_emul_cfgdata(struct vmctx *ctx, int vcpu, int in, int port, int bytes,
 	assert(bytes == 1 || bytes == 2 || bytes == 4);
 	
 	if (cfgbus == 0)
-		pi = pci_slotinfo[cfgslot][cfgfunc].si_devi;
+		pi = pci_slotinfo[cfgslot].si_funcs[cfgfunc].fi_devi;
 	else
 		pi = NULL;
 
diff --git a/usr.sbin/bhyve/pci_emul.h b/usr.sbin/bhyve/pci_emul.h
index cb559f1..480b631 100644
--- a/usr.sbin/bhyve/pci_emul.h
+++ b/usr.sbin/bhyve/pci_emul.h
@@ -32,6 +32,7 @@
 #include <sys/types.h>
 #include <sys/queue.h>
 #include <sys/kernel.h>
+#include <sys/_pthreadtypes.h>
 
 #include <dev/pci/pcireg.h>
 
@@ -102,16 +103,27 @@ struct msix_table_entry {
 #define MAX_MSIX_TABLE_ENTRIES	2048
 #define PBA_TABLE_ENTRY_SIZE	8
 
+enum lintr_stat {
+	IDLE,
+	ASSERTED,
+	PENDING
+};
+
 struct pci_devinst {
 	struct pci_devemu *pi_d;
 	struct vmctx *pi_vmctx;
 	uint8_t	  pi_bus, pi_slot, pi_func;
-	int8_t    pi_lintr_pin;
-	int8_t	  pi_lintr_state;
 	char	  pi_name[PI_NAMESZ];
 	int	  pi_bar_getsize;
 
 	struct {
+		int8_t    	pin;
+		enum lintr_stat	state;
+		int	  	ioapic_irq;
+		pthread_mutex_t	lock;
+	} pi_lintr;
+
+	struct {
 		int		enabled;
 		uint64_t	addr;
 		uint64_t	msg_data;
@@ -187,6 +199,8 @@ struct pciecap {
 	uint16_t	slot_status2;
 } __packed;
 
+typedef void (*pci_lintr_cb)(int slot, int pin, int ioapic_irq, void *arg);
+
 int	init_pci(struct vmctx *ctx);
 void	msicap_cfgwrite(struct pci_devinst *pi, int capoff, int offset,
 	    int bytes, uint32_t val);
@@ -203,7 +217,7 @@ void	pci_generate_msi(struct pci_devinst *pi, int msgnum);
 void	pci_generate_msix(struct pci_devinst *pi, int msgnum);
 void	pci_lintr_assert(struct pci_devinst *pi);
 void	pci_lintr_deassert(struct pci_devinst *pi);
-int	pci_lintr_request(struct pci_devinst *pi, int ivec);
+int	pci_lintr_request(struct pci_devinst *pi);
 int	pci_msi_enabled(struct pci_devinst *pi);
 int	pci_msix_enabled(struct pci_devinst *pi);
 int	pci_msix_table_bar(struct pci_devinst *pi);
@@ -215,6 +229,8 @@ int	pci_emul_add_msixcap(struct pci_devinst *pi, int msgnum, int barnum);
 int	pci_emul_msix_twrite(struct pci_devinst *pi, uint64_t offset, int size,
 			     uint64_t value);
 uint64_t pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int size);
+int	pci_count_lintr(void);
+void	pci_walk_lintr(pci_lintr_cb cb, void *arg);
 void	pci_write_dsdt(void);
 
 static __inline void 
diff --git a/usr.sbin/bhyve/pci_uart.c b/usr.sbin/bhyve/pci_uart.c
index 38df832..21b93bf 100644
--- a/usr.sbin/bhyve/pci_uart.c
+++ b/usr.sbin/bhyve/pci_uart.c
@@ -91,7 +91,7 @@ pci_uart_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	struct uart_softc *sc;
 
 	pci_emul_alloc_bar(pi, 0, PCIBAR_IO, UART_IO_BAR_SIZE);
-	pci_lintr_request(pi, -1);
+	pci_lintr_request(pi);
 
 	/* initialize config space */
 	pci_set_cfgdata16(pi, PCIR_DEVICE, COM_DEV);
diff --git a/usr.sbin/bhyve/pci_virtio_block.c b/usr.sbin/bhyve/pci_virtio_block.c
index a642b11..2f37804 100644
--- a/usr.sbin/bhyve/pci_virtio_block.c
+++ b/usr.sbin/bhyve/pci_virtio_block.c
@@ -117,6 +117,7 @@ static int pci_vtblk_debug;
  */
 struct pci_vtblk_softc {
 	struct virtio_softc vbsc_vs;
+	pthread_mutex_t vsc_mtx;
 	struct vqueue_info vbsc_vq;
 	int		vbsc_fd;
 	struct vtblk_config vbsc_cfg;	
@@ -304,8 +305,12 @@ pci_vtblk_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	/* record fd of storage device/file */
 	sc->vbsc_fd = fd;
 
+	pthread_mutex_init(&sc->vsc_mtx, NULL);
+
 	/* init virtio softc and virtqueues */
 	vi_softc_linkup(&sc->vbsc_vs, &vtblk_vi_consts, sc, pi, &sc->vbsc_vq);
+	sc->vbsc_vs.vs_mtx = &sc->vsc_mtx;
+
 	sc->vbsc_vq.vq_qsize = VTBLK_RINGSZ;
 	/* sc->vbsc_vq.vq_notify = we have no per-queue notify */
 
@@ -339,6 +344,8 @@ pci_vtblk_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
 	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_BLOCK);
 
+	pci_lintr_request(pi);
+
 	if (vi_intr_init(&sc->vbsc_vs, 1, fbsdrun_virtio_msix()))
 		return (1);
 	vi_set_io_bar(&sc->vbsc_vs, 0);
diff --git a/usr.sbin/bhyve/pci_virtio_net.c b/usr.sbin/bhyve/pci_virtio_net.c
index 57ab041..e7b54bc 100644
--- a/usr.sbin/bhyve/pci_virtio_net.c
+++ b/usr.sbin/bhyve/pci_virtio_net.c
@@ -519,6 +519,8 @@ pci_vtnet_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pthread_mutex_init(&sc->vsc_mtx, NULL);
 
 	vi_softc_linkup(&sc->vsc_vs, &vtnet_vi_consts, sc, pi, sc->vsc_queues);
+	sc->vsc_vs.vs_mtx = &sc->vsc_mtx;
+
 	sc->vsc_queues[VTNET_RXQ].vq_qsize = VTNET_RINGSZ;
 	sc->vsc_queues[VTNET_RXQ].vq_notify = pci_vtnet_ping_rxq;
 	sc->vsc_queues[VTNET_TXQ].vq_qsize = VTNET_RINGSZ;
@@ -608,6 +610,8 @@ pci_vtnet_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK);
 	pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_TYPE_NET);
 
+	pci_lintr_request(pi);
+
 	/* link always up */
 	sc->vsc_config.status = 1;
 	
diff --git a/usr.sbin/bhyve/virtio.c b/usr.sbin/bhyve/virtio.c
index bd98990..4e58dd6 100644
--- a/usr.sbin/bhyve/virtio.c
+++ b/usr.sbin/bhyve/virtio.c
@@ -99,7 +99,11 @@ vi_reset_dev(struct virtio_softc *vs)
 	vs->vs_negotiated_caps = 0;
 	vs->vs_curq = 0;
 	/* vs->vs_status = 0; -- redundant */
+	VS_LOCK(vs);
+	if (vs->vs_isr)
+		pci_lintr_deassert(vs->vs_pi);
 	vs->vs_isr = 0;
+	VS_UNLOCK(vs);
 	vs->vs_msix_cfg_idx = VIRTIO_MSI_NO_VECTOR;
 }
 
@@ -137,11 +141,10 @@ vi_intr_init(struct virtio_softc *vs, int barnum, int use_msix)
 		nvec = vs->vs_vc->vc_nvq + 1;
 		if (pci_emul_add_msixcap(vs->vs_pi, nvec, barnum))
 			return (1);
-	} else {
+	} else
 		vs->vs_flags &= ~VIRTIO_USE_MSIX;
-		/* Only 1 MSI vector for bhyve */
-		pci_emul_add_msicap(vs->vs_pi, 1);
-	}
+	/* Only 1 MSI vector for bhyve */
+	pci_emul_add_msicap(vs->vs_pi, 1);
 	return (0);
 }
 
@@ -591,6 +594,8 @@ bad:
 	case VTCFG_R_ISR:
 		value = vs->vs_isr;
 		vs->vs_isr = 0;		/* a read clears this flag */
+		if (value)
+			pci_lintr_deassert(pi);
 		break;
 	case VTCFG_R_CFGVEC:
 		value = vs->vs_msix_cfg_idx;
diff --git a/usr.sbin/bhyve/virtio.h b/usr.sbin/bhyve/virtio.h
index 8975bf7..7877bae5 100644
--- a/usr.sbin/bhyve/virtio.h
+++ b/usr.sbin/bhyve/virtio.h
@@ -328,6 +328,18 @@ struct virtio_softc {
 	uint16_t vs_msix_cfg_idx;	/* MSI-X vector for config event */
 };
 
+#define	VS_LOCK(vs)							\
+do {									\
+	if (vs->vs_mtx)							\
+		pthread_mutex_lock(vs->vs_mtx);				\
+} while (0)
+
+#define	VS_UNLOCK(vs)							\
+do {									\
+	if (vs->vs_mtx)							\
+		pthread_mutex_unlock(vs->vs_mtx);			\
+} while (0)
+
 struct virtio_consts {
 	const char *vc_name;		/* name of driver (for diagnostics) */
 	int	vc_nvq;			/* number of virtual queues */
@@ -431,11 +443,14 @@ static inline void
 vq_interrupt(struct virtio_softc *vs, struct vqueue_info *vq)
 {
 
-	if (vs->vs_flags & VIRTIO_USE_MSIX)
+	if (pci_msix_enabled(vs->vs_pi))
 		pci_generate_msix(vs->vs_pi, vq->vq_msix_idx);
 	else {
+		VS_LOCK(vs);
 		vs->vs_isr |= VTCFG_ISR_QUEUES;
 		pci_generate_msi(vs->vs_pi, 0);
+		pci_lintr_assert(vs->vs_pi);
+		VS_UNLOCK(vs);
 	}
 }