20 files changed, 927 insertions, 40 deletions

diff --git a/sys/amd64/include/intr_machdep.h b/sys/amd64/include/intr_machdep.h
index fb71b5a..0a5ced4 100644
--- a/sys/amd64/include/intr_machdep.h
+++ b/sys/amd64/include/intr_machdep.h
@@ -106,6 +106,7 @@ struct pic {
 	int (*pic_config_intr)(struct intsrc *, enum intr_trigger,
 	    enum intr_polarity);
 	int (*pic_assign_cpu)(struct intsrc *, u_int apic_id);
+	void (*pic_reprogram_pin)(struct intsrc *);
 	TAILQ_ENTRY(pic) pics;
 };
 
@@ -172,6 +173,7 @@ int	intr_register_source(struct intsrc *isrc);
 int	intr_remove_handler(void *cookie);
 void	intr_resume(bool suspend_cancelled);
 void	intr_suspend(void);
+void	intr_reprogram(void);
 void	intrcnt_add(const char *name, u_long **countp);
 void	nexus_add_irq(u_long irq);
 int	msi_alloc(device_t dev, int count, int maxcount, int *irqs);
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 7e0fb50..4910903 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -542,6 +542,7 @@ x86/iommu/intel_drv.c		optional	acpi acpi_dmar pci
 x86/iommu/intel_fault.c		optional	acpi acpi_dmar pci
 x86/iommu/intel_gas.c		optional	acpi acpi_dmar pci
 x86/iommu/intel_idpgtbl.c	optional	acpi acpi_dmar pci
+x86/iommu/intel_intrmap.c	optional	acpi acpi_dmar pci
 x86/iommu/intel_qi.c		optional	acpi acpi_dmar pci
 x86/iommu/intel_quirks.c	optional	acpi acpi_dmar pci
 x86/iommu/intel_utils.c		optional	acpi acpi_dmar pci
diff --git a/sys/conf/files.i386 b/sys/conf/files.i386
index 5bd584b..1873514 100644
--- a/sys/conf/files.i386
+++ b/sys/conf/files.i386
@@ -560,6 +560,7 @@ x86/iommu/intel_drv.c		optional acpi acpi_dmar pci
 x86/iommu/intel_fault.c		optional acpi acpi_dmar pci
 x86/iommu/intel_gas.c		optional acpi acpi_dmar pci
 x86/iommu/intel_idpgtbl.c	optional acpi acpi_dmar pci
+x86/iommu/intel_intrmap.c	optional acpi acpi_dmar pci
 x86/iommu/intel_qi.c		optional acpi acpi_dmar pci
 x86/iommu/intel_quirks.c	optional acpi acpi_dmar pci
 x86/iommu/intel_utils.c		optional acpi acpi_dmar pci
diff --git a/sys/i386/include/intr_machdep.h b/sys/i386/include/intr_machdep.h
index 8fb61a5..082b649 100644
--- a/sys/i386/include/intr_machdep.h
+++ b/sys/i386/include/intr_machdep.h
@@ -112,6 +112,7 @@ struct pic {
 	int (*pic_config_intr)(struct intsrc *, enum intr_trigger,
 	    enum intr_polarity);
 	int (*pic_assign_cpu)(struct intsrc *, u_int apic_id);
+	void (*pic_reprogram_pin)(struct intsrc *);
 	TAILQ_ENTRY(pic) pics;
 };
 
@@ -168,6 +169,7 @@ int	intr_register_source(struct intsrc *isrc);
 int	intr_remove_handler(void *cookie);
 void	intr_resume(bool suspend_cancelled);
 void	intr_suspend(void);
+void	intr_reprogram(void);
 void	intrcnt_add(const char *name, u_long **countp);
 void	nexus_add_irq(u_long irq);
 int	msi_alloc(device_t dev, int count, int maxcount, int *irqs);
diff --git a/sys/x86/include/apicvar.h b/sys/x86/include/apicvar.h
index 0ed65de..818a831 100644
--- a/sys/x86/include/apicvar.h
+++ b/sys/x86/include/apicvar.h
@@ -155,6 +155,11 @@
 #define	APIC_BUS_PCI		2
 #define	APIC_BUS_MAX		APIC_BUS_PCI
 
+#define	IRQ_EXTINT		(NUM_IO_INTS + 1)
+#define	IRQ_NMI			(NUM_IO_INTS + 2)
+#define	IRQ_SMI			(NUM_IO_INTS + 3)
+#define	IRQ_DISABLED		(NUM_IO_INTS + 4)
+
 /*
  * An APIC enumerator is a psuedo bus driver that enumerates APIC's including
  * CPU's and I/O APIC's.
diff --git a/sys/x86/iommu/busdma_dmar.c b/sys/x86/iommu/busdma_dmar.c
index 4efd7b8..3f86c78 100644
--- a/sys/x86/iommu/busdma_dmar.c
+++ b/sys/x86/iommu/busdma_dmar.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
 #include <sys/uio.h>
+#include <sys/vmem.h>
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 #include <vm/vm.h>
@@ -92,7 +93,7 @@ dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
  * domain, and must collectively be assigned to use either DMAR or
  * bounce mapping.
  */
-static device_t
+device_t
 dmar_get_requester(device_t dev, uint16_t *rid)
 {
 	devclass_t pci_class;
@@ -255,6 +256,8 @@ dmar_get_dma_tag(device_t dev, device_t child)
 	/* Not in scope of any DMAR ? */
 	if (dmar == NULL)
 		return (NULL);
+	if (!dmar->dma_enabled)
+		return (NULL);
 	dmar_quirks_pre_use(dmar);
 	dmar_instantiate_rmrr_ctxs(dmar);
 
@@ -852,6 +855,8 @@ int
 dmar_init_busdma(struct dmar_unit *unit)
 {
 
+	unit->dma_enabled = 1;
+	TUNABLE_INT_FETCH("hw.dmar.dma", &unit->dma_enabled);
 	TAILQ_INIT(&unit->delayed_maps);
 	TASK_INIT(&unit->dmamap_load_task, 0, dmar_bus_task_dmamap, unit);
 	unit->delayed_taskqueue = taskqueue_create("dmar", M_WAITOK,
diff --git a/sys/x86/iommu/intel_ctx.c b/sys/x86/iommu/intel_ctx.c
index 2106da1..a18adcf 100644
--- a/sys/x86/iommu/intel_ctx.c
+++ b/sys/x86/iommu/intel_ctx.c
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
 #include <sys/uio.h>
+#include <sys/vmem.h>
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
 #include <vm/vm_kern.h>
diff --git a/sys/x86/iommu/intel_dmar.h b/sys/x86/iommu/intel_dmar.h
index 0f5c6c9..2865ab5 100644
--- a/sys/x86/iommu/intel_dmar.h
+++ b/sys/x86/iommu/intel_dmar.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2013 The FreeBSD Foundation
+ * Copyright (c) 2013-2015 The FreeBSD Foundation
  * All rights reserved.
  *
  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
@@ -185,6 +185,13 @@ struct dmar_unit {
 	u_int inv_seq_waiters;	/* count of waiters for seq */
 	u_int inv_queue_full;	/* informational counter */
 
+	/* IR */
+	int ir_enabled;
+	vm_paddr_t irt_phys;
+	dmar_irte_t *irt;
+	u_int irte_cnt;
+	vmem_t *irtids;
+
 	/* Delayed freeing of map entries queue processing */
 	struct dmar_map_entries_tailq tlb_flush_entries;
 	struct task qi_task;
@@ -194,6 +201,8 @@ struct dmar_unit {
 	struct task dmamap_load_task;
 	TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps;
 	struct taskqueue *delayed_taskqueue;
+
+	int dma_enabled;
 };
 
 #define	DMAR_LOCK(dmar)		mtx_lock(&(dmar)->lock)
@@ -206,12 +215,16 @@ struct dmar_unit {
 
 #define	DMAR_IS_COHERENT(dmar)	(((dmar)->hw_ecap & DMAR_ECAP_C) != 0)
 #define	DMAR_HAS_QI(dmar)	(((dmar)->hw_ecap & DMAR_ECAP_QI) != 0)
+#define	DMAR_X2APIC(dmar) \
+	(x2apic_mode && ((dmar)->hw_ecap & DMAR_ECAP_EIM) != 0)
 
 /* Barrier ids */
 #define	DMAR_BARRIER_RMRR	0
 #define	DMAR_BARRIER_USEQ	1
 
 struct dmar_unit *dmar_find(device_t dev);
+struct dmar_unit *dmar_find_hpet(device_t dev, uint16_t *rid);
+struct dmar_unit *dmar_find_ioapic(u_int apic_id, uint16_t *rid);
 
 u_int dmar_nd2mask(u_int nd);
 bool dmar_pglvl_supported(struct dmar_unit *unit, int pglvl);
@@ -238,6 +251,9 @@ void dmar_flush_ctx_to_ram(struct dmar_unit *unit, dmar_ctx_entry_t *dst);
 void dmar_flush_root_to_ram(struct dmar_unit *unit, dmar_root_entry_t *dst);
 int dmar_enable_translation(struct dmar_unit *unit);
 int dmar_disable_translation(struct dmar_unit *unit);
+int dmar_load_irt_ptr(struct dmar_unit *unit);
+int dmar_enable_ir(struct dmar_unit *unit);
+int dmar_disable_ir(struct dmar_unit *unit);
 bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id);
 void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id);
 
@@ -256,6 +272,8 @@ void dmar_qi_invalidate_locked(struct dmar_ctx *ctx, dmar_gaddr_t start,
     dmar_gaddr_t size, struct dmar_qi_genseq *pseq);
 void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
 void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
+void dmar_qi_invalidate_iec_glob(struct dmar_unit *unit);
+void dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt);
 
 vm_object_t ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr);
 void put_idmap_pgtbl(vm_object_t obj);
@@ -282,6 +300,7 @@ void dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free);
 
 int dmar_init_busdma(struct dmar_unit *unit);
 void dmar_fini_busdma(struct dmar_unit *unit);
+device_t dmar_get_requester(device_t dev, uint16_t *rid);
 
 void dmar_gas_init_ctx(struct dmar_ctx *ctx);
 void dmar_gas_fini_ctx(struct dmar_ctx *ctx);
@@ -304,6 +323,9 @@ int dmar_instantiate_rmrr_ctxs(struct dmar_unit *dmar);
 void dmar_quirks_post_ident(struct dmar_unit *dmar);
 void dmar_quirks_pre_use(struct dmar_unit *dmar);
 
+int dmar_init_irt(struct dmar_unit *unit);
+void dmar_fini_irt(struct dmar_unit *unit);
+
 #define	DMAR_GM_CANWAIT	0x0001
 #define	DMAR_GM_CANSPLIT 0x0002
 
@@ -374,13 +396,16 @@ dmar_write8(const struct dmar_unit *unit, int reg, uint64_t val)
  * containing the P or R and W bits, is set only after the high word
  * is written.  For clear, the P bit is cleared first, then the high
  * word is cleared.
+ *
+ * dmar_pte_update updates the pte.  For amd64, the update is atomic.
+ * For i386, it first disables the entry by clearing the word
+ * containing the P bit, and then defer to dmar_pte_store.  The locked
+ * cmpxchg8b is probably available on any machine having DMAR support,
+ * but interrupt translation table may be mapped uncached.
  */
 static inline void
-dmar_pte_store(volatile uint64_t *dst, uint64_t val)
+dmar_pte_store1(volatile uint64_t *dst, uint64_t val)
 {
-
-	KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx",
-	    dst, (uintmax_t)*dst, (uintmax_t)val));
 #ifdef __i386__
 	volatile uint32_t *p;
 	uint32_t hi, lo;
@@ -396,6 +421,28 @@ dmar_pte_store(volatile uint64_t *dst, uint64_t val)
 }
 
 static inline void
+dmar_pte_store(volatile uint64_t *dst, uint64_t val)
+{
+
+	KASSERT(*dst == 0, ("used pte %p oldval %jx newval %jx",
+	    dst, (uintmax_t)*dst, (uintmax_t)val));
+	dmar_pte_store1(dst, val);
+}
+
+static inline void
+dmar_pte_update(volatile uint64_t *dst, uint64_t val)
+{
+
+#ifdef __i386__
+	volatile uint32_t *p;
+
+	p = (volatile uint32_t *)dst;
+	*p = 0;
+#endif
+	dmar_pte_store1(dst, val);
+}
+
+static inline void
 dmar_pte_clear(volatile uint64_t *dst)
 {
 #ifdef __i386__
diff --git a/sys/x86/iommu/intel_drv.c b/sys/x86/iommu/intel_drv.c
index cbec246..c239579 100644
--- a/sys/x86/iommu/intel_drv.c
+++ b/sys/x86/iommu/intel_drv.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2013 The FreeBSD Foundation
+ * Copyright (c) 2013-2015 The FreeBSD Foundation
  * All rights reserved.
  *
  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
@@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/smp.h>
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
+#include <sys/vmem.h>
 #include <machine/bus.h>
 #include <contrib/dev/acpica/include/acpi.h>
 #include <contrib/dev/acpica/include/accommon.h>
@@ -65,6 +66,7 @@ __FBSDID("$FreeBSD$");
 #include <x86/iommu/intel_reg.h>
 #include <x86/iommu/busdma_dmar.h>
 #include <x86/iommu/intel_dmar.h>
+#include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 
 #ifdef DEV_APIC
@@ -243,6 +245,7 @@ dmar_release_resources(device_t dev, struct dmar_unit *unit)
 	int i;
 
 	dmar_fini_busdma(unit);
+	dmar_fini_irt(unit);
 	dmar_fini_qi(unit);
 	dmar_fini_fault_log(unit);
 	for (i = 0; i < DMAR_INTR_TOTAL; i++)
@@ -509,6 +512,11 @@ dmar_attach(device_t dev)
 		dmar_release_resources(dev, unit);
 		return (error);
 	}
+	error = dmar_init_irt(unit);
+	if (error != 0) {
+		dmar_release_resources(dev, unit);
+		return (error);
+	}
 	error = dmar_init_busdma(unit);
 	if (error != 0) {
 		dmar_release_resources(dev, unit);
@@ -763,6 +771,76 @@ found:
 	return (device_get_softc(dmar_dev));
 }
 
+static struct dmar_unit *
+dmar_find_nonpci(u_int id, u_int entry_type, uint16_t *rid)
+{
+	device_t dmar_dev;
+	struct dmar_unit *unit;
+	ACPI_DMAR_HARDWARE_UNIT *dmarh;
+	ACPI_DMAR_DEVICE_SCOPE *devscope;
+	ACPI_DMAR_PCI_PATH *path;
+	char *ptr, *ptrend;
+	int i;
+
+	for (i = 0; i < dmar_devcnt; i++) {
+		dmar_dev = dmar_devs[i];
+		if (dmar_dev == NULL)
+			continue;
+		unit = (struct dmar_unit *)device_get_softc(dmar_dev);
+		dmarh = dmar_find_by_index(i);
+		if (dmarh == NULL)
+			continue;
+		ptr = (char *)dmarh + sizeof(*dmarh);
+		ptrend = (char *)dmarh + dmarh->Header.Length;
+		for (;;) {
+			if (ptr >= ptrend)
+				break;
+			devscope = (ACPI_DMAR_DEVICE_SCOPE *)ptr;
+			ptr += devscope->Length;
+			if (devscope->EntryType != entry_type)
+				continue;
+			if (devscope->EnumerationId != id)
+				continue;
+			if (devscope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE)
+			    == 2) {
+				if (rid != NULL) {
+					path = (ACPI_DMAR_PCI_PATH *)
+					    (devscope + 1);
+					*rid = PCI_RID(devscope->Bus,
+					    path->Device, path->Function);
+				}
+				return (unit);
+			} else {
+				/* XXXKIB */
+				printf(
+		       "dmar_find_nonpci: id %d type %d path length != 2\n",
+				    id, entry_type);
+			}
+		}
+	}
+	return (NULL);
+}
+
+
+struct dmar_unit *
+dmar_find_hpet(device_t dev, uint16_t *rid)
+{
+	ACPI_HANDLE handle;
+	uint32_t hpet_id;
+
+	handle = acpi_get_handle(dev);
+	if (ACPI_FAILURE(acpi_GetInteger(handle, "_UID", &hpet_id)))
+		return (NULL);
+	return (dmar_find_nonpci(hpet_id, ACPI_DMAR_SCOPE_TYPE_HPET, rid));
+}
+
+struct dmar_unit *
+dmar_find_ioapic(u_int apic_id, uint16_t *rid)
+{
+
+	return (dmar_find_nonpci(apic_id, ACPI_DMAR_SCOPE_TYPE_IOAPIC, rid));
+}
+
 struct rmrr_iter_args {
 	struct dmar_ctx *ctx;
 	device_t dev;
diff --git a/sys/x86/iommu/intel_fault.c b/sys/x86/iommu/intel_fault.c
index 033bbd4..5ccf88e 100644
--- a/sys/x86/iommu/intel_fault.c
+++ b/sys/x86/iommu/intel_fault.c
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
+#include <sys/vmem.h>
 #include <machine/bus.h>
 #include <contrib/dev/acpica/include/acpi.h>
 #include <contrib/dev/acpica/include/accommon.h>
diff --git a/sys/x86/iommu/intel_gas.c b/sys/x86/iommu/intel_gas.c
index 2049744..d7b5f8e 100644
--- a/sys/x86/iommu/intel_gas.c
+++ b/sys/x86/iommu/intel_gas.c
@@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
 #include <sys/uio.h>
+#include <sys/vmem.h>
 #include <dev/pci/pcivar.h>
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
diff --git a/sys/x86/iommu/intel_idpgtbl.c b/sys/x86/iommu/intel_idpgtbl.c
index 8257974..e2cce1a 100644
--- a/sys/x86/iommu/intel_idpgtbl.c
+++ b/sys/x86/iommu/intel_idpgtbl.c
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
 #include <sys/uio.h>
+#include <sys/vmem.h>
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
 #include <vm/vm_kern.h>
diff --git a/sys/x86/iommu/intel_intrmap.c b/sys/x86/iommu/intel_intrmap.c
new file mode 100644
index 0000000..745e617
--- /dev/null
+++ b/sys/x86/iommu/intel_intrmap.c
@@ -0,0 +1,380 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/malloc.h>
+#include <sys/memdesc.h>
+#include <sys/rman.h>
+#include <sys/rwlock.h>
+#include <sys/taskqueue.h>
+#include <sys/tree.h>
+#include <sys/vmem.h>
+#include <machine/bus.h>
+#include <machine/intr_machdep.h>
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_object.h>
+#include <vm/vm_page.h>
+#include <x86/include/apicreg.h>
+#include <x86/include/apicvar.h>
+#include <x86/include/busdma_impl.h>
+#include <x86/iommu/intel_reg.h>
+#include <x86/iommu/busdma_dmar.h>
+#include <x86/iommu/intel_dmar.h>
+#include <dev/pci/pcivar.h>
+#include <x86/iommu/iommu_intrmap.h>
+
+static struct dmar_unit *dmar_ir_find(device_t src, uint16_t *rid,
+    int *is_dmar);
+static void dmar_ir_program_irte(struct dmar_unit *unit, u_int idx,
+    uint64_t low, uint16_t rid);
+static int dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie);
+
+int
+iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count)
+{
+	struct dmar_unit *unit;
+	vmem_addr_t vmem_res;
+	u_int idx, i;
+	int error;
+
+	unit = dmar_ir_find(src, NULL, NULL);
+	if (unit == NULL || !unit->ir_enabled) {
+		for (i = 0; i < count; i++)
+			cookies[i] = -1;
+		return (EOPNOTSUPP);
+	}
+
+	error = vmem_alloc(unit->irtids, count, M_FIRSTFIT | M_NOWAIT,
+	    &vmem_res);
+	if (error != 0) {
+		KASSERT(error != EOPNOTSUPP,
+		    ("impossible EOPNOTSUPP from vmem"));
+		return (error);
+	}
+	idx = vmem_res;
+	for (i = 0; i < count; i++)
+		cookies[i] = idx + i;
+	return (0);
+}
+
+int
+iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
+    uint64_t *addr, uint32_t *data)
+{
+	struct dmar_unit *unit;
+	uint64_t low;
+	uint16_t rid;
+	int is_dmar;
+
+	unit = dmar_ir_find(src, &rid, &is_dmar);
+	if (is_dmar) {
+		KASSERT(unit == NULL, ("DMAR cannot translate itself"));
+
+		/*
+		 * See VT-d specification, 5.1.6 Remapping Hardware -
+		 * Interrupt Programming.
+		 */
+		*data = vector;
+		*addr = MSI_INTEL_ADDR_BASE | ((cpu & 0xff) << 12);
+		if (x2apic_mode)
+			*addr |= ((uint64_t)cpu & 0xffffff00) << 32;
+		else
+			KASSERT(cpu <= 0xff, ("cpu id too big %d", cpu));
+		return (0);
+	}
+	if (unit == NULL || !unit->ir_enabled || cookie == -1)
+		return (EOPNOTSUPP);
+
+	low = (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :
+	    DMAR_IRTE1_DST_xAPIC(cpu)) | DMAR_IRTE1_V(vector) |
+	    DMAR_IRTE1_DLM_FM | DMAR_IRTE1_TM_EDGE | DMAR_IRTE1_RH_DIRECT |
+	    DMAR_IRTE1_DM_PHYSICAL | DMAR_IRTE1_P;
+	dmar_ir_program_irte(unit, cookie, low, rid);
+
+	if (addr != NULL) {
+		/*
+		 * See VT-d specification, 5.1.5.2 MSI and MSI-X
+		 * Register Programming.
+		 */
+		*addr = MSI_INTEL_ADDR_BASE | ((cookie & 0x7fff) << 5) |
+		    ((cookie & 0x8000) << 2) | 0x18;
+		*data = 0;
+	}
+	return (0);
+}
+
+int
+iommu_unmap_msi_intr(device_t src, u_int cookie)
+{
+	struct dmar_unit *unit;
+
+	if (cookie == -1)
+		return (0);
+	unit = dmar_ir_find(src, NULL, NULL);
+	return (dmar_ir_free_irte(unit, cookie));
+}
+
+int
+iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
+    bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo)
+{
+	struct dmar_unit *unit;
+	vmem_addr_t vmem_res;
+	uint64_t low, iorte;
+	u_int idx;
+	int error;
+	uint16_t rid;
+
+	unit = dmar_find_ioapic(ioapic_id, &rid);
+	if (unit == NULL || !unit->ir_enabled) {
+		*cookie = -1;
+		return (EOPNOTSUPP);
+	}
+
+	error = vmem_alloc(unit->irtids, 1, M_FIRSTFIT | M_NOWAIT, &vmem_res);
+	if (error != 0) {
+		KASSERT(error != EOPNOTSUPP,
+		    ("impossible EOPNOTSUPP from vmem"));
+		return (error);
+	}
+	idx = vmem_res;
+	low = 0;
+	switch (irq) {
+	case IRQ_EXTINT:
+		low |= DMAR_IRTE1_DLM_ExtINT;
+		break;
+	case IRQ_NMI:
+		low |= DMAR_IRTE1_DLM_NMI;
+		break;
+	case IRQ_SMI:
+		low |= DMAR_IRTE1_DLM_SMI;
+		break;
+	default:
+		KASSERT(vector != 0, ("No vector for IRQ %u", irq));
+		low |= DMAR_IRTE1_DLM_FM | DMAR_IRTE1_V(vector);
+		break;
+	}
+	low |= (DMAR_X2APIC(unit) ? DMAR_IRTE1_DST_x2APIC(cpu) :
+	    DMAR_IRTE1_DST_xAPIC(cpu)) |
+	    (edge ? DMAR_IRTE1_TM_EDGE : DMAR_IRTE1_TM_LEVEL) |
+	    DMAR_IRTE1_RH_DIRECT | DMAR_IRTE1_DM_PHYSICAL | DMAR_IRTE1_P;
+	dmar_ir_program_irte(unit, idx, low, rid);
+
+	if (hi != NULL) {
+		/*
+		 * See VT-d specification, 5.1.5.1 I/OxAPIC
+		 * Programming.
+		 */
+		iorte = (1ULL << 48) | ((uint64_t)(idx & 0x7fff) << 49) |
+		    ((idx & 0x8000) != 0 ? (1 << 11) : 0) |
+		    (edge ? IOART_TRGREDG : IOART_TRGRLVL) |
+		    (activehi ? IOART_INTAHI : IOART_INTALO) |
+		    IOART_DELFIXED | vector;
+		*hi = iorte >> 32;
+		*lo = iorte;
+	}
+	*cookie = idx;
+	return (0);
+}
+
+int
+iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie)
+{
+	struct dmar_unit *unit;
+	u_int idx;
+
+	idx = *cookie;
+	if (idx == -1)
+		return (0);
+	*cookie = -1;
+	unit = dmar_find_ioapic(ioapic_id, NULL);
+	KASSERT(unit != NULL && unit->ir_enabled,
+	    ("unmap: cookie %d unit %p", idx, unit));
+	return (dmar_ir_free_irte(unit, idx));
+}
+
+static struct dmar_unit *
+dmar_ir_find(device_t src, uint16_t *rid, int *is_dmar)
+{
+	devclass_t src_class;
+	struct dmar_unit *unit;
+
+	/*
+	 * We need to determine if the interrupt source generates FSB
+	 * interrupts.  If yes, it is either DMAR, in which case
+	 * interrupts are not remapped.  Or it is HPET, and interrupts
+	 * are remapped.  For HPET, source id is reported by HPET
+	 * record in DMAR ACPI table.
+	 */
+	if (is_dmar != NULL)
+		*is_dmar = FALSE;
+	src_class = device_get_devclass(src);
+	if (src_class == devclass_find("dmar")) {
+		unit = NULL;
+		if (is_dmar != NULL)
+			*is_dmar = TRUE;
+	} else if (src_class == devclass_find("hpet")) {
+		unit = dmar_find_hpet(src, rid);
+	} else {
+		unit = dmar_find(src);
+		if (unit != NULL && rid != NULL)
+			dmar_get_requester(src, rid);
+	}
+	return (unit);
+}
+
+static void
+dmar_ir_program_irte(struct dmar_unit *unit, u_int idx, uint64_t low,
+    uint16_t rid)
+{
+	dmar_irte_t *irte;
+	uint64_t high;
+
+	KASSERT(idx < unit->irte_cnt,
+	    ("bad cookie %d %d", idx, unit->irte_cnt));
+	irte = &(unit->irt[idx]);
+	high = DMAR_IRTE2_SVT_RID | DMAR_IRTE2_SQ_RID |
+	    DMAR_IRTE2_SID_RID(rid);
+	device_printf(unit->dev,
+	    "programming irte[%d] rid %#x high %#jx low %#jx\n",
+	    idx, rid, (uintmax_t)high, (uintmax_t)low);
+	DMAR_LOCK(unit);
+	if ((irte->irte1 & DMAR_IRTE1_P) != 0) {
+		/*
+		 * The rte is already valid.  Assume that the request
+		 * is to remap the interrupt for balancing.  Only low
+		 * word of rte needs to be changed.  Assert that the
+		 * high word contains expected value.
+		 */
+		KASSERT(irte->irte2 == high,
+		    ("irte2 mismatch, %jx %jx", (uintmax_t)irte->irte2,
+		    (uintmax_t)high));
+		dmar_pte_update(&irte->irte1, low);
+	} else {
+		dmar_pte_store(&irte->irte2, high);
+		dmar_pte_store(&irte->irte1, low);
+	}
+	dmar_qi_invalidate_iec(unit, idx, 1);
+	DMAR_UNLOCK(unit);
+
+}
+
+static int
+dmar_ir_free_irte(struct dmar_unit *unit, u_int cookie)
+{
+	dmar_irte_t *irte;
+
+	KASSERT(unit != NULL && unit->ir_enabled,
+	    ("unmap: cookie %d unit %p", cookie, unit));
+	KASSERT(cookie < unit->irte_cnt,
+	    ("bad cookie %u %u", cookie, unit->irte_cnt));
+	irte = &(unit->irt[cookie]);
+	dmar_pte_clear(&irte->irte1);
+	dmar_pte_clear(&irte->irte2);
+	DMAR_LOCK(unit);
+	dmar_qi_invalidate_iec(unit, cookie, 1);
+	DMAR_UNLOCK(unit);
+	vmem_free(unit->irtids, cookie, 1);
+	return (0);
+}
+
+static u_int
+clp2(u_int v)
+{
+
+	return (powerof2(v) ? v : 1 << fls(v));
+}
+
+int
+dmar_init_irt(struct dmar_unit *unit)
+{
+
+	if ((unit->hw_ecap & DMAR_ECAP_IR) == 0)
+		return (0);
+	unit->ir_enabled = 1;
+	TUNABLE_INT_FETCH("hw.dmar.ir", &unit->ir_enabled);
+	if (!unit->ir_enabled)
+		return (0);
+	if (!unit->qi_enabled) {
+		unit->ir_enabled = 0;
+		if (bootverbose)
+			device_printf(unit->dev,
+	     "QI disabled, disabling interrupt remapping\n");
+		return (0);
+	}
+	unit->irte_cnt = clp2(NUM_IO_INTS);
+	unit->irt = (dmar_irte_t *)(uintptr_t)kmem_alloc_contig(kernel_arena,
+	    unit->irte_cnt * sizeof(dmar_irte_t), M_ZERO | M_WAITOK, 0,
+	    dmar_high, PAGE_SIZE, 0, DMAR_IS_COHERENT(unit) ?
+	    VM_MEMATTR_DEFAULT : VM_MEMATTR_UNCACHEABLE);
+	if (unit->irt == NULL)
+		return (ENOMEM);
+	unit->irt_phys = pmap_kextract((vm_offset_t)unit->irt);
+	unit->irtids = vmem_create("dmarirt", 0, unit->irte_cnt, 1, 0,
+	    M_FIRSTFIT | M_NOWAIT);
+	DMAR_LOCK(unit);
+	dmar_load_irt_ptr(unit);
+	dmar_qi_invalidate_iec_glob(unit);
+	DMAR_UNLOCK(unit);
+
+	/*
+	 * Initialize mappings for already configured interrupt pins.
+	 * Required, because otherwise the interrupts fault without
+	 * irtes.
+	 */
+	intr_reprogram();
+
+	DMAR_LOCK(unit);
+	dmar_enable_ir(unit);
+	DMAR_UNLOCK(unit);
+	return (0);
+}
+
+void
+dmar_fini_irt(struct dmar_unit *unit)
+{
+
+	unit->ir_enabled = 0;
+	if (unit->irt != NULL) {
+		dmar_disable_ir(unit);
+		dmar_qi_invalidate_iec_glob(unit);
+		vmem_destroy(unit->irtids);
+		kmem_free(kernel_arena, (vm_offset_t)unit->irt,
+		    unit->irte_cnt * sizeof(dmar_irte_t));
+	}
+}
diff --git a/sys/x86/iommu/intel_qi.c b/sys/x86/iommu/intel_qi.c
index a682c93..ce7b041 100644
--- a/sys/x86/iommu/intel_qi.c
+++ b/sys/x86/iommu/intel_qi.c
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
+#include <sys/vmem.h>
 #include <machine/bus.h>
 #include <contrib/dev/acpica/include/acpi.h>
 #include <contrib/dev/acpica/include/accommon.h>
@@ -194,13 +195,14 @@ dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq)
 }
 
 static void
-dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq)
+dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq,
+    bool nowait)
 {
 
 	DMAR_ASSERT_LOCKED(unit);
 	unit->inv_seq_waiters++;
 	while (!dmar_qi_seq_processed(unit, gseq)) {
-		if (cold) {
+		if (cold || nowait) {
 			cpu_spinwait();
 		} else {
 			msleep(&unit->inv_seq_waiters, &unit->lock, 0,
@@ -246,7 +248,7 @@ dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
 	dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
 	dmar_qi_emit_wait_seq(unit, &gseq);
 	dmar_qi_advance_tail(unit);
-	dmar_qi_wait_for_seq(unit, &gseq);
+	dmar_qi_wait_for_seq(unit, &gseq, false);
 }
 
 void
@@ -260,7 +262,60 @@ dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
 	    DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
 	dmar_qi_emit_wait_seq(unit, &gseq);
 	dmar_qi_advance_tail(unit);
-	dmar_qi_wait_for_seq(unit, &gseq);
+	dmar_qi_wait_for_seq(unit, &gseq, false);
+}
+
+void
+dmar_qi_invalidate_iec_glob(struct dmar_unit *unit)
+{
+	struct dmar_qi_genseq gseq;
+
+	DMAR_ASSERT_LOCKED(unit);
+	dmar_qi_ensure(unit, 2);
+	dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0);
+	dmar_qi_emit_wait_seq(unit, &gseq);
+	dmar_qi_advance_tail(unit);
+	dmar_qi_wait_for_seq(unit, &gseq, false);
+}
+
+void
+dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt)
+{
+	struct dmar_qi_genseq gseq;
+	u_int c, l;
+
+	DMAR_ASSERT_LOCKED(unit);
+	KASSERT(start < unit->irte_cnt && start < start + cnt &&
+	    start + cnt <= unit->irte_cnt,
+	    ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt));
+	for (; cnt > 0; cnt -= c, start += c) {
+		l = ffs(start | cnt) - 1;
+		c = 1 << l;
+		dmar_qi_ensure(unit, 1);
+		dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV |
+		    DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) |
+		    DMAR_IQ_DESCR_IEC_IM(l), 0);
+	}
+	dmar_qi_ensure(unit, 1);
+	dmar_qi_emit_wait_seq(unit, &gseq);
+	dmar_qi_advance_tail(unit);
+
+	/*
+	 * The caller of the function, in particular,
+	 * dmar_ir_program_irte(), may be called from the context
+	 * where the sleeping is forbidden (in fact, the
+	 * intr_table_lock mutex may be held, locked from
+	 * intr_shuffle_irqs()).  Wait for the invalidation completion
+	 * using the busy wait.
+	 *
+	 * The impact on the interrupt input setup code is small, the
+	 * expected overhead is comparable with the chipset register
+	 * read.  It is more harmful for the parallel DMA operations,
+	 * since we own the dmar unit lock until whole invalidation
+	 * queue is processed, which includes requests possibly issued
+	 * before our request.
+	 */
+	dmar_qi_wait_for_seq(unit, &gseq, true);
 }
 
 int
@@ -377,7 +432,7 @@ dmar_fini_qi(struct dmar_unit *unit)
 	dmar_qi_ensure(unit, 1);
 	dmar_qi_emit_wait_seq(unit, &gseq);
 	dmar_qi_advance_tail(unit);
-	dmar_qi_wait_for_seq(unit, &gseq);
+	dmar_qi_wait_for_seq(unit, &gseq, false);
 	/* only after the quisce, disable queue */
 	dmar_disable_qi(unit);
 	KASSERT(unit->inv_seq_waiters == 0,
diff --git a/sys/x86/iommu/intel_quirks.c b/sys/x86/iommu/intel_quirks.c
index 7c35ae6..83d3922 100644
--- a/sys/x86/iommu/intel_quirks.c
+++ b/sys/x86/iommu/intel_quirks.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2013 The FreeBSD Foundation
+ * Copyright (c) 2013, 2015 The FreeBSD Foundation
  * All rights reserved.
  *
  * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/smp.h>
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
+#include <sys/vmem.h>
 #include <machine/bus.h>
 #include <contrib/dev/acpica/include/acpi.h>
 #include <contrib/dev/acpica/include/accommon.h>
@@ -59,7 +60,7 @@ __FBSDID("$FreeBSD$");
 #include <x86/iommu/intel_dmar.h>
 #include <dev/pci/pcivar.h>
 
-typedef void (*dmar_quirk_fun)(struct dmar_unit *);
+typedef void (*dmar_quirk_cpu_fun)(struct dmar_unit *);
 
 struct intel_dmar_quirk_cpu {
 	u_int ext_family;
@@ -67,17 +68,21 @@ struct intel_dmar_quirk_cpu {
 	u_int family_code;
 	u_int model;
 	u_int stepping;
-	dmar_quirk_fun quirk;
+	dmar_quirk_cpu_fun quirk;
 	const char *descr;
 };
 
+typedef void (*dmar_quirk_nb_fun)(struct dmar_unit *, device_t nb);
+
 struct intel_dmar_quirk_nb {
 	u_int dev_id;
 	u_int rev_no;
-	dmar_quirk_fun quirk;
+	dmar_quirk_nb_fun quirk;
 	const char *descr;
 };
 
+#define	QUIRK_NB_ALL_REV	0xffffffff
+
 static void
 dmar_match_quirks(struct dmar_unit *dmar,
     const struct intel_dmar_quirk_nb *nb_quirks, int nb_quirks_len,
@@ -99,13 +104,14 @@ dmar_match_quirks(struct dmar_unit *dmar,
 			for (i = 0; i < nb_quirks_len; i++) {
 				nb_quirk = &nb_quirks[i];
 				if (nb_quirk->dev_id == dev_id &&
-				    nb_quirk->rev_no == rev_no) {
+				    (nb_quirk->rev_no == rev_no ||
+				    nb_quirk->rev_no == QUIRK_NB_ALL_REV)) {
 					if (bootverbose) {
 						device_printf(dmar->dev,
 						    "NB IOMMU quirk %s\n",
 						    nb_quirk->descr);
 					}
-					nb_quirk->quirk(dmar);
+					nb_quirk->quirk(dmar, nb);
 				}
 			}
 		} else {
@@ -139,12 +145,29 @@ dmar_match_quirks(struct dmar_unit *dmar,
 }
 
 static void
-nb_5400_no_low_high_prot_mem(struct dmar_unit *unit)
+nb_5400_no_low_high_prot_mem(struct dmar_unit *unit, device_t nb __unused)
 {
 
 	unit->hw_cap &= ~(DMAR_CAP_PHMR | DMAR_CAP_PLMR);
 }
 
+static void
+nb_no_ir(struct dmar_unit *unit, device_t nb __unused)
+{
+
+	unit->hw_ecap &= ~(DMAR_ECAP_IR | DMAR_ECAP_EIM);
+}
+
+static void
+nb_5500_no_ir_rev13(struct dmar_unit *unit, device_t nb)
+{
+	u_int rev_no;
+
+	rev_no = pci_get_revid(nb);
+	if (rev_no <= 0x13)
+		nb_no_ir(unit, nb);
+}
+
 static const struct intel_dmar_quirk_nb pre_use_nb[] = {
 	{
 	    .dev_id = 0x4001, .rev_no = 0x20,
@@ -156,6 +179,26 @@ static const struct intel_dmar_quirk_nb pre_use_nb[] = {
 	    .quirk = nb_5400_no_low_high_prot_mem,
 	    .descr = "5400 E23" /* no low/high protected memory */
 	},
+	{
+	    .dev_id = 0x3403, .rev_no = QUIRK_NB_ALL_REV,
+	    .quirk = nb_5500_no_ir_rev13,
+	    .descr = "5500 E47, E53" /* interrupt remapping does not work */
+	},
+	{
+	    .dev_id = 0x3405, .rev_no = QUIRK_NB_ALL_REV,
+	    .quirk = nb_5500_no_ir_rev13,
+	    .descr = "5500 E47, E53" /* interrupt remapping does not work */
+	},
+	{
+	    .dev_id = 0x3405, .rev_no = 0x22,
+	    .quirk = nb_no_ir,
+	    .descr = "5500 E47, E53" /* interrupt remapping does not work */
+	},
+	{
+	    .dev_id = 0x3406, .rev_no = QUIRK_NB_ALL_REV,
+	    .quirk = nb_5500_no_ir_rev13,
+	    .descr = "5500 E47, E53" /* interrupt remapping does not work */
+	},
 };
 
 static void
diff --git a/sys/x86/iommu/intel_utils.c b/sys/x86/iommu/intel_utils.c
index 8dc2f35..f696f9d 100644
--- a/sys/x86/iommu/intel_utils.c
+++ b/sys/x86/iommu/intel_utils.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
+#include <sys/vmem.h>
 #include <dev/pci/pcivar.h>
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
@@ -57,6 +58,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_pageout.h>
 #include <machine/bus.h>
 #include <machine/cpu.h>
+#include <machine/intr_machdep.h>
+#include <x86/include/apicvar.h>
 #include <x86/include/busdma_impl.h>
 #include <x86/iommu/intel_reg.h>
 #include <x86/iommu/busdma_dmar.h>
@@ -521,6 +524,55 @@ dmar_disable_translation(struct dmar_unit *unit)
 	return (0);
 }
 
+int
+dmar_load_irt_ptr(struct dmar_unit *unit)
+{
+	uint64_t irta, s;
+
+	DMAR_ASSERT_LOCKED(unit);
+	irta = unit->irt_phys;
+	if (DMAR_X2APIC(unit))
+		irta |= DMAR_IRTA_EIME;
+	s = fls(unit->irte_cnt) - 2;
+	KASSERT(unit->irte_cnt >= 2 && s <= DMAR_IRTA_S_MASK &&
+	    powerof2(unit->irte_cnt),
+	    ("IRTA_REG_S overflow %x", unit->irte_cnt));
+	irta |= s;
+	dmar_write8(unit, DMAR_IRTA_REG, irta);
+	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd | DMAR_GCMD_SIRTP);
+	/* XXXKIB should have a timeout */
+	while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRTPS) == 0)
+		cpu_spinwait();
+	return (0);
+}
+
+int
+dmar_enable_ir(struct dmar_unit *unit)
+{
+
+	DMAR_ASSERT_LOCKED(unit);
+	unit->hw_gcmd |= DMAR_GCMD_IRE;
+	unit->hw_gcmd &= ~DMAR_GCMD_CFI;
+	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
+	/* XXXKIB should have a timeout */
+	while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) == 0)
+		cpu_spinwait();
+	return (0);
+}
+
+int
+dmar_disable_ir(struct dmar_unit *unit)
+{
+
+	DMAR_ASSERT_LOCKED(unit);
+	unit->hw_gcmd &= ~DMAR_GCMD_IRE;
+	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
+	/* XXXKIB should have a timeout */
+	while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_IRES) != 0)
+		cpu_spinwait();
+	return (0);
+}
+
 #define BARRIER_F				\
 	u_int f_done, f_inproc, f_wakeup;	\
 						\
diff --git a/sys/x86/iommu/iommu_intrmap.h b/sys/x86/iommu/iommu_intrmap.h
new file mode 100644
index 0000000..92cdf5a
--- /dev/null
+++ b/sys/x86/iommu/iommu_intrmap.h
@@ -0,0 +1,43 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef __X86_IOMMU_IOMMU_INTRMAP_H
+#define	__X86_IOMMU_IOMMU_INTRMAP_H
+
+int iommu_alloc_msi_intr(device_t src, u_int *cookies, u_int count);
+int iommu_map_msi_intr(device_t src, u_int cpu, u_int vector, u_int cookie,
+    uint64_t *addr, uint32_t *data);
+int iommu_unmap_msi_intr(device_t src, u_int cookie);
+int iommu_map_ioapic_intr(u_int ioapic_id, u_int cpu, u_int vector, bool edge,
+    bool activehi, int irq, u_int *cookie, uint32_t *hi, uint32_t *lo);
+int iommu_unmap_ioapic_intr(u_int ioapic_id, u_int *cookie);
+
+#endif
diff --git a/sys/x86/x86/intr_machdep.c b/sys/x86/x86/intr_machdep.c
index 7241b12..b27df4a 100644
--- a/sys/x86/x86/intr_machdep.c
+++ b/sys/x86/x86/intr_machdep.c
@@ -423,6 +423,23 @@ intr_describe(u_int vector, void *ih, const char *descr)
 	return (0);
 }
 
+void
+intr_reprogram(void)
+{
+	struct intsrc *is;
+	int v;
+
+	mtx_lock(&intr_table_lock);
+	for (v = 0; v < NUM_IO_INTS; v++) {
+		is = interrupt_sources[v];
+		if (is == NULL)
+			continue;
+		if (is->is_pic->pic_reprogram_pin != NULL)
+			is->is_pic->pic_reprogram_pin(is);
+	}
+	mtx_unlock(&intr_table_lock);
+}
+
 #ifdef DDB
 /*
  * Dump data about interrupt handlers
diff --git a/sys/x86/x86/io_apic.c b/sys/x86/x86/io_apic.c
index 125663c..44609e6 100644
--- a/sys/x86/x86/io_apic.c
+++ b/sys/x86/x86/io_apic.c
@@ -27,6 +27,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_acpi.h"
 #include "opt_isa.h"
 
 #include <sys/param.h>
@@ -51,17 +52,13 @@ __FBSDID("$FreeBSD$");
 #include <x86/apicvar.h>
 #include <machine/resource.h>
 #include <machine/segments.h>
+#include <x86/iommu/iommu_intrmap.h>
 
 #define IOAPIC_ISA_INTS		16
 #define	IOAPIC_MEM_REGION	32
 #define	IOAPIC_REDTBL_LO(i)	(IOAPIC_REDTBL + (i) * 2)
 #define	IOAPIC_REDTBL_HI(i)	(IOAPIC_REDTBL_LO(i) + 1)
 
-#define	IRQ_EXTINT		(NUM_IO_INTS + 1)
-#define	IRQ_NMI			(NUM_IO_INTS + 2)
-#define	IRQ_SMI			(NUM_IO_INTS + 3)
-#define	IRQ_DISABLED		(NUM_IO_INTS + 4)
-
 static MALLOC_DEFINE(M_IOAPIC, "io_apic", "I/O APIC structures");
 
 /*
@@ -83,12 +80,13 @@ struct ioapic_intsrc {
 	u_int io_irq;
 	u_int io_intpin:8;
 	u_int io_vector:8;
-	u_int io_cpu:8;
+	u_int io_cpu;
 	u_int io_activehi:1;
 	u_int io_edgetrigger:1;
 	u_int io_masked:1;
 	int io_bus:4;
 	uint32_t io_lowreg;
+	u_int io_remap_cookie;
 };
 
 struct ioapic {
@@ -120,13 +118,23 @@ static int	ioapic_config_intr(struct intsrc *isrc, enum intr_trigger trig,
 static void	ioapic_resume(struct pic *pic, bool suspend_cancelled);
 static int	ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id);
 static void	ioapic_program_intpin(struct ioapic_intsrc *intpin);
+static void	ioapic_reprogram_intpin(struct intsrc *isrc);
 
 static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list);
-struct pic ioapic_template = { ioapic_enable_source, ioapic_disable_source,
-			       ioapic_eoi_source, ioapic_enable_intr,
-			       ioapic_disable_intr, ioapic_vector,
-			       ioapic_source_pending, NULL, ioapic_resume,
-			       ioapic_config_intr, ioapic_assign_cpu };
+struct pic ioapic_template = {
+	.pic_enable_source = ioapic_enable_source,
+	.pic_disable_source = ioapic_disable_source,
+	.pic_eoi_source = ioapic_eoi_source,
+	.pic_enable_intr = ioapic_enable_intr,
+	.pic_disable_intr = ioapic_disable_intr,
+	.pic_vector = ioapic_vector,
+	.pic_source_pending = ioapic_source_pending,
+	.pic_suspend = NULL,
+	.pic_resume = ioapic_resume,
+	.pic_config_intr = ioapic_config_intr,
+	.pic_assign_cpu = ioapic_assign_cpu,
+	.pic_reprogram_pin = ioapic_reprogram_intpin,
+};
 
 static int next_ioapic_base;
 static u_int next_id;
@@ -295,6 +303,9 @@ ioapic_program_intpin(struct ioapic_intsrc *intpin)
 {
 	struct ioapic *io = (struct ioapic *)intpin->io_intsrc.is_pic;
 	uint32_t low, high, value;
+#ifdef ACPI_DMAR
+	int error;
+#endif
 
 	/*
 	 * If a pin is completely invalid or if it is valid but hasn't
@@ -309,9 +320,34 @@ ioapic_program_intpin(struct ioapic_intsrc *intpin)
 			ioapic_write(io->io_addr,
 			    IOAPIC_REDTBL_LO(intpin->io_intpin),
 			    low | IOART_INTMSET);
+#ifdef ACPI_DMAR
+		mtx_unlock_spin(&icu_lock);
+		iommu_unmap_ioapic_intr(io->io_apic_id,
+		    &intpin->io_remap_cookie);
+		mtx_lock_spin(&icu_lock);
+#endif
 		return;
 	}
 
+#ifdef ACPI_DMAR
+	mtx_unlock_spin(&icu_lock);
+	error = iommu_map_ioapic_intr(io->io_apic_id,
+	    intpin->io_cpu, intpin->io_vector, intpin->io_edgetrigger,
+	    intpin->io_activehi, intpin->io_irq, &intpin->io_remap_cookie,
+	    &high, &low);
+	mtx_lock_spin(&icu_lock);
+	if (error == 0) {
+		ioapic_write(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin),
+		    high);
+		intpin->io_lowreg = low;
+		ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin),
+		    low);
+		return;
+	} else if (error != EOPNOTSUPP) {
+		return;
+	}
+#endif
+
 	/* Set the destination. */
 	low = IOART_DESTPHY;
 	high = intpin->io_cpu << APIC_ID_SHIFT;
@@ -358,6 +394,15 @@ ioapic_program_intpin(struct ioapic_intsrc *intpin)
 	ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), low);
 }
 
+static void
+ioapic_reprogram_intpin(struct intsrc *isrc)
+{
+
+	mtx_lock_spin(&icu_lock);
+	ioapic_program_intpin((struct ioapic_intsrc *)isrc);
+	mtx_unlock_spin(&icu_lock);
+}
+
 static int
 ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id)
 {
@@ -643,6 +688,15 @@ ioapic_create(vm_paddr_t addr, int32_t apic_id, int intbase)
 		intpin->io_cpu = PCPU_GET(apic_id);
 		value = ioapic_read(apic, IOAPIC_REDTBL_LO(i));
 		ioapic_write(apic, IOAPIC_REDTBL_LO(i), value | IOART_INTMSET);
+#ifdef ACPI_DMAR
+		/* dummy, but sets cookie */
+		mtx_unlock_spin(&icu_lock);
+		iommu_map_ioapic_intr(io->io_apic_id,
+		    intpin->io_cpu, intpin->io_vector, intpin->io_edgetrigger,
+		    intpin->io_activehi, intpin->io_irq,
+		    &intpin->io_remap_cookie, NULL, NULL);
+		mtx_lock_spin(&icu_lock);
+#endif
 	}
 	mtx_unlock_spin(&icu_lock);
 
diff --git a/sys/x86/x86/msi.c b/sys/x86/x86/msi.c
index 2d3a202..20fda2c 100644
--- a/sys/x86/x86/msi.c
+++ b/sys/x86/x86/msi.c
@@ -37,6 +37,8 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_acpi.h"
+
 #include <sys/param.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
@@ -51,6 +53,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/frame.h>
 #include <machine/intr_machdep.h>
 #include <x86/apicvar.h>
+#include <x86/iommu/iommu_intrmap.h>
 #include <machine/specialreg.h>
 #include <dev/pci/pcivar.h>
 
@@ -111,10 +114,11 @@ struct msi_intsrc {
 	u_int msi_irq;			/* IRQ cookie. */
 	u_int msi_msix;			/* MSI-X message. */
 	u_int msi_vector:8;		/* IDT vector. */
-	u_int msi_cpu:8;		/* Local APIC ID. (g) */
+	u_int msi_cpu;			/* Local APIC ID. (g) */
 	u_int msi_count:8;		/* Messages in this group. (g) */
 	u_int msi_maxcount:8;		/* Alignment for this group. (g) */
 	int *msi_irqs;			/* Group's IRQ list. (g) */
+	u_int msi_remap_cookie;
 };
 
 static void	msi_create_source(void);
@@ -129,10 +133,20 @@ static int	msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
 		    enum intr_polarity pol);
 static int	msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
 
-struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
-		       msi_enable_intr, msi_disable_intr, msi_vector,
-		       msi_source_pending, NULL, NULL, msi_config_intr,
-		       msi_assign_cpu };
+struct pic msi_pic = {
+	.pic_enable_source = msi_enable_source,
+	.pic_disable_source = msi_disable_source,
+	.pic_eoi_source = msi_eoi_source,
+	.pic_enable_intr = msi_enable_intr,
+	.pic_disable_intr = msi_disable_intr,
+	.pic_vector = msi_vector,
+	.pic_source_pending = msi_source_pending,
+	.pic_suspend = NULL,
+	.pic_resume = NULL,
+	.pic_config_intr = msi_config_intr,
+	.pic_assign_cpu = msi_assign_cpu,
+	.pic_reprogram_pin = NULL,
+};
 
 static int msi_enabled;
 static int msi_last_irq;
@@ -320,6 +334,10 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs)
 	struct msi_intsrc *msi, *fsrc;
 	u_int cpu;
 	int cnt, i, *mirqs, vector;
+#ifdef ACPI_DMAR
+	u_int cookies[count];
+	int error;
+#endif
 
 	if (!msi_enabled)
 		return (ENXIO);
@@ -379,6 +397,24 @@ again:
 		return (ENOSPC);
 	}
 
+#ifdef ACPI_DMAR
+	mtx_unlock(&msi_lock);
+	error = iommu_alloc_msi_intr(dev, cookies, count);
+	mtx_lock(&msi_lock);
+	if (error == EOPNOTSUPP)
+		error = 0;
+	if (error != 0) {
+		for (i = 0; i < count; i++)
+			apic_free_vector(cpu, vector + i, irqs[i]);
+		free(mirqs, M_MSI);
+		return (error);
+	}
+	for (i = 0; i < count; i++) {
+		msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
+		msi->msi_remap_cookie = cookies[i];
+	}
+#endif
+
 	/* Assign IDT vectors and make these messages owned by 'dev'. */
 	fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
 	for (i = 0; i < count; i++) {
@@ -400,7 +436,6 @@ again:
 		bcopy(irqs, mirqs, count * sizeof(*mirqs));
 	fsrc->msi_irqs = mirqs;
 	mtx_unlock(&msi_lock);
-
 	return (0);
 }
 
@@ -444,6 +479,9 @@ msi_release(int *irqs, int count)
 		msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
 		KASSERT(msi->msi_first == first, ("message not in group"));
 		KASSERT(msi->msi_dev == first->msi_dev, ("owner mismatch"));
+#ifdef ACPI_DMAR
+		iommu_unmap_msi_intr(first->msi_dev, msi->msi_remap_cookie);
+#endif
 		msi->msi_first = NULL;
 		msi->msi_dev = NULL;
 		apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);
@@ -451,6 +489,11 @@ msi_release(int *irqs, int count)
 	}
 
 	/* Clear out the first message. */
+#ifdef ACPI_DMAR
+	mtx_unlock(&msi_lock);
+	iommu_unmap_msi_intr(first->msi_dev, first->msi_remap_cookie);
+	mtx_lock(&msi_lock);
+#endif
 	first->msi_first = NULL;
 	first->msi_dev = NULL;
 	apic_free_vector(first->msi_cpu, first->msi_vector, first->msi_irq);
@@ -468,6 +511,11 @@ int
 msi_map(int irq, uint64_t *addr, uint32_t *data)
 {
 	struct msi_intsrc *msi;
+	int error;
+#ifdef ACPI_DMAR
+	struct msi_intsrc *msi1;
+	int i, k;
+#endif
 
 	mtx_lock(&msi_lock);
 	msi = (struct msi_intsrc *)intr_lookup_source(irq);
@@ -495,10 +543,36 @@ msi_map(int irq, uint64_t *addr, uint32_t *data)
 		msi = msi->msi_first;
 	}
 
-	*addr = INTEL_ADDR(msi);
-	*data = INTEL_DATA(msi);
+#ifdef ACPI_DMAR
+	if (!msi->msi_msix) {
+		for (k = msi->msi_count - 1, i = FIRST_MSI_INT; k > 0 &&
+		    i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
+			if (i == msi->msi_irq)
+				continue;
+			msi1 = (struct msi_intsrc *)intr_lookup_source(i);
+			if (!msi1->msi_msix && msi1->msi_first == msi) {
+				mtx_unlock(&msi_lock);
+				iommu_map_msi_intr(msi1->msi_dev,
+				    msi1->msi_cpu, msi1->msi_vector,
+				    msi1->msi_remap_cookie, NULL, NULL);
+				k--;
+				mtx_lock(&msi_lock);
+			}
+		}
+	}
 	mtx_unlock(&msi_lock);
-	return (0);
+	error = iommu_map_msi_intr(msi->msi_dev, msi->msi_cpu,
+	    msi->msi_vector, msi->msi_remap_cookie, addr, data);
+#else
+	mtx_unlock(&msi_lock);
+	error = EOPNOTSUPP;
+#endif
+	if (error == EOPNOTSUPP) {
+		*addr = INTEL_ADDR(msi);
+		*data = INTEL_DATA(msi);
+		error = 0;
+	}
+	return (error);
 }
 
 int
@@ -507,6 +581,10 @@ msix_alloc(device_t dev, int *irq)
 	struct msi_intsrc *msi;
 	u_int cpu;
 	int i, vector;
+#ifdef ACPI_DMAR
+	u_int cookie;
+	int error;
+#endif
 
 	if (!msi_enabled)
 		return (ENXIO);
@@ -548,13 +626,28 @@ again:
 		mtx_unlock(&msi_lock);
 		return (ENOSPC);
 	}
+
+	msi->msi_dev = dev;
+#ifdef ACPI_DMAR
+	mtx_unlock(&msi_lock);
+	error = iommu_alloc_msi_intr(dev, &cookie, 1);
+	mtx_lock(&msi_lock);
+	if (error == EOPNOTSUPP)
+		error = 0;
+	if (error != 0) {
+		msi->msi_dev = NULL;
+		apic_free_vector(cpu, vector, i);
+		return (error);
+	}
+	msi->msi_remap_cookie = cookie;
+#endif
+
 	if (bootverbose)
 		printf("msi: routing MSI-X IRQ %d to local APIC %u vector %u\n",
 		    msi->msi_irq, cpu, vector);
 
 	/* Setup source. */
 	msi->msi_cpu = cpu;
-	msi->msi_dev = dev;
 	msi->msi_first = msi;
 	msi->msi_vector = vector;
 	msi->msi_msix = 1;
@@ -590,6 +683,11 @@ msix_release(int irq)
 	KASSERT(msi->msi_dev != NULL, ("unowned message"));
 
 	/* Clear out the message. */
+#ifdef ACPI_DMAR
+	mtx_unlock(&msi_lock);
+	iommu_unmap_msi_intr(msi->msi_dev, msi->msi_remap_cookie);
+	mtx_lock(&msi_lock);
+#endif
 	msi->msi_first = NULL;
 	msi->msi_dev = NULL;
 	apic_free_vector(msi->msi_cpu, msi->msi_vector, msi->msi_irq);