Add support for queued invalidation.

Right now, the semaphore write is scheduled after each batch, which is
not optimal and must be tuned.

Discussed with:	alc
Tested by:	pho
MFC after:	1 month

9 files changed, 843 insertions, 162 deletions

diff --git a/sys/x86/iommu/intel_ctx.c b/sys/x86/iommu/intel_ctx.c
index b8da09d..fc5fe09 100644
--- a/sys/x86/iommu/intel_ctx.c
+++ b/sys/x86/iommu/intel_ctx.c
@@ -385,17 +385,29 @@ dmar_get_ctx(struct dmar_unit *dmar, device_t dev, bool id_mapped, bool rmrr_ini
 	 * negative TLB entries.
 	 */
 	if ((dmar->hw_cap & DMAR_CAP_CM) != 0 || enable) {
-		error = dmar_inv_ctx_glob(dmar);
-		if (error == 0 &&
-		    (dmar->hw_ecap & DMAR_ECAP_DI) != 0)
-			error = dmar_inv_iotlb_glob(dmar);
-		if (error != 0) {
-			dmar_free_ctx_locked(dmar, ctx);
-			TD_PINNED_ASSERT;
-			return (NULL);
+		if (dmar->qi_enabled) {
+			dmar_qi_invalidate_ctx_glob_locked(dmar);
+			if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0)
+				dmar_qi_invalidate_iotlb_glob_locked(dmar);
+		} else {
+			error = dmar_inv_ctx_glob(dmar);
+			if (error == 0 &&
+			    (dmar->hw_ecap & DMAR_ECAP_DI) != 0)
+				error = dmar_inv_iotlb_glob(dmar);
+			if (error != 0) {
+				dmar_free_ctx_locked(dmar, ctx);
+				TD_PINNED_ASSERT;
+				return (NULL);
+			}
 		}
 	}
-	if (enable && !rmrr_init) {
+
+	/*
+	 * The dmar lock was potentially dropped between check for the
+	 * empty context list and now.  Recheck the state of GCMD_TE
+	 * to avoid unneeded command.
+	 */
+	if (enable && !rmrr_init && (dmar->hw_gcmd & DMAR_GCMD_TE) == 0) {
 		error = dmar_enable_translation(dmar);
 		if (error != 0) {
 			dmar_free_ctx_locked(dmar, ctx);
@@ -469,8 +481,12 @@ dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx)
 	dmar_pte_clear(&ctxp->ctx1);
 	ctxp->ctx2 = 0;
 	dmar_inv_ctx_glob(dmar);
-	if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0)
-		dmar_inv_iotlb_glob(dmar);
+	if ((dmar->hw_ecap & DMAR_ECAP_DI) != 0) {
+		if (dmar->qi_enabled)
+			dmar_qi_invalidate_iotlb_glob_locked(dmar);
+		else
+			dmar_inv_iotlb_glob(dmar);
+	}
 	LIST_REMOVE(ctx, link);
 	DMAR_UNLOCK(dmar);
 
@@ -512,24 +528,86 @@ dmar_find_ctx_locked(struct dmar_unit *dmar, int bus, int slot, int func)
 }
 
 void
+dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free)
+{
+	struct dmar_ctx *ctx;
+
+	ctx = entry->ctx;
+	DMAR_CTX_LOCK(ctx);
+	if ((entry->flags & DMAR_MAP_ENTRY_RMRR) != 0)
+		dmar_gas_free_region(ctx, entry);
+	else
+		dmar_gas_free_space(ctx, entry);
+	DMAR_CTX_UNLOCK(ctx);
+	if (free)
+		dmar_gas_free_entry(ctx, entry);
+	else
+		entry->flags = 0;
+}
+
+void
+dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free)
+{
+	struct dmar_unit *unit;
+
+	unit = entry->ctx->dmar;
+	if (unit->qi_enabled) {
+		DMAR_LOCK(unit);
+		dmar_qi_invalidate_locked(entry->ctx, entry->start,
+		    entry->end - entry->start, &entry->gseq);
+		if (!free)
+			entry->flags |= DMAR_MAP_ENTRY_QI_NF;
+		TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
+		DMAR_UNLOCK(unit);
+	} else {
+		ctx_flush_iotlb_sync(entry->ctx, entry->start, entry->end -
+		    entry->start);
+		dmar_ctx_free_entry(entry, free);
+	}
+}
+
+void
 dmar_ctx_unload(struct dmar_ctx *ctx, struct dmar_map_entries_tailq *entries,
     bool cansleep)
 {
-	struct dmar_map_entry *entry;
+	struct dmar_unit *unit;
+	struct dmar_map_entry *entry, *entry1;
+	struct dmar_qi_genseq gseq;
 	int error;
 
-	while ((entry = TAILQ_FIRST(entries)) != NULL) {
+	unit = ctx->dmar;
+
+	TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
 		KASSERT((entry->flags & DMAR_MAP_ENTRY_MAP) != 0,
 		    ("not mapped entry %p %p", ctx, entry));
-		TAILQ_REMOVE(entries, entry, dmamap_link);
 		error = ctx_unmap_buf(ctx, entry->start, entry->end -
 		    entry->start, cansleep ? DMAR_PGF_WAITOK : 0);
 		KASSERT(error == 0, ("unmap %p error %d", ctx, error));
-		DMAR_CTX_LOCK(ctx);
-		dmar_gas_free_space(ctx, entry);
-		DMAR_CTX_UNLOCK(ctx);
-		dmar_gas_free_entry(ctx, entry);
+		if (!unit->qi_enabled) {
+			ctx_flush_iotlb_sync(ctx, entry->start,
+			    entry->end - entry->start);
+			TAILQ_REMOVE(entries, entry, dmamap_link);
+			dmar_ctx_free_entry(entry, true);
+		}
+	}
+	if (TAILQ_EMPTY(entries))
+		return;
+
+	KASSERT(unit->qi_enabled, ("loaded entry left"));
+	DMAR_LOCK(unit);
+	TAILQ_FOREACH(entry, entries, dmamap_link) {
+		entry->gseq.gen = 0;
+		entry->gseq.seq = 0;
+		dmar_qi_invalidate_locked(ctx, entry->start, entry->end -
+		    entry->start, TAILQ_NEXT(entry, dmamap_link) == NULL ?
+		    &gseq : NULL);
+	}
+	TAILQ_FOREACH_SAFE(entry, entries, dmamap_link, entry1) {
+		entry->gseq = gseq;
+		TAILQ_REMOVE(entries, entry, dmamap_link);
+		TAILQ_INSERT_TAIL(&unit->tlb_flush_entries, entry, dmamap_link);
 	}
+	DMAR_UNLOCK(unit);
 }	
 
 static void
diff --git a/sys/x86/iommu/intel_dmar.h b/sys/x86/iommu/intel_dmar.h
index 0c84856..994e5e1 100644
--- a/sys/x86/iommu/intel_dmar.h
+++ b/sys/x86/iommu/intel_dmar.h
@@ -37,6 +37,11 @@ typedef uint64_t dmar_haddr_t;
 /* Guest or bus address, before translation. */
 typedef uint64_t dmar_gaddr_t;
 
+struct dmar_qi_genseq {
+	u_int gen;
+	uint32_t seq;
+};
+
 struct dmar_map_entry {
 	dmar_gaddr_t start;
 	dmar_gaddr_t end;
@@ -48,6 +53,8 @@ struct dmar_map_entry {
 	RB_ENTRY(dmar_map_entry) rb_entry;	 /* Links for ctx entries */
 	TAILQ_ENTRY(dmar_map_entry) unroll_link; /* Link for unroll after
 						    dmamap_load failure */
+	struct dmar_ctx *ctx;
+	struct dmar_qi_genseq gseq;
 };
 
 RB_HEAD(dmar_gas_entries_tree, dmar_map_entry);
@@ -60,6 +67,7 @@ RB_PROTOTYPE(dmar_gas_entries_tree, dmar_map_entry, rb_entry,
 #define	DMAR_MAP_ENTRY_MAP	0x0004	/* Busdma created, linked by
 					   dmamap_link */
 #define	DMAR_MAP_ENTRY_UNMAPPED	0x0010	/* No backing pages */
+#define	DMAR_MAP_ENTRY_QI_NF	0x0020	/* qi task, do not free entry */
 #define	DMAR_MAP_ENTRY_READ	0x1000	/* Read permitted */
 #define	DMAR_MAP_ENTRY_WRITE	0x2000	/* Write permitted */
 #define	DMAR_MAP_ENTRY_SNOOP	0x4000	/* Snoop */
@@ -113,6 +121,24 @@ struct dmar_ctx {
 #define	DMAR_CTX_UNLOCK(ctx)	mtx_unlock(&(ctx)->lock)
 #define	DMAR_CTX_ASSERT_LOCKED(ctx) mtx_assert(&(ctx)->lock, MA_OWNED)
 
+struct dmar_msi_data {
+	int irq;
+	int irq_rid;
+	struct resource *irq_res;
+	void *intr_handle;
+	int (*handler)(void *);
+	int msi_data_reg;
+	int msi_addr_reg;
+	int msi_uaddr_reg;
+	void (*enable_intr)(struct dmar_unit *);
+	void (*disable_intr)(struct dmar_unit *);
+	const char *name;
+};
+
+#define	DMAR_INTR_FAULT		0
+#define	DMAR_INTR_QI		1
+#define	DMAR_INTR_TOTAL		2
+
 struct dmar_unit {
 	device_t dev;
 	int unit;
@@ -122,10 +148,8 @@ struct dmar_unit {
 	/* Resources */
 	int reg_rid;
 	struct resource *regs;
-	int irq;
-	int irq_rid;
-	struct resource *irq_res;
-	void *intr_handle;
+
+	struct dmar_msi_data intrs[DMAR_INTR_TOTAL];
 
 	/* Hardware registers cache */
 	uint32_t hw_ver;
@@ -149,6 +173,25 @@ struct dmar_unit {
 	struct task fault_task;
 	struct taskqueue *fault_taskqueue;
 
+	/* QI */
+	int qi_enabled;
+	vm_offset_t inv_queue;
+	vm_size_t inv_queue_size;
+	uint32_t inv_queue_avail;
+	uint32_t inv_queue_tail;
+	volatile uint32_t inv_waitd_seq_hw; /* hw writes there on wait
+					       descr completion */
+	uint64_t inv_waitd_seq_hw_phys;
+	uint32_t inv_waitd_seq; /* next sequence number to use for wait descr */
+	u_int inv_waitd_gen;	/* seq number generation AKA seq overflows */
+	u_int inv_seq_waiters;	/* count of waiters for seq */
+	u_int inv_queue_full;	/* informational counter */
+
+	/* Delayed freeing of map entries queue processing */
+	struct dmar_map_entries_tailq tlb_flush_entries;
+	struct task qi_task;
+	struct taskqueue *qi_taskqueue;
+
 	/* Busdma delayed map load */
 	struct task dmamap_load_task;
 	TAILQ_HEAD(, bus_dmamap_dmar) delayed_maps;
@@ -164,6 +207,7 @@ struct dmar_unit {
 #define	DMAR_FAULT_ASSERT_LOCKED(dmar) mtx_assert(&(dmar)->fault_lock, MA_OWNED)
 
 #define	DMAR_IS_COHERENT(dmar)	(((dmar)->hw_ecap & DMAR_ECAP_C) != 0)
+#define	DMAR_HAS_QI(dmar)	(((dmar)->hw_ecap & DMAR_ECAP_QI) != 0)
 
 /* Barrier ids */
 #define	DMAR_BARRIER_RMRR	0
@@ -180,6 +224,8 @@ vm_pindex_t pglvl_max_pages(int pglvl);
 int ctx_is_sp_lvl(struct dmar_ctx *ctx, int lvl);
 dmar_gaddr_t pglvl_page_size(int total_pglvl, int lvl);
 dmar_gaddr_t ctx_page_size(struct dmar_ctx *ctx, int lvl);
+int calc_am(struct dmar_unit *unit, dmar_gaddr_t base, dmar_gaddr_t size,
+    dmar_gaddr_t *isizep);
 struct vm_page *dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags);
 void dmar_pgfree(vm_object_t obj, vm_pindex_t idx, int flags);
 void *dmar_map_pgtbl(vm_object_t obj, vm_pindex_t idx, int flags,
@@ -191,21 +237,33 @@ int dmar_inv_iotlb_glob(struct dmar_unit *unit);
 int dmar_flush_write_bufs(struct dmar_unit *unit);
 int dmar_enable_translation(struct dmar_unit *unit);
 int dmar_disable_translation(struct dmar_unit *unit);
-void dmar_enable_intr(struct dmar_unit *unit);
-void dmar_disable_intr(struct dmar_unit *unit);
 bool dmar_barrier_enter(struct dmar_unit *dmar, u_int barrier_id);
 void dmar_barrier_exit(struct dmar_unit *dmar, u_int barrier_id);
 
-int dmar_intr(void *arg);
+int dmar_fault_intr(void *arg);
+void dmar_enable_fault_intr(struct dmar_unit *unit);
+void dmar_disable_fault_intr(struct dmar_unit *unit);
 int dmar_init_fault_log(struct dmar_unit *unit);
 void dmar_fini_fault_log(struct dmar_unit *unit);
 
+int dmar_qi_intr(void *arg);
+void dmar_enable_qi_intr(struct dmar_unit *unit);
+void dmar_disable_qi_intr(struct dmar_unit *unit);
+int dmar_init_qi(struct dmar_unit *unit);
+void dmar_fini_qi(struct dmar_unit *unit);
+void dmar_qi_invalidate_locked(struct dmar_ctx *ctx, dmar_gaddr_t start,
+    dmar_gaddr_t size, struct dmar_qi_genseq *pseq);
+void dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit);
+void dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit);
+
 vm_object_t ctx_get_idmap_pgtbl(struct dmar_ctx *ctx, dmar_gaddr_t maxaddr);
 void put_idmap_pgtbl(vm_object_t obj);
 int ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
     vm_page_t *ma, uint64_t pflags, int flags);
 int ctx_unmap_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
     int flags);
+void ctx_flush_iotlb_sync(struct dmar_ctx *ctx, dmar_gaddr_t base,
+    dmar_gaddr_t size);
 int ctx_alloc_pgtbl(struct dmar_ctx *ctx);
 void ctx_free_pgtbl(struct dmar_ctx *ctx);
 
@@ -217,8 +275,10 @@ void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx);
 void dmar_free_ctx(struct dmar_ctx *ctx);
 struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, int bus,
     int slot, int func);
+void dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free);
 void dmar_ctx_unload(struct dmar_ctx *ctx,
     struct dmar_map_entries_tailq *entries, bool cansleep);
+void dmar_ctx_free_entry(struct dmar_map_entry *entry, bool free);
 
 int dmar_init_busdma(struct dmar_unit *unit);
 void dmar_fini_busdma(struct dmar_unit *unit);
@@ -231,6 +291,7 @@ void dmar_gas_free_space(struct dmar_ctx *ctx, struct dmar_map_entry *entry);
 int dmar_gas_map(struct dmar_ctx *ctx, const struct bus_dma_tag_common *common,
     dmar_gaddr_t size, u_int eflags, u_int flags, vm_page_t *ma,
     struct dmar_map_entry **res);
+void dmar_gas_free_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry);
 int dmar_gas_map_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
     u_int eflags, u_int flags, vm_page_t *ma);
 int dmar_gas_reserve_region(struct dmar_ctx *ctx, dmar_gaddr_t start,
diff --git a/sys/x86/iommu/intel_drv.c b/sys/x86/iommu/intel_drv.c
index c84a410..c4f14ce 100644
--- a/sys/x86/iommu/intel_drv.c
+++ b/sys/x86/iommu/intel_drv.c
@@ -71,8 +71,9 @@ __FBSDID("$FreeBSD$");
 #include "pcib_if.h"
 #endif
 
-#define	DMAR_REG_RID	1
-#define	DMAR_IRQ_RID	0
+#define	DMAR_FAULT_IRQ_RID	0
+#define	DMAR_QI_IRQ_RID		1
+#define	DMAR_REG_RID		2
 
 static devclass_t dmar_devclass;
 static device_t *dmar_devs;
@@ -221,20 +222,31 @@ dmar_probe(device_t dev)
 }
 
 static void
+dmar_release_intr(device_t dev, struct dmar_unit *unit, int idx)
+{
+	struct dmar_msi_data *dmd;
+
+	dmd = &unit->intrs[idx];
+	if (dmd->irq == -1)
+		return;
+	bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle);
+	bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res);
+	bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid);
+	PCIB_RELEASE_MSIX(device_get_parent(device_get_parent(dev)),
+	    dev, dmd->irq);
+	dmd->irq = -1;
+}
+
+static void
 dmar_release_resources(device_t dev, struct dmar_unit *unit)
 {
+	int i;
 
 	dmar_fini_busdma(unit);
+	dmar_fini_qi(unit);
 	dmar_fini_fault_log(unit);
-	if (unit->irq != -1) {
-		bus_teardown_intr(dev, unit->irq_res, unit->intr_handle);
-		bus_release_resource(dev, SYS_RES_IRQ, unit->irq_rid,
-		    unit->irq_res);
-		bus_delete_resource(dev, SYS_RES_IRQ, unit->irq_rid);
-		PCIB_RELEASE_MSIX(device_get_parent(device_get_parent(dev)),
-		    dev, unit->irq);
-		unit->irq = -1;
-	}
+	for (i = 0; i < DMAR_INTR_TOTAL; i++)
+		dmar_release_intr(dev, unit, i);
 	if (unit->regs != NULL) {
 		bus_deactivate_resource(dev, SYS_RES_MEMORY, unit->reg_rid,
 		    unit->regs);
@@ -253,62 +265,66 @@ dmar_release_resources(device_t dev, struct dmar_unit *unit)
 }
 
 static int
-dmar_alloc_irq(device_t dev, struct dmar_unit *unit)
+dmar_alloc_irq(device_t dev, struct dmar_unit *unit, int idx)
 {
 	device_t pcib;
+	struct dmar_msi_data *dmd;
 	uint64_t msi_addr;
 	uint32_t msi_data;
 	int error;
 
+	dmd = &unit->intrs[idx];
 	pcib = device_get_parent(device_get_parent(dev)); /* Really not pcib */
-	error = PCIB_ALLOC_MSIX(pcib, dev, &unit->irq);
+	error = PCIB_ALLOC_MSIX(pcib, dev, &dmd->irq);
 	if (error != 0) {
-		device_printf(dev, "cannot allocate fault interrupt, %d\n",
-		    error);
+		device_printf(dev, "cannot allocate %s interrupt, %d\n",
+		    dmd->name, error);
 		goto err1;
 	}
-	unit->irq_rid = DMAR_IRQ_RID;
-	error = bus_set_resource(dev, SYS_RES_IRQ, unit->irq_rid, unit->irq,
-	    1);
+	error = bus_set_resource(dev, SYS_RES_IRQ, dmd->irq_rid,
+	    dmd->irq, 1);
 	if (error != 0) {
-		device_printf(dev, "cannot set interrupt resource, %d\n",
-		    error);
+		device_printf(dev, "cannot set %s interrupt resource, %d\n",
+		    dmd->name, error);
 		goto err2;
 	}
-	unit->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
-	    &unit->irq_rid, RF_ACTIVE);
-	if (unit->irq_res == NULL) {
-		device_printf(dev, "cannot map fault interrupt\n");
+	dmd->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+	    &dmd->irq_rid, RF_ACTIVE);
+	if (dmd->irq_res == NULL) {
+		device_printf(dev,
+		    "cannot allocate resource for %s interrupt\n", dmd->name);
 		error = ENXIO;
 		goto err3;
 	}
-	error = bus_setup_intr(dev, unit->irq_res, INTR_TYPE_MISC,
-	    dmar_intr, NULL, unit, &unit->intr_handle);
+	error = bus_setup_intr(dev, dmd->irq_res, INTR_TYPE_MISC,
+	    dmd->handler, NULL, unit, &dmd->intr_handle);
 	if (error != 0) {
-		device_printf(dev, "cannot setup fault interrupt, %d\n", error);
+		device_printf(dev, "cannot setup %s interrupt, %d\n",
+		    dmd->name, error);
 		goto err4;
 	}
-	bus_describe_intr(dev, unit->irq_res, unit->intr_handle, "fault");
-	error = PCIB_MAP_MSI(pcib, dev, unit->irq, &msi_addr, &msi_data);
+	bus_describe_intr(dev, dmd->irq_res, dmd->intr_handle, dmd->name);
+	error = PCIB_MAP_MSI(pcib, dev, dmd->irq, &msi_addr, &msi_data);
 	if (error != 0) {
-		device_printf(dev, "cannot map interrupt, %d\n", error);
+		device_printf(dev, "cannot map %s interrupt, %d\n",
+		    dmd->name, error);
 		goto err5;
 	}
-	dmar_write4(unit, DMAR_FEDATA_REG, msi_data);
-	dmar_write4(unit, DMAR_FEADDR_REG, msi_addr);
+	dmar_write4(unit, dmd->msi_data_reg, msi_data);
+	dmar_write4(unit, dmd->msi_addr_reg, msi_addr);
 	/* Only for xAPIC mode */
-	dmar_write4(unit, DMAR_FEUADDR_REG, msi_addr >> 32);
+	dmar_write4(unit, dmd->msi_uaddr_reg, msi_addr >> 32);
 	return (0);
 
 err5:
-	bus_teardown_intr(dev, unit->irq_res, unit->intr_handle);
+	bus_teardown_intr(dev, dmd->irq_res, dmd->intr_handle);
 err4:
-	bus_release_resource(dev, SYS_RES_IRQ, unit->irq_rid, unit->irq_res);
+	bus_release_resource(dev, SYS_RES_IRQ, dmd->irq_rid, dmd->irq_res);
 err3:
-	bus_delete_resource(dev, SYS_RES_IRQ, unit->irq_rid);
+	bus_delete_resource(dev, SYS_RES_IRQ, dmd->irq_rid);
 err2:
-	PCIB_RELEASE_MSIX(pcib, dev, unit->irq);
-	unit->irq = -1;
+	PCIB_RELEASE_MSIX(pcib, dev, dmd->irq);
+	dmd->irq = -1;
 err1:
 	return (error);
 }
@@ -318,23 +334,31 @@ static int
 dmar_remap_intr(device_t dev, device_t child, u_int irq)
 {
 	struct dmar_unit *unit;
+	struct dmar_msi_data *dmd;
 	uint64_t msi_addr;
 	uint32_t msi_data;
-	int error;
+	int i, error;
 
 	unit = device_get_softc(dev);
-	if (irq != unit->irq)
-		return (ENOENT);
-	error = PCIB_MAP_MSI(device_get_parent(device_get_parent(dev)), dev,
-	    irq, &msi_addr, &msi_data);
-	if (error != 0)
-		return (error);
-	dmar_disable_intr(unit);
-	dmar_write4(unit, DMAR_FEDATA_REG, msi_data);
-	dmar_write4(unit, DMAR_FEADDR_REG, msi_addr);
-	dmar_write4(unit, DMAR_FEUADDR_REG, msi_addr >> 32);
-	dmar_enable_intr(unit);
-	return (0);
+	for (i = 0; i < DMAR_INTR_TOTAL; i++) {
+		dmd = &unit->intrs[i];
+		if (irq == dmd->irq) {
+			error = PCIB_MAP_MSI(device_get_parent(
+			    device_get_parent(dev)),
+			    dev, irq, &msi_addr, &msi_data);
+			if (error != 0)
+				return (error);
+			DMAR_LOCK(unit);
+			(dmd->disable_intr)(unit);
+			dmar_write4(unit, dmd->msi_data_reg, msi_data);
+			dmar_write4(unit, dmd->msi_addr_reg, msi_addr);
+			dmar_write4(unit, dmd->msi_uaddr_reg, msi_addr >> 32);
+			(dmd->enable_intr)(unit);
+			DMAR_UNLOCK(unit);
+			return (0);
+		}
+	}
+	return (ENOENT);
 }
 #endif
 
@@ -372,7 +396,7 @@ dmar_attach(device_t dev)
 {
 	struct dmar_unit *unit;
 	ACPI_DMAR_HARDWARE_UNIT *dmaru;
-	int error;
+	int i, error;
 
 	unit = device_get_softc(dev);
 	unit->dev = dev;
@@ -380,7 +404,6 @@ dmar_attach(device_t dev)
 	dmaru = dmar_find_by_index(unit->unit);
 	if (dmaru == NULL)
 		return (EINVAL);
-	unit->irq = -1;
 	unit->segment = dmaru->Segment;
 	unit->base = dmaru->Address;
 	unit->reg_rid = DMAR_REG_RID;
@@ -397,11 +420,38 @@ dmar_attach(device_t dev)
 		dmar_print_caps(dev, unit, dmaru);
 	dmar_quirks_post_ident(unit);
 
-	error = dmar_alloc_irq(dev, unit);
+	for (i = 0; i < DMAR_INTR_TOTAL; i++)
+		unit->intrs[i].irq = -1;
+
+	unit->intrs[DMAR_INTR_FAULT].name = "fault";
+	unit->intrs[DMAR_INTR_FAULT].irq_rid = DMAR_FAULT_IRQ_RID;
+	unit->intrs[DMAR_INTR_FAULT].handler = dmar_fault_intr;
+	unit->intrs[DMAR_INTR_FAULT].msi_data_reg = DMAR_FEDATA_REG;
+	unit->intrs[DMAR_INTR_FAULT].msi_addr_reg = DMAR_FEADDR_REG;
+	unit->intrs[DMAR_INTR_FAULT].msi_uaddr_reg = DMAR_FEUADDR_REG;
+	unit->intrs[DMAR_INTR_FAULT].enable_intr = dmar_enable_fault_intr;
+	unit->intrs[DMAR_INTR_FAULT].disable_intr = dmar_disable_fault_intr;
+	error = dmar_alloc_irq(dev, unit, DMAR_INTR_FAULT);
 	if (error != 0) {
 		dmar_release_resources(dev, unit);
 		return (error);
 	}
+	if (DMAR_HAS_QI(unit)) {
+		unit->intrs[DMAR_INTR_QI].name = "qi";
+		unit->intrs[DMAR_INTR_QI].irq_rid = DMAR_QI_IRQ_RID;
+		unit->intrs[DMAR_INTR_QI].handler = dmar_qi_intr;
+		unit->intrs[DMAR_INTR_QI].msi_data_reg = DMAR_IEDATA_REG;
+		unit->intrs[DMAR_INTR_QI].msi_addr_reg = DMAR_IEADDR_REG;
+		unit->intrs[DMAR_INTR_QI].msi_uaddr_reg = DMAR_IEUADDR_REG;
+		unit->intrs[DMAR_INTR_QI].enable_intr = dmar_enable_qi_intr;
+		unit->intrs[DMAR_INTR_QI].disable_intr = dmar_disable_qi_intr;
+		error = dmar_alloc_irq(dev, unit, DMAR_INTR_QI);
+		if (error != 0) {
+			dmar_release_resources(dev, unit);
+			return (error);
+		}
+	}
+
 	mtx_init(&unit->lock, "dmarhw", NULL, MTX_DEF);
 	unit->domids = new_unrhdr(0, dmar_nd2mask(DMAR_CAP_ND(unit->hw_cap)),
 	    &unit->lock);
@@ -453,6 +503,11 @@ dmar_attach(device_t dev)
 		dmar_release_resources(dev, unit);
 		return (error);
 	}
+	error = dmar_init_qi(unit);
+	if (error != 0) {
+		dmar_release_resources(dev, unit);
+		return (error);
+	}
 	error = dmar_init_busdma(unit);
 	if (error != 0) {
 		dmar_release_resources(dev, unit);
@@ -1058,6 +1113,33 @@ dmar_print_one(int idx, bool show_ctxs, bool show_mappings)
 		    (uintmax_t)dmar_read8(unit, frir),
 		    (uintmax_t)dmar_read8(unit, frir + 8));
 	}
+	if (DMAR_HAS_QI(unit)) {
+		db_printf("ied 0x%x iea 0x%x ieua 0x%x\n",
+		    dmar_read4(unit, DMAR_IEDATA_REG),
+		    dmar_read4(unit, DMAR_IEADDR_REG),
+		    dmar_read4(unit, DMAR_IEUADDR_REG));
+		if (unit->qi_enabled) {
+			db_printf("qi is enabled: queue @0x%jx (IQA 0x%jx) "
+			    "size 0x%jx\n"
+		    "  head 0x%x tail 0x%x avail 0x%x status 0x%x ctrl 0x%x\n"
+		    "  hw compl 0x%x@%p/phys@%jx next seq 0x%x gen 0x%x\n",
+			    (uintmax_t)unit->inv_queue,
+			    (uintmax_t)dmar_read8(unit, DMAR_IQA_REG),
+			    (uintmax_t)unit->inv_queue_size,
+			    dmar_read4(unit, DMAR_IQH_REG),
+			    dmar_read4(unit, DMAR_IQT_REG),
+			    unit->inv_queue_avail,
+			    dmar_read4(unit, DMAR_ICS_REG),
+			    dmar_read4(unit, DMAR_IECTL_REG),
+			    unit->inv_waitd_seq_hw,
+			    &unit->inv_waitd_seq_hw,
+			    (uintmax_t)unit->inv_waitd_seq_hw_phys,
+			    unit->inv_waitd_seq,
+			    unit->inv_waitd_gen);
+		} else {
+			db_printf("qi is disabled\n");
+		}
+	}
 	if (show_ctxs) {
 		db_printf("contexts:\n");
 		LIST_FOREACH(ctx, &unit->contexts, link) {
diff --git a/sys/x86/iommu/intel_fault.c b/sys/x86/iommu/intel_fault.c
index 5c5a94d..11d4b37 100644
--- a/sys/x86/iommu/intel_fault.c
+++ b/sys/x86/iommu/intel_fault.c
@@ -85,7 +85,7 @@ dmar_fault_next(struct dmar_unit *unit, int faultp)
 }
 
 static void
-dmar_intr_clear(struct dmar_unit *unit, uint32_t fsts)
+dmar_fault_intr_clear(struct dmar_unit *unit, uint32_t fsts)
 {
 	uint32_t clear;
 
@@ -117,7 +117,7 @@ dmar_intr_clear(struct dmar_unit *unit, uint32_t fsts)
 }
 
 int
-dmar_intr(void *arg)
+dmar_fault_intr(void *arg)
 {
 	struct dmar_unit *unit;
 	uint64_t fault_rec[2];
@@ -128,7 +128,7 @@ dmar_intr(void *arg)
 	unit = arg;
 	enqueue = false;
 	fsts = dmar_read4(unit, DMAR_FSTS_REG);
-	dmar_intr_clear(unit, fsts);
+	dmar_fault_intr_clear(unit, fsts);
 
 	if ((fsts & DMAR_FSTS_PPF) == 0)
 		goto done;
@@ -263,9 +263,11 @@ dmar_init_fault_log(struct dmar_unit *unit)
 	taskqueue_start_threads(&unit->fault_taskqueue, 1, PI_AV,
 	    "dmar%d fault taskq", unit->unit);
 
-	dmar_disable_intr(unit);
+	DMAR_LOCK(unit);
+	dmar_disable_fault_intr(unit);
 	dmar_clear_faults(unit);
-	dmar_enable_intr(unit);
+	dmar_enable_fault_intr(unit);
+	DMAR_UNLOCK(unit);
 
 	return (0);
 }
@@ -274,16 +276,40 @@ void
 dmar_fini_fault_log(struct dmar_unit *unit)
 {
 
-	dmar_disable_intr(unit);
+	DMAR_LOCK(unit);
+	dmar_disable_fault_intr(unit);
+	DMAR_UNLOCK(unit);
 
 	if (unit->fault_taskqueue == NULL)
 		return;
 
 	taskqueue_drain(unit->fault_taskqueue, &unit->fault_task);
 	taskqueue_free(unit->fault_taskqueue);
+	unit->fault_taskqueue = NULL;
 	mtx_destroy(&unit->fault_lock);
 
 	free(unit->fault_log, M_DEVBUF);
 	unit->fault_log = NULL;
 	unit->fault_log_head = unit->fault_log_tail = 0;
 }
+
+void
+dmar_enable_fault_intr(struct dmar_unit *unit)
+{
+	uint32_t fectl;
+
+	DMAR_ASSERT_LOCKED(unit);
+	fectl = dmar_read4(unit, DMAR_FECTL_REG);
+	fectl &= ~DMAR_FECTL_IM;
+	dmar_write4(unit, DMAR_FECTL_REG, fectl);
+}
+
+void
+dmar_disable_fault_intr(struct dmar_unit *unit)
+{
+	uint32_t fectl;
+
+	DMAR_ASSERT_LOCKED(unit);
+	fectl = dmar_read4(unit, DMAR_FECTL_REG);
+	dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM);
+}
diff --git a/sys/x86/iommu/intel_gas.c b/sys/x86/iommu/intel_gas.c
index bd3c400..51ad151 100644
--- a/sys/x86/iommu/intel_gas.c
+++ b/sys/x86/iommu/intel_gas.c
@@ -92,8 +92,10 @@ dmar_gas_alloc_entry(struct dmar_ctx *ctx, u_int flags)
 
 	res = uma_zalloc(dmar_map_entry_zone, ((flags & DMAR_PGF_WAITOK) !=
 	    0 ? M_WAITOK : M_NOWAIT) | M_ZERO);
-	if (res != NULL)
+	if (res != NULL) {
+		res->ctx = ctx;
 		atomic_add_int(&ctx->entries_cnt, 1);
+	}
 	return (res);
 }
 
@@ -101,6 +103,9 @@ void
 dmar_gas_free_entry(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
 {
 
+	KASSERT(ctx == entry->ctx,
+	    ("mismatched free ctx %p entry %p entry->ctx %p", ctx,
+	    entry, entry->ctx));
 	atomic_subtract_int(&ctx->entries_cnt, 1);
 	uma_zfree(dmar_map_entry_zone, entry);
 }
@@ -170,6 +175,9 @@ dmar_gas_check_free(struct dmar_ctx *ctx)
 	dmar_gaddr_t v;
 
 	RB_FOREACH(entry, dmar_gas_entries_tree, &ctx->rb_root) {
+		KASSERT(ctx == entry->ctx,
+		    ("mismatched free ctx %p entry %p entry->ctx %p", ctx,
+		    entry, entry->ctx));
 		next = RB_NEXT(dmar_gas_entries_tree, &ctx->rb_root, entry);
 		if (next == NULL) {
 			MPASS(entry->free_after == ctx->end - entry->end);
@@ -583,7 +591,7 @@ dmar_gas_free_space(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
 #endif
 }
 
-static void
+void
 dmar_gas_free_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry)
 {
 	struct dmar_map_entry *next, *prev;
@@ -644,10 +652,7 @@ dmar_gas_map(struct dmar_ctx *ctx, const struct bus_dma_tag_common *common,
 	    ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0),
 	    (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0);
 	if (error == ENOMEM) {
-		DMAR_CTX_LOCK(ctx);
-		dmar_gas_free_space(ctx, entry);
-		DMAR_CTX_UNLOCK(ctx);
-		dmar_gas_free_entry(ctx, entry);
+		dmar_ctx_unload_entry(entry, true);
 		return (error);
 	}
 	KASSERT(error == 0,
@@ -689,10 +694,7 @@ dmar_gas_map_region(struct dmar_ctx *ctx, struct dmar_map_entry *entry,
 	    ((eflags & DMAR_MAP_ENTRY_TM) != 0 ? DMAR_PTE_TM : 0),
 	    (flags & DMAR_GM_CANWAIT) != 0 ? DMAR_PGF_WAITOK : 0);
 	if (error == ENOMEM) {
-		DMAR_CTX_LOCK(ctx);
-		dmar_gas_free_region(ctx, entry);
-		DMAR_CTX_UNLOCK(ctx);
-		entry->flags = 0;
+		dmar_ctx_unload_entry(entry, false);
 		return (error);
 	}
 	KASSERT(error == 0,
diff --git a/sys/x86/iommu/intel_idpgtbl.c b/sys/x86/iommu/intel_idpgtbl.c
index a1773aa..f62b1c1 100644
--- a/sys/x86/iommu/intel_idpgtbl.c
+++ b/sys/x86/iommu/intel_idpgtbl.c
@@ -67,8 +67,6 @@ __FBSDID("$FreeBSD$");
 
 static int ctx_unmap_buf_locked(struct dmar_ctx *ctx, dmar_gaddr_t base,
     dmar_gaddr_t size, int flags);
-static void ctx_flush_iotlb(struct dmar_ctx *ctx, dmar_gaddr_t base,
-    dmar_gaddr_t size, int flags);
 
 /*
  * The cache of the identity mapping page tables for the DMARs.  Using
@@ -412,7 +410,6 @@ static int
 ctx_map_buf_locked(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
     vm_page_t *ma, uint64_t pflags, int flags)
 {
-	struct dmar_unit *unit;
 	dmar_pte_t *pte;
 	struct sf_buf *sf;
 	dmar_gaddr_t pg_sz, base1, size1;
@@ -482,17 +479,6 @@ ctx_map_buf_locked(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
 	}
 	if (sf != NULL)
 		dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar));
-	DMAR_CTX_PGUNLOCK(ctx);
-	unit = ctx->dmar;
-	if ((unit->hw_cap & DMAR_CAP_CM) != 0)
-		ctx_flush_iotlb(ctx, base1, size1, flags);
-	else if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) {
-		/* See 11.1 Write Buffer Flushing. */
-		DMAR_LOCK(unit);
-		dmar_flush_write_bufs(unit);
-		DMAR_UNLOCK(unit);
-	}
-
 	TD_PINNED_ASSERT;
 	return (0);
 }
@@ -501,6 +487,10 @@ int
 ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
     vm_page_t *ma, uint64_t pflags, int flags)
 {
+	struct dmar_unit *unit;
+	int error;
+
+	unit = ctx->dmar;
 
 	KASSERT((ctx->flags & DMAR_CTX_IDMAP) == 0,
 	    ("modifying idmap pagetable ctx %p", ctx));
@@ -527,17 +517,30 @@ ctx_map_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
 	    DMAR_PTE_TM)) == 0,
 	    ("invalid pte flags %jx", (uintmax_t)pflags));
 	KASSERT((pflags & DMAR_PTE_SNP) == 0 ||
-	    (ctx->dmar->hw_ecap & DMAR_ECAP_SC) != 0,
+	    (unit->hw_ecap & DMAR_ECAP_SC) != 0,
 	    ("PTE_SNP for dmar without snoop control %p %jx",
 	    ctx, (uintmax_t)pflags));
 	KASSERT((pflags & DMAR_PTE_TM) == 0 ||
-	    (ctx->dmar->hw_ecap & DMAR_ECAP_DI) != 0,
+	    (unit->hw_ecap & DMAR_ECAP_DI) != 0,
 	    ("PTE_TM for dmar without DIOTLB %p %jx",
 	    ctx, (uintmax_t)pflags));
 	KASSERT((flags & ~DMAR_PGF_WAITOK) == 0, ("invalid flags %x", flags));
 
 	DMAR_CTX_PGLOCK(ctx);
-	return (ctx_map_buf_locked(ctx, base, size, ma, pflags, flags));
+	error = ctx_map_buf_locked(ctx, base, size, ma, pflags, flags);
+	DMAR_CTX_PGUNLOCK(ctx);
+	if (error != 0)
+		return (error);
+
+	if ((unit->hw_cap & DMAR_CAP_CM) != 0)
+		ctx_flush_iotlb_sync(ctx, base, size);
+	else if ((unit->hw_cap & DMAR_CAP_RWBF) != 0) {
+		/* See 11.1 Write Buffer Flushing. */
+		DMAR_LOCK(unit);
+		dmar_flush_write_bufs(unit);
+		DMAR_UNLOCK(unit);
+	}
+	return (0);
 }
 
 static void ctx_unmap_clear_pte(struct dmar_ctx *ctx, dmar_gaddr_t base,
@@ -646,8 +649,6 @@ ctx_unmap_buf_locked(struct dmar_ctx *ctx, dmar_gaddr_t base,
 	}
 	if (sf != NULL)
 		dmar_unmap_pgtbl(sf, DMAR_IS_COHERENT(ctx->dmar));
-	DMAR_CTX_PGUNLOCK(ctx);
-	ctx_flush_iotlb(ctx, base1, size1, flags);
 	/*
 	 * See 11.1 Write Buffer Flushing for an explanation why RWBF
 	 * can be ignored there.
@@ -661,9 +662,12 @@ int
 ctx_unmap_buf(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
     int flags)
 {
+	int error;
 
 	DMAR_CTX_PGLOCK(ctx);
-	return (ctx_unmap_buf_locked(ctx, base, size, flags));
+	error = ctx_unmap_buf_locked(ctx, base, size, flags);
+	DMAR_CTX_PGUNLOCK(ctx);
+	return (error);
 }
 
 int
@@ -730,13 +734,8 @@ ctx_wait_iotlb_flush(struct dmar_unit *unit, uint64_t wt, int iro)
 	return (iotlbr);
 }
 
-/*
- * flags is only intended for PGF_WAITOK, to disallow queued
- * invalidation.
- */
-static void
-ctx_flush_iotlb(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
-    int flags)
+void
+ctx_flush_iotlb_sync(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size)
 {
 	struct dmar_unit *unit;
 	dmar_gaddr_t isize;
@@ -744,20 +743,8 @@ ctx_flush_iotlb(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
 	int am, iro;
 
 	unit = ctx->dmar;
-#if 0
-	if ((unit->hw_ecap & DMAR_ECAP_QI) != 0 &&
-	    (flags & DMAR_PGF_WAITOK) != 0) {
-		/*
-		 * XXXKIB: There, a queued invalidation interface
-		 * could be used.  But since queued and registered
-		 * interfaces cannot be used simultaneously, and we
-		 * must use sleep-less (i.e. register) interface when
-		 * DMAR_PGF_WAITOK is not specified, only register
-		 * interface is suitable.
-		 */
-		return;
-	}
-#endif
+	KASSERT(!unit->qi_enabled, ("dmar%d: sync iotlb flush call",
+	    unit->unit));
 	iro = DMAR_ECAP_IRO(unit->hw_ecap) * 16;
 	DMAR_LOCK(unit);
 	if ((unit->hw_cap & DMAR_CAP_PSI) == 0 || size > 2 * 1024 * 1024) {
@@ -769,13 +756,7 @@ ctx_flush_iotlb(struct dmar_ctx *ctx, dmar_gaddr_t base, dmar_gaddr_t size,
 		    (uintmax_t)iotlbr));
 	} else {
 		for (; size > 0; base += isize, size -= isize) {
-			for (am = DMAR_CAP_MAMV(unit->hw_cap);; am--) {
-				isize = 1ULL << (am + DMAR_PAGE_SHIFT);
-				if ((base & (isize - 1)) == 0 && size >= isize)
-					break;
-				if (am == 0)
-					break;
-			}
+			am = calc_am(unit, base, size, &isize);
 			dmar_write8(unit, iro, base | am);
 			iotlbr = ctx_wait_iotlb_flush(unit,
 			    DMAR_IOTLB_IIRG_PAGE | DMAR_IOTLB_DID(ctx->domain),
diff --git a/sys/x86/iommu/intel_qi.c b/sys/x86/iommu/intel_qi.c
new file mode 100644
index 0000000..a682c93
--- /dev/null
+++ b/sys/x86/iommu/intel_qi.c
@@ -0,0 +1,414 @@
+/*-
+ * Copyright (c) 2013 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "opt_acpi.h"
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/memdesc.h>
+#include <sys/module.h>
+#include <sys/rman.h>
+#include <sys/taskqueue.h>
+#include <sys/tree.h>
+#include <machine/bus.h>
+#include <contrib/dev/acpica/include/acpi.h>
+#include <contrib/dev/acpica/include/accommon.h>
+#include <dev/acpica/acpivar.h>
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+#include <machine/cpu.h>
+#include <x86/include/busdma_impl.h>
+#include <x86/iommu/intel_reg.h>
+#include <x86/iommu/busdma_dmar.h>
+#include <x86/iommu/intel_dmar.h>
+
+static bool
+dmar_qi_seq_processed(const struct dmar_unit *unit,
+    const struct dmar_qi_genseq *pseq)
+{
+
+	return (pseq->gen < unit->inv_waitd_gen ||
+	    (pseq->gen == unit->inv_waitd_gen &&
+	     pseq->seq <= unit->inv_waitd_seq_hw));
+}
+
+static int
+dmar_enable_qi(struct dmar_unit *unit)
+{
+
+	DMAR_ASSERT_LOCKED(unit);
+	unit->hw_gcmd |= DMAR_GCMD_QIE;
+	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
+	/* XXXKIB should have a timeout */
+	while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) == 0)
+		cpu_spinwait();
+	return (0);
+}
+
+static int
+dmar_disable_qi(struct dmar_unit *unit)
+{
+
+	DMAR_ASSERT_LOCKED(unit);
+	unit->hw_gcmd &= ~DMAR_GCMD_QIE;
+	dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd);
+	/* XXXKIB should have a timeout */
+	while ((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) != 0)
+		cpu_spinwait();
+	return (0);
+}
+
+static void
+dmar_qi_advance_tail(struct dmar_unit *unit)
+{
+
+	DMAR_ASSERT_LOCKED(unit);
+	dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail);
+}
+
+static void
+dmar_qi_ensure(struct dmar_unit *unit, int descr_count)
+{
+	uint32_t head;
+	int bytes;
+
+	DMAR_ASSERT_LOCKED(unit);
+	bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT;
+	for (;;) {
+		if (bytes <= unit->inv_queue_avail)
+			break;
+		/* refill */
+		head = dmar_read4(unit, DMAR_IQH_REG);
+		head &= DMAR_IQH_MASK;
+		unit->inv_queue_avail = head - unit->inv_queue_tail -
+		    DMAR_IQ_DESCR_SZ;
+		if (head <= unit->inv_queue_tail)
+			unit->inv_queue_avail += unit->inv_queue_size;
+		if (bytes <= unit->inv_queue_avail)
+			break;
+
+		/*
+		 * No space in the queue, do busy wait.  Hardware must
+		 * make a progress.  But first advance the tail to
+		 * inform the descriptor streamer about entries we
+		 * might have already filled, otherwise they could
+		 * clog the whole queue..
+		 */
+		dmar_qi_advance_tail(unit);
+		unit->inv_queue_full++;
+		cpu_spinwait();
+	}
+	unit->inv_queue_avail -= bytes;
+}
+
+static void
+dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2)
+{
+
+	DMAR_ASSERT_LOCKED(unit);
+	*(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1;
+	unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
+	KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
+	    ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
+	    (uintmax_t)unit->inv_queue_size));
+	unit->inv_queue_tail &= unit->inv_queue_size - 1;
+	*(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2;
+	unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2;
+	KASSERT(unit->inv_queue_tail <= unit->inv_queue_size,
+	    ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail,
+	    (uintmax_t)unit->inv_queue_size));
+	unit->inv_queue_tail &= unit->inv_queue_size - 1;
+}
+
+static void
+dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr,
+    bool memw, bool fence)
+{
+
+	DMAR_ASSERT_LOCKED(unit);
+	dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID |
+	    (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) |
+	    (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) |
+	    (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) |
+	    (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0),
+	    memw ? unit->inv_waitd_seq_hw_phys : 0);
+}
+
+static void
+dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq)
+{
+	struct dmar_qi_genseq gsec;
+	uint32_t seq;
+
+	KASSERT(pseq != NULL, ("wait descriptor with no place for seq"));
+	DMAR_ASSERT_LOCKED(unit);
+	if (unit->inv_waitd_seq == 0xffffffff) {
+		gsec.gen = unit->inv_waitd_gen;
+		gsec.seq = unit->inv_waitd_seq;
+		dmar_qi_ensure(unit, 1);
+		dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false);
+		dmar_qi_advance_tail(unit);
+		while (!dmar_qi_seq_processed(unit, &gsec))
+			cpu_spinwait();
+		unit->inv_waitd_gen++;
+		unit->inv_waitd_seq = 1;
+	}
+	seq = unit->inv_waitd_seq++;
+	pseq->gen = unit->inv_waitd_gen;
+	pseq->seq = seq;
+	dmar_qi_emit_wait_descr(unit, seq, true, true, false);
+}
+
+static void
+dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq)
+{
+
+	DMAR_ASSERT_LOCKED(unit);
+	unit->inv_seq_waiters++;
+	while (!dmar_qi_seq_processed(unit, gseq)) {
+		if (cold) {
+			cpu_spinwait();
+		} else {
+			msleep(&unit->inv_seq_waiters, &unit->lock, 0,
+			    "dmarse", hz);
+		}
+	}
+	unit->inv_seq_waiters--;
+}
+
+void
+dmar_qi_invalidate_locked(struct dmar_ctx *ctx, dmar_gaddr_t base,
+    dmar_gaddr_t size, struct dmar_qi_genseq *pseq)
+{
+	struct dmar_unit *unit;
+	dmar_gaddr_t isize;
+	int am;
+
+	unit = ctx->dmar;
+	DMAR_ASSERT_LOCKED(unit);
+	for (; size > 0; base += isize, size -= isize) {
+		am = calc_am(unit, base, size, &isize);
+		dmar_qi_ensure(unit, 1);
+		dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV |
+		    DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW |
+		    DMAR_IQ_DESCR_IOTLB_DR |
+		    DMAR_IQ_DESCR_IOTLB_DID(ctx->domain),
+		    base | am);
+	}
+	if (pseq != NULL) {
+		dmar_qi_ensure(unit, 1);
+		dmar_qi_emit_wait_seq(unit, pseq);
+	}
+	dmar_qi_advance_tail(unit);
+}
+
+void
+dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit)
+{
+	struct dmar_qi_genseq gseq;
+
+	DMAR_ASSERT_LOCKED(unit);
+	dmar_qi_ensure(unit, 2);
+	dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0);
+	dmar_qi_emit_wait_seq(unit, &gseq);
+	dmar_qi_advance_tail(unit);
+	dmar_qi_wait_for_seq(unit, &gseq);
+}
+
+void
+dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit)
+{
+	struct dmar_qi_genseq gseq;
+
+	DMAR_ASSERT_LOCKED(unit);
+	dmar_qi_ensure(unit, 2);
+	dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB |
+	    DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0);
+	dmar_qi_emit_wait_seq(unit, &gseq);
+	dmar_qi_advance_tail(unit);
+	dmar_qi_wait_for_seq(unit, &gseq);
+}
+
+int
+dmar_qi_intr(void *arg)
+{
+	struct dmar_unit *unit;
+
+	unit = arg;
+	KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit));
+	taskqueue_enqueue_fast(unit->qi_taskqueue, &unit->qi_task);
+	return (FILTER_HANDLED);
+}
+
+static void
+dmar_qi_task(void *arg, int pending __unused)
+{
+	struct dmar_unit *unit;
+	struct dmar_map_entry *entry;
+	uint32_t ics;
+
+	unit = arg;
+
+	DMAR_LOCK(unit);
+	for (;;) {
+		entry = TAILQ_FIRST(&unit->tlb_flush_entries);
+		if (entry == NULL)
+			break;
+		if ((entry->gseq.gen == 0 && entry->gseq.seq == 0) ||
+		    !dmar_qi_seq_processed(unit, &entry->gseq))
+			break;
+		TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link);
+		DMAR_UNLOCK(unit);
+		dmar_ctx_free_entry(entry, (entry->flags &
+		    DMAR_MAP_ENTRY_QI_NF) == 0);
+		DMAR_LOCK(unit);
+	}
+	ics = dmar_read4(unit, DMAR_ICS_REG);
+	if ((ics & DMAR_ICS_IWC) != 0) {
+		ics = DMAR_ICS_IWC;
+		dmar_write4(unit, DMAR_ICS_REG, ics);
+	}
+	if (unit->inv_seq_waiters > 0)
+		wakeup(&unit->inv_seq_waiters);
+	DMAR_UNLOCK(unit);
+}
+
+int
+dmar_init_qi(struct dmar_unit *unit)
+{
+	uint64_t iqa;
+	uint32_t ics;
+	int qi_sz;
+
+	if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0)
+		return (0);
+	unit->qi_enabled = 1;
+	TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled);
+	if (!unit->qi_enabled)
+		return (0);
+
+	TAILQ_INIT(&unit->tlb_flush_entries);
+	TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit);
+	unit->qi_taskqueue = taskqueue_create_fast("dmar", M_WAITOK,
+	    taskqueue_thread_enqueue, &unit->qi_taskqueue);
+	taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV,
+	    "dmar%d qi taskq", unit->unit);
+
+	unit->inv_waitd_gen = 0;
+	unit->inv_waitd_seq = 1;
+
+	qi_sz = DMAR_IQA_QS_DEF;
+	TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz);
+	if (qi_sz > DMAR_IQA_QS_MAX)
+		qi_sz = DMAR_IQA_QS_MAX;
+	unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE;
+	/* Reserve one descriptor to prevent wraparound. */
+	unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ;
+
+	/* The invalidation queue reads by DMARs are always coherent. */
+	unit->inv_queue = kmem_alloc_contig(kernel_arena, unit->inv_queue_size,
+	    M_WAITOK | M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
+	unit->inv_waitd_seq_hw_phys = pmap_kextract(
+	    (vm_offset_t)&unit->inv_waitd_seq_hw);
+
+	DMAR_LOCK(unit);
+	dmar_write8(unit, DMAR_IQT_REG, 0);
+	iqa = pmap_kextract(unit->inv_queue);
+	iqa |= qi_sz;
+	dmar_write8(unit, DMAR_IQA_REG, iqa);
+	dmar_enable_qi(unit);
+	ics = dmar_read4(unit, DMAR_ICS_REG);
+	if ((ics & DMAR_ICS_IWC) != 0) {
+		ics = DMAR_ICS_IWC;
+		dmar_write4(unit, DMAR_ICS_REG, ics);
+	}
+	DMAR_UNLOCK(unit);
+
+	return (0);
+}
+
+void
+dmar_fini_qi(struct dmar_unit *unit)
+{
+	struct dmar_qi_genseq gseq;
+
+	if (unit->qi_enabled)
+		return;
+	taskqueue_drain(unit->qi_taskqueue, &unit->qi_task);
+	taskqueue_free(unit->qi_taskqueue);
+	unit->qi_taskqueue = NULL;
+
+	DMAR_LOCK(unit);
+	/* quisce */
+	dmar_qi_ensure(unit, 1);
+	dmar_qi_emit_wait_seq(unit, &gseq);
+	dmar_qi_advance_tail(unit);
+	dmar_qi_wait_for_seq(unit, &gseq);
+	/* only after the quisce, disable queue */
+	dmar_disable_qi(unit);
+	KASSERT(unit->inv_seq_waiters == 0,
+	    ("dmar%d: waiters on disabled queue", unit->unit));
+	DMAR_UNLOCK(unit);
+
+	kmem_free(kernel_arena, unit->inv_queue, unit->inv_queue_size);
+	unit->inv_queue = 0;
+	unit->inv_queue_size = 0;
+	unit->qi_enabled = 0;
+}
+
+void
+dmar_enable_qi_intr(struct dmar_unit *unit)
+{
+	uint32_t iectl;
+
+	DMAR_ASSERT_LOCKED(unit);
+	KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
+	iectl = dmar_read4(unit, DMAR_IECTL_REG);
+	iectl &= ~DMAR_IECTL_IM;
+	dmar_write4(unit, DMAR_IECTL_REG, iectl);
+}
+
+void
+dmar_disable_qi_intr(struct dmar_unit *unit)
+{
+	uint32_t iectl;
+
+	DMAR_ASSERT_LOCKED(unit);
+	KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit));
+	iectl = dmar_read4(unit, DMAR_IECTL_REG);
+	dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM);
+}
diff --git a/sys/x86/iommu/intel_reg.h b/sys/x86/iommu/intel_reg.h
index 1157a9b..4c266de 100644
--- a/sys/x86/iommu/intel_reg.h
+++ b/sys/x86/iommu/intel_reg.h
@@ -260,14 +260,50 @@ typedef struct dmar_pte {
 /* Protected High-Memory Limit register */
 #define	DMAR_PHMLIMIT_REG	0x78
 
+/* Queued Invalidation Descriptors */
+#define	DMAR_IQ_DESCR_SZ_SHIFT	4	/* Shift for descriptor count
+					   to ring offset */
+#define	DMAR_IQ_DESCR_SZ	(1 << DMAR_IQ_DESCR_SZ_SHIFT)
+					/* Descriptor size */
+
+#define	DMAR_IQ_DESCR_CTX_INV	0x1	/* Context-cache Invalidate
+					   Descriptor */
+#define	DMAR_IQ_DESCR_CTX_GLOB	(0x1 << 4)	/* Granularity: Global */
+#define	DMAR_IQ_DESCR_CTX_DOM	(0x2 << 4)	/* Granularity: Domain */
+#define	DMAR_IQ_DESCR_CTX_DEV	(0x3 << 4)	/* Granularity: Device */
+#define	DMAR_IQ_DESCR_CTX_DID(x) (((uint32_t)(x)) << 16) /* Domain Id */
+#define	DMAR_IQ_DESCR_CTX_SRC(x) (((uint64_t)(x)) << 32) /* Source Id */
+#define	DMAR_IQ_DESCR_CTX_FM(x)  (((uint64_t)(x)) << 48) /* Function Mask */
+
+#define	DMAR_IQ_DESCR_IOTLB_INV	0x2	/* IOTLB Invalidate Descriptor */
+#define	DMAR_IQ_DESCR_IOTLB_GLOB (0x1 << 4)	/* Granularity: Global */
+#define	DMAR_IQ_DESCR_IOTLB_DOM	 (0x2 << 4)	/* Granularity: Domain */
+#define	DMAR_IQ_DESCR_IOTLB_PAGE (0x3 << 4)	/* Granularity: Page */
+#define	DMAR_IQ_DESCR_IOTLB_DW	(1 << 6)	/* Drain Writes */
+#define	DMAR_IQ_DESCR_IOTLB_DR	(1 << 7)	/* Drain Reads */
+#define	DMAR_IQ_DESCR_IOTLB_DID(x) (((uint32_t)(x)) << 16) /* Domain Id */
+
+#define	DMAR_IQ_DESCR_WAIT_ID	0x5	/* Invalidation Wait Descriptor */
+#define	DMAR_IQ_DESCR_WAIT_IF	(1 << 4)	/* Interrupt Flag */
+#define	DMAR_IQ_DESCR_WAIT_SW	(1 << 5)	/* Status Write */
+#define	DMAR_IQ_DESCR_WAIT_FN	(1 << 6)	/* Fence */
+#define	DMAR_IQ_DESCR_WAIT_SD(x) (((uint64_t)(x)) << 32) /* Status Data */
+
 /* Invalidation Queue Head register */
 #define	DMAR_IQH_REG		0x80
+#define	DMAR_IQH_MASK		0x7fff0		/* Next cmd index mask */
 
 /* Invalidation Queue Tail register */
 #define	DMAR_IQT_REG		0x88
+#define	DMAR_IQT_MASK		0x7fff0
 
 /* Invalidation Queue Address register */
 #define	DMAR_IQA_REG		0x90
+#define	DMAR_IQA_IQA_MASK	0xfffffffffffff000 /* Invalidation Queue
+						      Base Address mask */
+#define	DMAR_IQA_QS_MASK	0x7		/* Queue Size in pages */
+#define	DMAR_IQA_QS_MAX		0x7		/* Max Queue size */
+#define	DMAR_IQA_QS_DEF		3
 
  /* Invalidation Completion Status register */
 #define	DMAR_ICS_REG		0x9c
diff --git a/sys/x86/iommu/intel_utils.c b/sys/x86/iommu/intel_utils.c
index c9d047b..d81ec04 100644
--- a/sys/x86/iommu/intel_utils.c
+++ b/sys/x86/iommu/intel_utils.c
@@ -230,6 +230,24 @@ ctx_page_size(struct dmar_ctx *ctx, int lvl)
 	return (pglvl_page_size(ctx->pglvl, lvl));
 }
 
+int
+calc_am(struct dmar_unit *unit, dmar_gaddr_t base, dmar_gaddr_t size,
+    dmar_gaddr_t *isizep)
+{
+	dmar_gaddr_t isize;
+	int am;
+
+	for (am = DMAR_CAP_MAMV(unit->hw_cap);; am--) {
+		isize = 1ULL << (am + DMAR_PAGE_SHIFT);
+		if ((base & (isize - 1)) == 0 && size >= isize)
+			break;
+		if (am == 0)
+			break;
+	}
+	*isizep = isize;
+	return (am);
+}
+
 dmar_haddr_t dmar_high;
 int haw;
 int dmar_tbl_pagecnt;
@@ -390,6 +408,7 @@ dmar_inv_ctx_glob(struct dmar_unit *unit)
 	 * command is submitted.
 	 */
 	DMAR_ASSERT_LOCKED(unit);
+	KASSERT(!unit->qi_enabled, ("QI enabled"));
 
 	/*
 	 * The DMAR_CCMD_ICC bit in the upper dword should be written
@@ -413,6 +432,7 @@ dmar_inv_iotlb_glob(struct dmar_unit *unit)
 	int reg;
 
 	DMAR_ASSERT_LOCKED(unit);
+	KASSERT(!unit->qi_enabled, ("QI enabled"));
 
 	reg = 16 * DMAR_ECAP_IRO(unit->hw_ecap);
 	/* See a comment about DMAR_CCMD_ICC in dmar_inv_ctx_glob. */
@@ -474,25 +494,6 @@ dmar_disable_translation(struct dmar_unit *unit)
 	return (0);
 }
 
-void
-dmar_enable_intr(struct dmar_unit *unit)
-{
-	uint32_t fectl;
-
-	fectl = dmar_read4(unit, DMAR_FECTL_REG);
-	fectl &= ~DMAR_FECTL_IM;
-	dmar_write4(unit, DMAR_FECTL_REG, fectl);
-}
-
-void
-dmar_disable_intr(struct dmar_unit *unit)
-{
-	uint32_t fectl;
-
-	fectl = dmar_read4(unit, DMAR_FECTL_REG);
-	dmar_write4(unit, DMAR_FECTL_REG, fectl | DMAR_FECTL_IM);
-}
-
 #define BARRIER_F				\
 	u_int f_done, f_inproc, f_wakeup;	\
 						\