MFC r264007,r264008,r264009,r264011,r264012,r264013

MFC support for PCI Alternate RID Interpretation.  ARI is an optional PCIe
feature that allows PCI devices to present up to 256 functions on a bus.
This is effectively a prerequisite for PCI SR-IOV support.

r264007:
   Add a method to get the PCI RID for a device.

   Reviewed by:  kib
   MFC after:    2 months
   Sponsored by: Sandvine Inc.

r264008:
   Re-implement the DMAR I/O MMU code in terms of PCI RIDs

   Under the hood the VT-d spec is really implemented in terms of
   PCI RIDs instead of bus/slot/function, even though the spec makes
   pains to convert back to bus/slot/function in examples.  However
   working with bus/slot/function is not correct when PCI ARI is
   in use, so convert to using RIDs in most cases.  bus/slot/function
   will only be used when reporting errors to a user.

   Reviewed by:  kib
   MFC after:    2 months
   Sponsored by: Sandvine Inc.

r264009:
   Re-write bhyve's I/O MMU handling in terms of PCI RID.

   Reviewed by:  neel
   MFC after:    2 months
   Sponsored by: Sandvine Inc.

r264011:
   Add support for PCIe ARI

   PCIe Alternate RID Interpretation (ARI) is an optional feature that
   allows devices to have up to 256 different functions.  It is
   implemented by always setting the PCI slot number to 0 and
   re-purposing the 5 bits used to encode the slot number to instead
   contain the function number.  Combined with the original 3 bits
   allocated for the function number, this allows for 256 functions.

   This is enabled by default, but it's expected to be a no-op on currently
   supported hardware.  It's a prerequisite for supporting PCI SR-IOV, and
   I want the ARI support to go in early to help shake out any bugs in it.
   ARI can be disabled by setting the tunable hw.pci.enable_ari=0.

   Reviewed by:  kib
   MFC after:    2 months
   Sponsored by: Sandvine Inc.

r264012:
   Print status of ARI capability in pciconf -c

   Teach pciconf how to print out the status (enabled/disabled) of the ARI
   capability on PCI Root Complexes and Downstream Ports.

   MFC after:    2 months
   Sponsored by: Sandvine Inc.

r264013:
   Add missing copyright date.

   MFC after:    2 months

6 files changed, 64 insertions, 47 deletions

diff --git a/sys/x86/iommu/busdma_dmar.c b/sys/x86/iommu/busdma_dmar.c
index 488c7bb..a397f42 100644
--- a/sys/x86/iommu/busdma_dmar.c
+++ b/sys/x86/iommu/busdma_dmar.c
@@ -93,7 +93,7 @@ dmar_bus_dma_is_dev_disabled(int domain, int bus, int slot, int func)
  * bounce mapping.
  */
 static device_t
-dmar_get_requester(device_t dev, int *bus, int *slot, int *func)
+dmar_get_requester(device_t dev, uint16_t *rid)
 {
 	devclass_t pci_class;
 	device_t pci, pcib, requester;
@@ -102,9 +102,7 @@ dmar_get_requester(device_t dev, int *bus, int *slot, int *func)
 	pci_class = devclass_find("pci");
 	requester = dev;
 
-	*bus = pci_get_bus(dev);
-	*slot = pci_get_slot(dev);
-	*func = pci_get_function(dev);
+	*rid = pci_get_rid(dev);
 
 	/*
 	 * Walk the bridge hierarchy from the target device to the
@@ -161,8 +159,7 @@ dmar_get_requester(device_t dev, int *bus, int *slot, int *func)
 				 * same page tables for taken and
 				 * non-taken transactions.
 				 */
-				*bus = pci_get_bus(dev);
-				*slot = *func = 0;
+				*rid = PCI_RID(pci_get_bus(dev), 0, 0);
 			} else {
 				/*
 				 * Neither the device nor the bridge
@@ -171,9 +168,7 @@ dmar_get_requester(device_t dev, int *bus, int *slot, int *func)
 				 * will use the bridge's BSF as the
 				 * requester ID.
 				 */
-				*bus = pci_get_bus(pcib);
-				*slot = pci_get_slot(pcib);
-				*func = pci_get_function(pcib);
+				*rid = pci_get_rid(pcib);
 			}
 		}
 		/*
@@ -193,9 +188,9 @@ dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr)
 	device_t requester;
 	struct dmar_ctx *ctx;
 	bool disabled;
-	int bus, slot, func;
+	uint16_t rid;
 
-	requester = dmar_get_requester(dev, &bus, &slot, &func);
+	requester = dmar_get_requester(dev, &rid);
 
 	/*
 	 * If the user requested the IOMMU disabled for the device, we
@@ -204,9 +199,10 @@ dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev, bool rmrr)
 	 * Instead provide the identity mapping for the device
 	 * context.
 	 */
-	disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(dev), bus,
-	    slot, func);
-	ctx = dmar_get_ctx(dmar, requester, bus, slot, func, disabled, rmrr);
+	disabled = dmar_bus_dma_is_dev_disabled(pci_get_domain(requester), 
+	    pci_get_bus(requester), pci_get_slot(requester), 
+	    pci_get_function(requester));
+	ctx = dmar_get_ctx(dmar, requester, rid, disabled, rmrr);
 	if (ctx == NULL)
 		return (NULL);
 	if (disabled) {
diff --git a/sys/x86/iommu/intel_ctx.c b/sys/x86/iommu/intel_ctx.c
index 97f4bc3..d6123be 100644
--- a/sys/x86/iommu/intel_ctx.c
+++ b/sys/x86/iommu/intel_ctx.c
@@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <x86/iommu/intel_reg.h>
 #include <x86/iommu/busdma_dmar.h>
 #include <x86/iommu/intel_dmar.h>
+#include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 
 static MALLOC_DEFINE(M_DMAR_CTX, "dmar_ctx", "Intel DMAR Context");
@@ -106,14 +107,14 @@ dmar_map_ctx_entry(struct dmar_ctx *ctx, struct sf_buf **sfp)
 {
 	dmar_ctx_entry_t *ctxp;
 
-	ctxp = dmar_map_pgtbl(ctx->dmar->ctx_obj, 1 + ctx->bus,
+	ctxp = dmar_map_pgtbl(ctx->dmar->ctx_obj, 1 + PCI_RID2BUS(ctx->rid),
 	    DMAR_PGF_NOALLOC | DMAR_PGF_WAITOK, sfp);
-	ctxp += ((ctx->slot & 0x1f) << 3) + (ctx->func & 0x7);
+	ctxp += ctx->rid & 0xff;
 	return (ctxp);
 }
 
 static void
-ctx_tag_init(struct dmar_ctx *ctx)
+ctx_tag_init(struct dmar_ctx *ctx, device_t dev)
 {
 	bus_addr_t maxaddr;
 
@@ -127,6 +128,7 @@ ctx_tag_init(struct dmar_ctx *ctx)
 	ctx->ctx_tag.common.nsegments = BUS_SPACE_UNRESTRICTED;
 	ctx->ctx_tag.common.maxsegsz = maxaddr;
 	ctx->ctx_tag.ctx = ctx;
+	ctx->ctx_tag.owner = dev;
 	/* XXXKIB initialize tag further */
 }
 
@@ -139,7 +141,10 @@ ctx_id_entry_init(struct dmar_ctx *ctx, dmar_ctx_entry_t *ctxp)
 	unit = ctx->dmar;
 	KASSERT(ctxp->ctx1 == 0 && ctxp->ctx2 == 0,
 	    ("dmar%d: initialized ctx entry %d:%d:%d 0x%jx 0x%jx",
-	    unit->unit, ctx->bus, ctx->slot, ctx->func, ctxp->ctx1,
+	    unit->unit, pci_get_bus(ctx->ctx_tag.owner),
+	    pci_get_slot(ctx->ctx_tag.owner),
+	    pci_get_function(ctx->ctx_tag.owner),
+	    ctxp->ctx1,
 	    ctxp->ctx2));
 	ctxp->ctx2 = DMAR_CTX2_DID(ctx->domain);
 	ctxp->ctx2 |= ctx->awlvl;
@@ -229,7 +234,7 @@ ctx_init_rmrr(struct dmar_ctx *ctx, device_t dev)
 }
 
 static struct dmar_ctx *
-dmar_get_ctx_alloc(struct dmar_unit *dmar, int bus, int slot, int func)
+dmar_get_ctx_alloc(struct dmar_unit *dmar, uint16_t rid)
 {
 	struct dmar_ctx *ctx;
 
@@ -239,9 +244,7 @@ dmar_get_ctx_alloc(struct dmar_unit *dmar, int bus, int slot, int func)
 	TASK_INIT(&ctx->unload_task, 0, dmar_ctx_unload_task, ctx);
 	mtx_init(&ctx->lock, "dmarctx", NULL, MTX_DEF);
 	ctx->dmar = dmar;
-	ctx->bus = bus;
-	ctx->slot = slot;
-	ctx->func = func;
+	ctx->rid = rid;
 	return (ctx);
 }
 
@@ -264,19 +267,22 @@ dmar_ctx_dtr(struct dmar_ctx *ctx, bool gas_inited, bool pgtbl_inited)
 }
 
 struct dmar_ctx *
-dmar_get_ctx(struct dmar_unit *dmar, device_t dev, int bus, int slot, int func,
-    bool id_mapped, bool rmrr_init)
+dmar_get_ctx(struct dmar_unit *dmar, device_t dev, uint16_t rid, bool id_mapped,
+    bool rmrr_init)
 {
 	struct dmar_ctx *ctx, *ctx1;
 	dmar_ctx_entry_t *ctxp;
 	struct sf_buf *sf;
-	int error, mgaw;
+	int bus, slot, func, error, mgaw;
 	bool enable;
 
+	bus = pci_get_bus(dev);
+	slot = pci_get_slot(dev);
+	func = pci_get_function(dev);
 	enable = false;
 	TD_PREP_PINNED_ASSERT;
 	DMAR_LOCK(dmar);
-	ctx = dmar_find_ctx_locked(dmar, bus, slot, func);
+	ctx = dmar_find_ctx_locked(dmar, rid);
 	error = 0;
 	if (ctx == NULL) {
 		/*
@@ -285,7 +291,7 @@ dmar_get_ctx(struct dmar_unit *dmar, device_t dev, int bus, int slot, int func,
 		 */
 		DMAR_UNLOCK(dmar);
 		dmar_ensure_ctx_page(dmar, bus);
-		ctx1 = dmar_get_ctx_alloc(dmar, bus, slot, func);
+		ctx1 = dmar_get_ctx_alloc(dmar, rid);
 
 		if (id_mapped) {
 			/*
@@ -353,7 +359,7 @@ dmar_get_ctx(struct dmar_unit *dmar, device_t dev, int bus, int slot, int func,
 		 * Recheck the contexts, other thread might have
 		 * already allocated needed one.
 		 */
-		ctx = dmar_find_ctx_locked(dmar, bus, slot, func);
+		ctx = dmar_find_ctx_locked(dmar, rid);
 		if (ctx == NULL) {
 			ctx = ctx1;
 			ctx->ctx_tag.owner = dev;
@@ -365,7 +371,7 @@ dmar_get_ctx(struct dmar_unit *dmar, device_t dev, int bus, int slot, int func,
 				TD_PINNED_ASSERT;
 				return (NULL);
 			}
-			ctx_tag_init(ctx);
+			ctx_tag_init(ctx, dev);
 
 			/*
 			 * This is the first activated context for the
@@ -527,14 +533,14 @@ dmar_free_ctx(struct dmar_ctx *ctx)
 }
 
 struct dmar_ctx *
-dmar_find_ctx_locked(struct dmar_unit *dmar, int bus, int slot, int func)
+dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid)
 {
 	struct dmar_ctx *ctx;
 
 	DMAR_ASSERT_LOCKED(dmar);
 
 	LIST_FOREACH(ctx, &dmar->contexts, link) {
-		if (ctx->bus == bus && ctx->slot == slot && ctx->func == func)
+		if (ctx->rid == rid)
 			return (ctx);
 	}
 	return (NULL);
diff --git a/sys/x86/iommu/intel_dmar.h b/sys/x86/iommu/intel_dmar.h
index ae06149..0f5c6c9 100644
--- a/sys/x86/iommu/intel_dmar.h
+++ b/sys/x86/iommu/intel_dmar.h
@@ -74,9 +74,7 @@ RB_PROTOTYPE(dmar_gas_entries_tree, dmar_map_entry, rb_entry,
 #define	DMAR_MAP_ENTRY_TM	0x8000	/* Transient */
 
 struct dmar_ctx {
-	int bus;	/* pci bus/slot/func */
-	int slot;
-	int func;
+	uint16_t rid;	/* pci RID */
 	int domain;	/* DID */
 	int mgaw;	/* Real max address width */
 	int agaw;	/* Adjusted guest address width */
@@ -272,12 +270,11 @@ void ctx_free_pgtbl(struct dmar_ctx *ctx);
 
 struct dmar_ctx *dmar_instantiate_ctx(struct dmar_unit *dmar, device_t dev,
     bool rmrr);
-struct dmar_ctx *dmar_get_ctx(struct dmar_unit *dmar, device_t dev,
-    int bus, int slot, int func, bool id_mapped, bool rmrr_init);
+struct dmar_ctx *dmar_get_ctx(struct dmar_unit *dmar, device_t dev, 
+    uint16_t rid, bool id_mapped, bool rmrr_init);
 void dmar_free_ctx_locked(struct dmar_unit *dmar, struct dmar_ctx *ctx);
 void dmar_free_ctx(struct dmar_ctx *ctx);
-struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, int bus,
-    int slot, int func);
+struct dmar_ctx *dmar_find_ctx_locked(struct dmar_unit *dmar, uint16_t rid);
 void dmar_ctx_unload_entry(struct dmar_map_entry *entry, bool free);
 void dmar_ctx_unload(struct dmar_ctx *ctx,
     struct dmar_map_entries_tailq *entries, bool cansleep);
diff --git a/sys/x86/iommu/intel_drv.c b/sys/x86/iommu/intel_drv.c
index c7c6bf1..59e54e7 100644
--- a/sys/x86/iommu/intel_drv.c
+++ b/sys/x86/iommu/intel_drv.c
@@ -1006,7 +1006,9 @@ dmar_print_ctx(struct dmar_ctx *ctx, bool show_mappings)
 	db_printf(
 	    "  @%p pci%d:%d:%d dom %d mgaw %d agaw %d pglvl %d end %jx\n"
 	    "    refs %d flags %x pgobj %p map_ents %u loads %lu unloads %lu\n",
-	    ctx, ctx->bus, ctx->slot, ctx->func, ctx->domain, ctx->mgaw,
+	    ctx, pci_get_bus(ctx->ctx_tag.owner),
+	    pci_get_slot(ctx->ctx_tag.owner),
+	    pci_get_function(ctx->ctx_tag.owner), ctx->domain, ctx->mgaw,
 	    ctx->agaw, ctx->pglvl, (uintmax_t)ctx->end, ctx->refs,
 	    ctx->flags, ctx->pgtbl_obj, ctx->entries_cnt, ctx->loads,
 	    ctx->unloads);
@@ -1079,8 +1081,10 @@ DB_FUNC(dmar_ctx, db_dmar_print_ctx, db_show_table, CS_OWN, NULL)
 	for (i = 0; i < dmar_devcnt; i++) {
 		unit = device_get_softc(dmar_devs[i]);
 		LIST_FOREACH(ctx, &unit->contexts, link) {
-			if (domain == unit->segment && bus == ctx->bus &&
-			    device == ctx->slot && function == ctx->func) {
+			if (domain == unit->segment && 
+			    bus == pci_get_bus(ctx->ctx_tag.owner) &&
+			    device == pci_get_slot(ctx->ctx_tag.owner) && 
+			    function == pci_get_function(ctx->ctx_tag.owner)) {
 				dmar_print_ctx(ctx, show_mappings);
 				goto out;
 			}
diff --git a/sys/x86/iommu/intel_fault.c b/sys/x86/iommu/intel_fault.c
index 18f8fef..af18010 100644
--- a/sys/x86/iommu/intel_fault.c
+++ b/sys/x86/iommu/intel_fault.c
@@ -45,6 +45,8 @@ __FBSDID("$FreeBSD$");
 #include <contrib/dev/acpica/include/acpi.h>
 #include <contrib/dev/acpica/include/accommon.h>
 #include <dev/acpica/acpivar.h>
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
 #include <vm/vm_kern.h>
@@ -203,19 +205,28 @@ dmar_fault_task(void *arg, int pending __unused)
 		DMAR_FAULT_UNLOCK(unit);
 
 		sid = DMAR_FRCD2_SID(fault_rec[1]);
-		bus = (sid >> 8) & 0xf;
-		slot = (sid >> 3) & 0x1f;
-		func = sid & 0x7;
 		printf("DMAR%d: ", unit->unit);
 		DMAR_LOCK(unit);
-		ctx = dmar_find_ctx_locked(unit, bus, slot, func);
+		ctx = dmar_find_ctx_locked(unit, sid);
 		if (ctx == NULL) {
 			printf("<unknown dev>:");
+
+			/*
+			 * Note that the slot and function will not be correct
+			 * if ARI is in use, but without a ctx entry we have
+			 * no way of knowing whether ARI is in use or not.
+			 */
+			bus = PCI_RID2BUS(sid);
+			slot = PCI_RID2SLOT(sid);
+			func = PCI_RID2FUNC(sid);
 		} else {
 			ctx->flags |= DMAR_CTX_FAULTED;
 			ctx->last_fault_rec[0] = fault_rec[0];
 			ctx->last_fault_rec[1] = fault_rec[1];
 			device_print_prettyname(ctx->ctx_tag.owner);
+			bus = pci_get_bus(ctx->ctx_tag.owner);
+			slot = pci_get_slot(ctx->ctx_tag.owner);
+			func = pci_get_function(ctx->ctx_tag.owner);
 		}
 		DMAR_UNLOCK(unit);
 		printf(
diff --git a/sys/x86/iommu/intel_utils.c b/sys/x86/iommu/intel_utils.c
index 77e1cea..2450fcb 100644
--- a/sys/x86/iommu/intel_utils.c
+++ b/sys/x86/iommu/intel_utils.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/taskqueue.h>
 #include <sys/tree.h>
+#include <dev/pci/pcivar.h>
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
 #include <vm/vm_kern.h>
@@ -129,8 +130,10 @@ ctx_set_agaw(struct dmar_ctx *ctx, int mgaw)
 	}
 	device_printf(ctx->dmar->dev,
 	    "context request mgaw %d for pci%d:%d:%d:%d, "
-	    "no agaw found, sagaw %x\n", mgaw, ctx->dmar->segment, ctx->bus,
-	     ctx->slot, ctx->func, sagaw);
+	    "no agaw found, sagaw %x\n", mgaw, ctx->dmar->segment, 
+	    pci_get_bus(ctx->ctx_tag.owner),
+	    pci_get_slot(ctx->ctx_tag.owner),
+	    pci_get_function(ctx->ctx_tag.owner), sagaw);
 	return (EINVAL);
 }