Support interrupts binding in GICv3 and ITS

- Add MOVI command and routine for the LPI migration
- Allow to search for the ITS device descriptor using
  not only devID but also LPI number.
- Bind SPIs in the Distributor
- Don't bind its_dev to collection. Keep track of the collection
  IDs for each LPI.

Reviewed by:   wma
Obtained from: Semihalf
Sponsored by:  Cavium
Differential Revision: https://reviews.freebsd.org/D5231

3 files changed, 167 insertions, 35 deletions

diff --git a/sys/arm64/arm64/gic_v3.c b/sys/arm64/arm64/gic_v3.c
index 9c24b39..d8434cf 100644
--- a/sys/arm64/arm64/gic_v3.c
+++ b/sys/arm64/arm64/gic_v3.c
@@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$");
 #include "gic_v3_var.h"
 
 /* Device and PIC methods */
+static int gic_v3_bind(device_t, u_int, u_int);
 static void gic_v3_dispatch(device_t, struct trapframe *);
 static void gic_v3_eoi(device_t, u_int);
 static void gic_v3_mask_irq(device_t, u_int);
@@ -72,6 +73,7 @@ static device_method_t gic_v3_methods[] = {
 	DEVMETHOD(device_detach,	gic_v3_detach),
 
 	/* PIC interface */
+	DEVMETHOD(pic_bind,		gic_v3_bind),
 	DEVMETHOD(pic_dispatch,		gic_v3_dispatch),
 	DEVMETHOD(pic_eoi,		gic_v3_eoi),
 	DEVMETHOD(pic_mask,		gic_v3_mask_irq),
@@ -244,6 +246,28 @@ gic_v3_detach(device_t dev)
 /*
  * PIC interface.
  */
+
+static int
+gic_v3_bind(device_t dev, u_int irq, u_int cpuid)
+{
+	uint64_t aff;
+	struct gic_v3_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (irq <= GIC_LAST_PPI) {
+		/* Can't bind PPI to another CPU but it's not an error */
+		return (0);
+	} else if (irq >= GIC_FIRST_SPI && irq <= GIC_LAST_SPI) {
+		aff = CPU_AFFINITY(cpuid);
+		gic_d_write(sc, 4, GICD_IROUTER(irq), aff);
+		return (0);
+	} else if (irq >= GIC_FIRST_LPI)
+		return (lpi_migrate(dev, irq, cpuid));
+
+	return (EINVAL);
+}
+
 static void
 gic_v3_dispatch(device_t dev, struct trapframe *frame)
 {
diff --git a/sys/arm64/arm64/gic_v3_its.c b/sys/arm64/arm64/gic_v3_its.c
index 8724683..7a547c1 100644
--- a/sys/arm64/arm64/gic_v3_its.c
+++ b/sys/arm64/arm64/gic_v3_its.c
@@ -92,9 +92,13 @@ static void its_free_tables(struct gic_v3_its_softc *);
 static void its_init_commandq(struct gic_v3_its_softc *);
 static void its_init_cpu_collection(struct gic_v3_its_softc *);
 static uint32_t its_get_devid(device_t);
+static struct its_dev * its_device_find_locked(struct gic_v3_its_softc *,
+    device_t, uint32_t);
 
 static int its_cmd_send(struct gic_v3_its_softc *, struct its_cmd_desc *);
 
+static void its_cmd_movi(struct gic_v3_its_softc *, struct its_dev *,
+    struct its_col *, uint32_t);
 static void its_cmd_mapc(struct gic_v3_its_softc *, struct its_col *, uint8_t);
 static void its_cmd_mapvi(struct gic_v3_its_softc *, struct its_dev *, uint32_t,
     uint32_t);
@@ -846,18 +850,28 @@ static int
 lpi_alloc_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic,
     u_int nvecs)
 {
+	u_int *col_ids;
 	int fclr; /* First cleared bit */
 	uint8_t *bitmap;
 	size_t nb, i;
 
+	col_ids = malloc(sizeof(*col_ids) * nvecs, M_GIC_V3_ITS,
+	    (M_NOWAIT | M_ZERO));
+	if (col_ids == NULL)
+		return (ENOMEM);
+
+	mtx_lock_spin(&sc->its_dev_lock);
 	bitmap = (uint8_t *)sc->its_lpi_bitmap;
 
 	fclr = 0;
 retry:
 	/* Check other bits - sloooow */
 	for (i = 0, nb = fclr; i < nvecs; i++, nb++) {
-		if (nb > sc->its_lpi_maxid)
+		if (nb > sc->its_lpi_maxid) {
+			mtx_unlock_spin(&sc->its_dev_lock);
+			free(col_ids, M_GIC_V3_ITS);
 			return (EINVAL);
+		}
 
 		if (isset(bitmap, nb)) {
 			/* To little free bits in this area. Move on. */
@@ -870,6 +884,15 @@ retry:
 	lpic->lpi_base = fclr + GIC_FIRST_LPI;
 	lpic->lpi_num = nvecs;
 	lpic->lpi_free = lpic->lpi_num;
+	lpic->lpi_col_ids = col_ids;
+	for (i = 0; i < lpic->lpi_num; i++) {
+		/*
+		 * Initially all interrupts go to CPU0 but can be moved
+		 * to another CPU by bus_bind_intr() or interrupts shuffling.
+		 */
+		lpic->lpi_col_ids[i] = 0;
+	}
+	mtx_unlock_spin(&sc->its_dev_lock);
 
 	return (0);
 }
@@ -885,6 +908,7 @@ lpi_free_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic)
 	KASSERT((lpic->lpi_free == lpic->lpi_num),
 	    ("Trying to free LPI chunk that is still in use.\n"));
 
+	mtx_lock_spin(&sc->its_dev_lock);
 	/* First bit of this chunk in a global bitmap */
 	start = lpic->lpi_base - GIC_FIRST_LPI;
 	/* and last bit of this chunk... */
@@ -892,6 +916,10 @@ lpi_free_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic)
 
 	/* Finally free this chunk */
 	bit_nclear(bitmap, start, end);
+	mtx_unlock_spin(&sc->its_dev_lock);
+
+	free(lpic->lpi_col_ids, M_GIC_V3_ITS);
+	lpic->lpi_col_ids = NULL;
 }
 
 static void
@@ -953,6 +981,32 @@ lpi_xmask_irq(device_t parent, uint32_t irq, boolean_t unmask)
 	    (unmask == TRUE) ? "unmask" : "mask", irq);
 }
 
+int
+lpi_migrate(device_t parent, uint32_t irq, u_int cpuid)
+{
+	struct gic_v3_its_softc *sc;
+	struct its_dev *its_dev;
+	struct its_col *col;
+
+	sc = its_sc;
+	mtx_lock_spin(&sc->its_dev_lock);
+	its_dev = its_device_find_locked(sc, NULL, irq);
+	mtx_unlock_spin(&sc->its_dev_lock);
+	if (its_dev == NULL) {
+		/* Cannot migrate not configured LPI */
+		return (ENXIO);
+	}
+
+	/* Find local device's interrupt identifier */
+	irq = irq - its_dev->lpis.lpi_base;
+	/* Move interrupt to another collection */
+	col = sc->its_cols[cpuid];
+	its_cmd_movi(sc, its_dev, col, irq);
+	its_dev->lpis.lpi_col_ids[irq] = cpuid;
+
+	return (0);
+}
+
 void
 lpi_unmask_irq(device_t parent, uint32_t irq)
 {
@@ -1053,6 +1107,20 @@ cmd_fix_endian(struct its_cmd *cmd)
 }
 
 static void
+its_cmd_movi(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
+    struct its_col *col, uint32_t id)
+{
+	struct its_cmd_desc desc;
+
+	desc.cmd_type = ITS_CMD_MOVI;
+	desc.cmd_desc_movi.its_dev = its_dev;
+	desc.cmd_desc_movi.col = col;
+	desc.cmd_desc_movi.id = id;
+
+	its_cmd_send(sc, &desc);
+}
+
+static void
 its_cmd_mapc(struct gic_v3_its_softc *sc, struct its_col *col, uint8_t valid)
 {
 	struct its_cmd_desc desc;
@@ -1073,9 +1141,15 @@ its_cmd_mapvi(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
     uint32_t id, uint32_t pid)
 {
 	struct its_cmd_desc desc;
+	struct its_col *col;
+	u_int col_id;
+
+	col_id = its_dev->lpis.lpi_col_ids[id];
+	col = sc->its_cols[col_id];
 
 	desc.cmd_type = ITS_CMD_MAPVI;
 	desc.cmd_desc_mapvi.its_dev = its_dev;
+	desc.cmd_desc_mapvi.col = col;
 	desc.cmd_desc_mapvi.id = id;
 	desc.cmd_desc_mapvi.pid = pid;
 
@@ -1083,14 +1157,23 @@ its_cmd_mapvi(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
 }
 
 static void __unused
-its_cmd_mapi(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
-    uint32_t lpinum)
+its_cmd_mapi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t pid)
 {
 	struct its_cmd_desc desc;
+	struct its_col *col;
+	u_int col_id;
+	uint32_t id;
+
+	KASSERT(pid >= its_dev->lpis.lpi_base,
+	    ("%s: invalid pid: %d for the ITS device", __func__, pid));
+	id = pid - its_dev->lpis.lpi_base;
+	col_id = its_dev->lpis.lpi_col_ids[id];
+	col = sc->its_cols[col_id];
 
 	desc.cmd_type = ITS_CMD_MAPI;
 	desc.cmd_desc_mapi.its_dev = its_dev;
-	desc.cmd_desc_mapi.lpinum = lpinum;
+	desc.cmd_desc_mapi.col = col;
+	desc.cmd_desc_mapi.pid = pid;
 
 	its_cmd_send(sc, &desc);
 }
@@ -1109,14 +1192,23 @@ its_cmd_mapd(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
 }
 
 static void
-its_cmd_inv(struct gic_v3_its_softc *sc, struct its_dev *its_dev,
-    uint32_t lpinum)
+its_cmd_inv(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t pid)
 {
 	struct its_cmd_desc desc;
+	struct its_col *col;
+	u_int col_id;
+	uint32_t id;
+
+	KASSERT(pid >= its_dev->lpis.lpi_base,
+	    ("%s: invalid pid: %d for the ITS device", __func__, pid));
+	id = pid - its_dev->lpis.lpi_base;
+	col_id = its_dev->lpis.lpi_col_ids[id];
+	col = sc->its_cols[col_id];
 
 	desc.cmd_type = ITS_CMD_INV;
-	desc.cmd_desc_inv.lpinum = lpinum - its_dev->lpis.lpi_base;
+	desc.cmd_desc_inv.pid = pid - its_dev->lpis.lpi_base;
 	desc.cmd_desc_inv.its_dev = its_dev;
+	desc.cmd_desc_inv.col = col;
 
 	its_cmd_send(sc, &desc);
 }
@@ -1216,13 +1308,19 @@ its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
 	target = ITS_TARGET_NONE;
 
 	switch (cmd_type) {
+	case ITS_CMD_MOVI:	/* Move interrupt ID to another collection */
+		target = desc->cmd_desc_movi.col->col_target;
+		cmd_format_command(cmd, ITS_CMD_MOVI);
+		cmd_format_id(cmd, desc->cmd_desc_movi.id);
+		cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
+		cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
+		break;
 	case ITS_CMD_SYNC:	/* Wait for previous commands completion */
 		target = desc->cmd_desc_sync.col->col_target;
 		cmd_format_command(cmd, ITS_CMD_SYNC);
 		cmd_format_target(cmd, target);
 		break;
 	case ITS_CMD_MAPD:	/* Assign ITT to device */
-		target = desc->cmd_desc_mapd.its_dev->col->col_target;
 		cmd_format_command(cmd, ITS_CMD_MAPD);
 		cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
 		/*
@@ -1249,25 +1347,25 @@ its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
 		cmd_format_target(cmd, target);
 		break;
 	case ITS_CMD_MAPVI:
-		target = desc->cmd_desc_mapvi.its_dev->col->col_target;
+		target = desc->cmd_desc_mapvi.col->col_target;
 		cmd_format_command(cmd, ITS_CMD_MAPVI);
 		cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
 		cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
 		cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
-		cmd_format_col(cmd, desc->cmd_desc_mapvi.its_dev->col->col_id);
+		cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
 		break;
 	case ITS_CMD_MAPI:
-		target = desc->cmd_desc_mapi.its_dev->col->col_target;
+		target = desc->cmd_desc_mapi.col->col_target;
 		cmd_format_command(cmd, ITS_CMD_MAPI);
 		cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
-		cmd_format_id(cmd, desc->cmd_desc_mapi.lpinum);
-		cmd_format_col(cmd, desc->cmd_desc_mapi.its_dev->col->col_id);
+		cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
+		cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
 		break;
 	case ITS_CMD_INV:
-		target = desc->cmd_desc_inv.its_dev->col->col_target;
+		target = desc->cmd_desc_inv.col->col_target;
 		cmd_format_command(cmd, ITS_CMD_INV);
 		cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
-		cmd_format_id(cmd, desc->cmd_desc_inv.lpinum);
+		cmd_format_id(cmd, desc->cmd_desc_inv.pid);
 		break;
 	case ITS_CMD_INVALL:
 		cmd_format_command(cmd, ITS_CMD_INVALL);
@@ -1367,16 +1465,28 @@ end:
 	return (0);
 }
 
+/* Find ITS device descriptor by pci_dev or irq number */
 static struct its_dev *
-its_device_find_locked(struct gic_v3_its_softc *sc, device_t pci_dev)
+its_device_find_locked(struct gic_v3_its_softc *sc, device_t pci_dev,
+    uint32_t irq)
 {
 	struct its_dev *its_dev;
+	struct lpi_chunk *lpis;
 
 	mtx_assert(&sc->its_dev_lock, MA_OWNED);
+	KASSERT((pci_dev == NULL || irq == 0),
+	    ("%s: Can't search by both pci_dev and irq number", __func__));
 	/* Find existing device if any */
 	TAILQ_FOREACH(its_dev, &sc->its_dev_list, entry) {
-		if (its_dev->pci_dev == pci_dev)
-			return (its_dev);
+		if (pci_dev != NULL) {
+			if (its_dev->pci_dev == pci_dev)
+				return (its_dev);
+		} else if (irq != 0) {
+			lpis = &its_dev->lpis;
+			if ((irq >= lpis->lpi_base) &&
+			    (irq < (lpis->lpi_base + lpis->lpi_num)))
+				return (its_dev);
+		}
 	}
 
 	return (NULL);
@@ -1389,13 +1499,12 @@ its_device_alloc(struct gic_v3_its_softc *sc, device_t pci_dev,
 	struct its_dev *newdev;
 	uint64_t typer;
 	uint32_t devid;
-	u_int cpuid;
 	size_t esize;
 	int err;
 
 	mtx_lock_spin(&sc->its_dev_lock);
 	/* Find existing device if any */
-	newdev = its_device_find_locked(sc, pci_dev);
+	newdev = its_device_find_locked(sc, pci_dev, 0);
 	mtx_unlock_spin(&sc->its_dev_lock);
 	if (newdev != NULL)
 		return (newdev);
@@ -1410,9 +1519,7 @@ its_device_alloc(struct gic_v3_its_softc *sc, device_t pci_dev,
 	newdev->pci_dev = pci_dev;
 	newdev->devid = devid;
 
-	mtx_lock_spin(&sc->its_dev_lock);
 	err = lpi_alloc_chunk(sc, &newdev->lpis, nvecs);
-	mtx_unlock_spin(&sc->its_dev_lock);
 	if (err != 0) {
 		free(newdev, M_GIC_V3_ITS);
 		return (NULL);
@@ -1429,20 +1536,11 @@ its_device_alloc(struct gic_v3_its_softc *sc, device_t pci_dev,
 	    roundup2(roundup2(nvecs, 2) * esize, 0x100), M_GIC_V3_ITS,
 	    (M_NOWAIT | M_ZERO), 0, ~0UL, 0x100, 0);
 	if (newdev->itt == 0) {
-		mtx_lock_spin(&sc->its_dev_lock);
 		lpi_free_chunk(sc, &newdev->lpis);
-		mtx_unlock_spin(&sc->its_dev_lock);
 		free(newdev, M_GIC_V3_ITS);
 		return (NULL);
 	}
 
-	/*
-	 * Initially all interrupts go to CPU0 but can be moved
-	 * to another CPU by bus_bind_intr() or interrupts shuffling.
-	 */
-	cpuid = 0;
-	newdev->col = sc->its_cols[cpuid];
-
 	mtx_lock_spin(&sc->its_dev_lock);
 	TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry);
 	mtx_unlock_spin(&sc->its_dev_lock);
diff --git a/sys/arm64/arm64/gic_v3_var.h b/sys/arm64/arm64/gic_v3_var.h
index 04fa67b..b3c0e52 100644
--- a/sys/arm64/arm64/gic_v3_var.h
+++ b/sys/arm64/arm64/gic_v3_var.h
@@ -96,6 +96,7 @@ struct lpi_chunk {
 	u_int	lpi_base;
 	u_int	lpi_num;
 	u_int	lpi_free;	/* First free LPI in set */
+	u_int	*lpi_col_ids;
 };
 
 /* ITS device */
@@ -109,8 +110,6 @@ struct its_dev {
 	struct lpi_chunk	lpis;
 	/* Virtual address of ITT */
 	vm_offset_t		itt;
-	/* Interrupt collection */
-	struct its_col *	col;
 };
 TAILQ_HEAD(its_dev_list, its_dev);
 
@@ -133,6 +132,7 @@ struct its_cmd {
 };
 
 /* ITS commands encoding */
+#define	ITS_CMD_MOVI		(0x01)
 #define	ITS_CMD_SYNC		(0x05)
 #define	ITS_CMD_MAPD		(0x08)
 #define	ITS_CMD_MAPC		(0x09)
@@ -172,6 +172,12 @@ struct its_cmd_desc {
 
 	union {
 		struct {
+			struct its_dev *its_dev;
+			struct its_col *col;
+			uint32_t id;
+		} cmd_desc_movi;
+
+		struct {
 			struct its_col *col;
 		} cmd_desc_sync;
 
@@ -182,13 +188,15 @@ struct its_cmd_desc {
 
 		struct {
 			struct its_dev *its_dev;
+			struct its_col *col;
 			uint32_t pid;
 			uint32_t id;
 		} cmd_desc_mapvi;
 
 		struct {
 			struct its_dev *its_dev;
-			uint32_t lpinum;
+			struct its_col *col;
+			uint32_t pid;
 		} cmd_desc_mapi;
 
 		struct {
@@ -198,7 +206,8 @@ struct its_cmd_desc {
 
 		struct {
 			struct its_dev *its_dev;
-			uint32_t lpinum;
+			struct its_col *col;
+			uint32_t pid;
 		} cmd_desc_inv;
 
 		struct {
@@ -257,6 +266,7 @@ int gic_v3_its_map_msi(device_t, device_t, int, uint64_t *, uint32_t *);
 
 int its_init_cpu(struct gic_v3_its_softc *);
 
+int lpi_migrate(device_t, uint32_t, u_int);
 void lpi_unmask_irq(device_t, uint32_t);
 void lpi_mask_irq(device_t, uint32_t);
 /*