For some buses, devices may have active resources assigned even though they

are not allocated by the device driver.  These resources should still appear
allocated from the system's perspective so that their assigned ranges are
not reused by other resource requests.  The PCI bus driver has used a hack
to effect this for a while now where it uses rman_set_device() to assign
devices to the PCI bus when they are first encountered and later assigns
them to the actual device when a driver allocates a BAR.  A few downsides of
this approach is that it results in somewhat confusing devinfo -r output as
well as not being very easily portable to other bus drivers.

This commit adds generic support for "reserved" resources to the resource
list API used by many bus drivers to manage the resources of child devices.
A resource may be reserved via resource_list_reserve().  This will allocate
the resource from the bus' parent without activating it.
resource_list_alloc() recognizes an attempt to allocate a reserved resource.
When this happens it activates the resource (if requested) and then returns
the reserved resource.  Similarly, when a reserved resource is released via
resource_list_release(), it is deactivated (if it is active) and the
resource is then marked reserved again, but is left allocated from the
bus' parent.  To completely remove a reserved resource, a bus driver may
use resource_list_unreserve().  A bus driver may use resource_list_busy()
to determine if a reserved resource is allocated by a child device or if
it can be unreserved.

The PCI bus driver has been changed to use this framework instead of
abusing rman_set_device() to keep track of reserved vs allocated resources.

Submitted by:	imp (an older version many moons ago)
MFC after:	1 month

3 files changed, 188 insertions, 64 deletions

diff --git a/sys/dev/pci/pci.c b/sys/dev/pci/pci.c
index b8a2dab..1bf5130 100644
--- a/sys/dev/pci/pci.c
+++ b/sys/dev/pci/pci.c
@@ -2451,7 +2451,7 @@ pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
 	 * driver for this device will later inherit this resource in
 	 * pci_alloc_resource().
 	 */
-	res = resource_list_alloc(rl, bus, dev, type, &reg, start, end, count,
+	res = resource_list_reserve(rl, bus, dev, type, &reg, start, end, count,
 	    prefetch ? RF_PREFETCHABLE : 0);
 	if (res == NULL) {
 		/*
@@ -2462,10 +2462,8 @@ pci_add_map(device_t bus, device_t dev, int reg, struct resource_list *rl,
 		 */
 		resource_list_delete(rl, type, reg);
 		start = 0;
-	} else {
+	} else
 		start = rman_get_start(res);
-		rman_set_device(res, bus);
-	}
 	pci_write_bar(dev, reg, start);
 	return (barlen);
 }
@@ -2504,14 +2502,12 @@ pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
 	} else {
 		rid = PCIR_BAR(0);
 		resource_list_add(rl, type, rid, 0x1f0, 0x1f7, 8);
-		r = resource_list_alloc(rl, bus, dev, type, &rid, 0x1f0, 0x1f7,
-		    8, 0);
-		rman_set_device(r, bus);
+		r = resource_list_reserve(rl, bus, dev, type, &rid, 0x1f0,
+		    0x1f7, 8, 0);
 		rid = PCIR_BAR(1);
 		resource_list_add(rl, type, rid, 0x3f6, 0x3f6, 1);
-		r = resource_list_alloc(rl, bus, dev, type, &rid, 0x3f6, 0x3f6,
-		    1, 0);
-		rman_set_device(r, bus);
+		r = resource_list_reserve(rl, bus, dev, type, &rid, 0x3f6,
+		    0x3f6, 1, 0);
 	}
 	if (progif & PCIP_STORAGE_IDE_MODESEC) {
 		pci_add_map(bus, dev, PCIR_BAR(2), rl, force,
@@ -2521,14 +2517,12 @@ pci_ata_maps(device_t bus, device_t dev, struct resource_list *rl, int force,
 	} else {
 		rid = PCIR_BAR(2);
 		resource_list_add(rl, type, rid, 0x170, 0x177, 8);
-		r = resource_list_alloc(rl, bus, dev, type, &rid, 0x170, 0x177,
-		    8, 0);
-		rman_set_device(r, bus);
+		r = resource_list_reserve(rl, bus, dev, type, &rid, 0x170,
+		    0x177, 8, 0);
 		rid = PCIR_BAR(3);
 		resource_list_add(rl, type, rid, 0x376, 0x376, 1);
-		r = resource_list_alloc(rl, bus, dev, type, &rid, 0x376, 0x376,
-		    1, 0);
-		rman_set_device(r, bus);
+		r = resource_list_reserve(rl, bus, dev, type, &rid, 0x376,
+		    0x376, 1, 0);
 	}
 	pci_add_map(bus, dev, PCIR_BAR(4), rl, force,
 	    prefetchmask & (1 << 4));
@@ -3564,7 +3558,7 @@ DB_SHOW_COMMAND(pciregs, db_pci_dump)
 #endif /* DDB */
 
 static struct resource *
-pci_alloc_map(device_t dev, device_t child, int type, int *rid,
+pci_reserve_map(device_t dev, device_t child, int type, int *rid,
     u_long start, u_long end, u_long count, u_int flags)
 {
 	struct pci_devinfo *dinfo = device_get_ivars(child);
@@ -3634,15 +3628,15 @@ pci_alloc_map(device_t dev, device_t child, int type, int *rid,
 		    count, *rid, type, start, end);
 		goto out;
 	}
-	rman_set_device(res, dev);
 	resource_list_add(rl, type, *rid, start, end, count);
 	rle = resource_list_find(rl, type, *rid);
 	if (rle == NULL)
-		panic("pci_alloc_map: unexpectedly can't find resource.");
+		panic("pci_reserve_map: unexpectedly can't find resource.");
 	rle->res = res;
 	rle->start = rman_get_start(res);
 	rle->end = rman_get_end(res);
 	rle->count = count;
+	rle->flags = RLE_RESERVED;
 	if (bootverbose)
 		device_printf(child,
 		    "Lazy allocation of %#lx bytes rid %#x type %d at %#lx\n",
@@ -3692,34 +3686,13 @@ pci_alloc_resource(device_t dev, device_t child, int type, int *rid,
 		break;
 	case SYS_RES_IOPORT:
 	case SYS_RES_MEMORY:
-		/* Allocate resources for this BAR if needed. */
+		/* Reserve resources for this BAR if needed. */
 		rle = resource_list_find(rl, type, *rid);
 		if (rle == NULL) {
-			res = pci_alloc_map(dev, child, type, rid, start, end,
+			res = pci_reserve_map(dev, child, type, rid, start, end,
 			    count, flags);
 			if (res == NULL)
 				return (NULL);
-			rle = resource_list_find(rl, type, *rid);
-		}
-
-		/*
-		 * If the resource belongs to the bus, then give it to
-		 * the child.  We need to activate it if requested
-		 * since the bus always allocates inactive resources.
-		 */
-		if (rle != NULL && rle->res != NULL &&
-		    rman_get_device(rle->res) == dev) {
-			if (bootverbose)
-				device_printf(child,
-			    "Reserved %#lx bytes for rid %#x type %d at %#lx\n",
-				    rman_get_size(rle->res), *rid, type,
-				    rman_get_start(rle->res));
-			rman_set_device(rle->res, child);
-			if ((flags & RF_ACTIVE) &&
-			    bus_activate_resource(child, type, *rid,
-			    rle->res) != 0)
-				return (NULL);
-			return (rle->res);
 		}
 	}
 	return (resource_list_alloc(rl, dev, child, type, rid,
@@ -3730,7 +3703,6 @@ int
 pci_release_resource(device_t dev, device_t child, int type, int rid,
     struct resource *r)
 {
-	int error;
 
 	if (device_get_parent(child) != dev)
 		return (BUS_RELEASE_RESOURCE(device_get_parent(dev), child,
@@ -3739,21 +3711,10 @@ pci_release_resource(device_t dev, device_t child, int type, int rid,
 	/*
 	 * For BARs we don't actually want to release the resource.
 	 * Instead, we deactivate the resource if needed and then give
-	 * ownership of the BAR back to the bus.
+	 * ownership of the BAR back to the bus.  This is handled for us
+	 * in resource_list_release() since we use resource_list_reserve()
+	 * for BARs.
 	 */
-	switch (type) {
-	case SYS_RES_IOPORT:
-	case SYS_RES_MEMORY:
-		if (rman_get_device(r) != child)
-			return (EINVAL);
-		if (rman_get_flags(r) & RF_ACTIVE) {
-			error = bus_deactivate_resource(child, type, rid, r);
-			if (error)
-				return (error);
-		}
-		rman_set_device(r, dev);
-		return (0);
-	}
 	return (bus_generic_rl_release_resource(dev, child, type, rid, r));
 }
 
@@ -3796,13 +3757,12 @@ pci_delete_resource(device_t dev, device_t child, int type, int rid)
 		return;
 
 	if (rle->res) {
-		if (rman_get_device(rle->res) != dev ||
-		    rman_get_flags(rle->res) & RF_ACTIVE) {
+		if (rman_get_flags(rle->res) & RF_ACTIVE ||
+		    resource_list_busy(rl, type, rid)) {
 			device_printf(dev, "delete_resource: "
 			    "Resource still owned by child, oops. "
 			    "(type=%d, rid=%d, addr=%lx)\n",
-			    rle->type, rle->rid,
-			    rman_get_start(rle->res));
+			    type, rid, rman_get_start(rle->res));
 			return;
 		}
 
@@ -3818,7 +3778,7 @@ pci_delete_resource(device_t dev, device_t child, int type, int rid)
 			break;
 		}
 #endif
-		bus_release_resource(dev, type, rid, rle->res);
+		resource_list_unreserve(rl, dev, child, type, rid);
 	}
 	resource_list_delete(rl, type, rid);
 }
diff --git a/sys/kern/subr_bus.c b/sys/kern/subr_bus.c
index eaec75b..8c2db32 100644
--- a/sys/kern/subr_bus.c
+++ b/sys/kern/subr_bus.c
@@ -2866,6 +2866,7 @@ resource_list_add(struct resource_list *rl, int type, int rid,
 		rle->type = type;
 		rle->rid = rid;
 		rle->res = NULL;
+		rle->flags = 0;
 	}
 
 	if (rle->res)
@@ -2878,6 +2879,31 @@ resource_list_add(struct resource_list *rl, int type, int rid,
 }
 
 /**
+ * @brief Determine if a resource entry is busy.
+ *
+ * Returns true if a resource entry is busy meaning that it has an
+ * associated resource that is not an unallocated "reserved" resource.
+ *
+ * @param rl		the resource list to search
+ * @param type		the resource entry type (e.g. SYS_RES_MEMORY)
+ * @param rid		the resource identifier
+ *
+ * @returns Non-zero if the entry is busy, zero otherwise.
+ */
+int
+resource_list_busy(struct resource_list *rl, int type, int rid)
+{
+	struct resource_list_entry *rle;
+
+	rle = resource_list_find(rl, type, rid);
+	if (rle == NULL || rle->res == NULL)
+		return (0);
+	if ((rle->flags & (RLE_RESERVED | RLE_ALLOCATED)) == RLE_RESERVED)
+		return (0);
+	return (1);
+}
+
+/**
  * @brief Find a resource entry by type and rid.
  *
  * @param rl		the resource list to search
@@ -2920,6 +2946,66 @@ resource_list_delete(struct resource_list *rl, int type, int rid)
 }
 
 /**
+ * @brief Allocate a reserved resource
+ *
+ * This can be used by busses to force the allocation of resources
+ * that are always active in the system even if they are not allocated
+ * by a driver (e.g. PCI BARs).  This function is usually called when
+ * adding a new child to the bus.  The resource is allocated from the
+ * parent bus when it is reserved.  The resource list entry is marked
+ * with RLE_RESERVED to note that it is a reserved resource.
+ *
+ * Subsequent attempts to allocate the resource with
+ * resource_list_alloc() will succeed the first time and will set
+ * RLE_ALLOCATED to note that it has been allocated.  When a reserved
+ * resource that has been allocated is released with
+ * resource_list_release() the resource RLE_ALLOCATED is cleared, but
+ * the actual resource remains allocated.  The resource can be released to
+ * the parent bus by calling resource_list_unreserve().
+ *
+ * @param rl		the resource list to allocate from
+ * @param bus		the parent device of @p child
+ * @param child		the device for which the resource is being reserved
+ * @param type		the type of resource to allocate
+ * @param rid		a pointer to the resource identifier
+ * @param start		hint at the start of the resource range - pass
+ *			@c 0UL for any start address
+ * @param end		hint at the end of the resource range - pass
+ *			@c ~0UL for any end address
+ * @param count		hint at the size of range required - pass @c 1
+ *			for any size
+ * @param flags		any extra flags to control the resource
+ *			allocation - see @c RF_XXX flags in
+ *			<sys/rman.h> for details
+ * 
+ * @returns		the resource which was allocated or @c NULL if no
+ *			resource could be allocated
+ */
+struct resource *
+resource_list_reserve(struct resource_list *rl, device_t bus, device_t child,
+    int type, int *rid, u_long start, u_long end, u_long count, u_int flags)
+{
+	struct resource_list_entry *rle = NULL;
+	int passthrough = (device_get_parent(child) != bus);
+	struct resource *r;
+
+	if (passthrough)
+		panic(
+    "resource_list_reserve() should only be called for direct children");
+	if (flags & RF_ACTIVE)
+		panic(
+    "resource_list_reserve() should only reserve inactive resources");
+
+	r = resource_list_alloc(rl, bus, child, type, rid, start, end, count,
+	    flags);
+	if (r != NULL) {
+		rle = resource_list_find(rl, type, *rid);
+		rle->flags |= RLE_RESERVED;
+	}
+	return (r);
+}
+
+/**
  * @brief Helper function for implementing BUS_ALLOC_RESOURCE()
  *
  * Implement BUS_ALLOC_RESOURCE() by looking up a resource from the list
@@ -2970,8 +3056,19 @@ resource_list_alloc(struct resource_list *rl, device_t bus, device_t child,
 	if (!rle)
 		return (NULL);		/* no resource of that type/rid */
 
-	if (rle->res)
+	if (rle->res) {
+		if (rle->flags & RLE_RESERVED) {
+			if (rle->flags & RLE_ALLOCATED)
+				return (NULL);
+			else if ((flags & RF_ACTIVE) &&
+			    bus_activate_resource(child, type, *rid,
+			    rle->res) != 0)
+				return (NULL);
+			else
+				return (rle->res);
+		}
 		panic("resource_list_alloc: resource entry is busy");
+	}
 
 	if (isdefault) {
 		start = rle->start;
@@ -3003,7 +3100,7 @@ resource_list_alloc(struct resource_list *rl, device_t bus, device_t child,
  * @param rl		the resource list which was allocated from
  * @param bus		the parent device of @p child
  * @param child		the device which is requesting a release
- * @param type		the type of resource to allocate
+ * @param type		the type of resource to release
  * @param rid		the resource identifier
  * @param res		the resource to release
  * 
@@ -3030,6 +3127,19 @@ resource_list_release(struct resource_list *rl, device_t bus, device_t child,
 		panic("resource_list_release: can't find resource");
 	if (!rle->res)
 		panic("resource_list_release: resource entry is not busy");
+	if (rle->flags & RLE_RESERVED) {
+		if (rle->flags & RLE_ALLOCATED) {
+			if (rman_get_flags(res) & RF_ACTIVE) {
+				error = bus_deactivate_resource(child, type,
+				    rid, res);
+				if (error)
+					return (error);
+			}
+			rle->flags &= ~RLE_ALLOCATED;
+			return (0);
+		}
+		return (EINVAL);
+	}
 
 	error = BUS_RELEASE_RESOURCE(device_get_parent(bus), child,
 	    type, rid, res);
@@ -3041,6 +3151,45 @@ resource_list_release(struct resource_list *rl, device_t bus, device_t child,
 }
 
 /**
+ * @brief Fully release a reserved resource
+ *
+ * Fully releases a resouce reserved via resource_list_reserve().
+ *
+ * @param rl		the resource list which was allocated from
+ * @param bus		the parent device of @p child
+ * @param child		the device whose reserved resource is being released
+ * @param type		the type of resource to release
+ * @param rid		the resource identifier
+ * @param res		the resource to release
+ * 
+ * @retval 0		success
+ * @retval non-zero	a standard unix error code indicating what
+ *			error condition prevented the operation
+ */
+int
+resource_list_unreserve(struct resource_list *rl, device_t bus, device_t child,
+    int type, int rid)
+{
+	struct resource_list_entry *rle = NULL;
+	int passthrough = (device_get_parent(child) != bus);
+
+	if (passthrough)
+		panic(
+    "resource_list_unreserve() should only be called for direct children");
+
+	rle = resource_list_find(rl, type, rid);
+
+	if (!rle)
+		panic("resource_list_unreserve: can't find resource");
+	if (!(rle->flags & RLE_RESERVED))
+		return (EINVAL);
+	if (rle->flags & RLE_ALLOCATED)
+		return (EBUSY);
+	rle->flags &= ~RLE_RESERVED;
+	return (resource_list_release(rl, bus, child, type, rid, rle->res));
+}
+
+/**
  * @brief Print a description of resources in a resource list
  *
  * Print all resources of a specified type, for use in BUS_PRINT_CHILD().
diff --git a/sys/sys/bus.h b/sys/sys/bus.h
index 92f4238..402d3cc 100644
--- a/sys/sys/bus.h
+++ b/sys/sys/bus.h
@@ -231,6 +231,7 @@ struct resource_list_entry {
 	STAILQ_ENTRY(resource_list_entry) link;
 	int	type;			/**< @brief type argument to alloc_resource */
 	int	rid;			/**< @brief resource identifier */
+	int	flags;			/**< @brief resource flags */
 	struct	resource *res;		/**< @brief the real resource when allocated */
 	u_long	start;			/**< @brief start of resource range */
 	u_long	end;			/**< @brief end of resource range */
@@ -238,6 +239,9 @@ struct resource_list_entry {
 };
 STAILQ_HEAD(resource_list, resource_list_entry);
 
+#define	RLE_RESERVED		0x0001	/* Reserved by the parent bus. */
+#define	RLE_ALLOCATED		0x0002	/* Reserved resource is allocated. */
+
 void	resource_list_init(struct resource_list *rl);
 void	resource_list_free(struct resource_list *rl);
 struct resource_list_entry *
@@ -247,6 +251,8 @@ struct resource_list_entry *
 int	resource_list_add_next(struct resource_list *rl,
 			  int type,
 			  u_long start, u_long end, u_long count);
+int	resource_list_busy(struct resource_list *rl,
+			   int type, int rid);
 struct resource_list_entry*
 	resource_list_find(struct resource_list *rl,
 			   int type, int rid);
@@ -261,6 +267,15 @@ struct resource *
 int	resource_list_release(struct resource_list *rl,
 			      device_t bus, device_t child,
 			      int type, int rid, struct resource *res);
+struct resource *
+	resource_list_reserve(struct resource_list *rl,
+			      device_t bus, device_t child,
+			      int type, int *rid,
+			      u_long start, u_long end,
+			      u_long count, u_int flags);
+int	resource_list_unreserve(struct resource_list *rl,
+				device_t bus, device_t child,
+				int type, int rid);
 void	resource_list_purge(struct resource_list *rl);
 int	resource_list_print_type(struct resource_list *rl,
 				 const char *name, int type,