Add SIM refcounting. This is slightly different from what DragonFly

does - in DragonFly, it's cam_sim_release() what actually frees the
SIM; cam_sim_free does nothing more than calling cam_sim_release().
Here, we drain in cam_sim_free, waiting for refcount to drop to zero.
We cannot do the same think DragonFly does, because after cam_sim_free
returns, client would destroy the sim->mtx, and CAM would trip over
an initialized mutex.

Reviewed by:	scottl
Approved by:	rwatson (mentor)
Sponsored by:	FreeBSD Foundation

3 files changed, 34 insertions, 0 deletions

diff --git a/sys/cam/cam_sim.c b/sys/cam/cam_sim.c
index cfc2044..d7fcaad 100644
--- a/sys/cam/cam_sim.c
+++ b/sys/cam/cam_sim.c
@@ -84,6 +84,7 @@ cam_sim_alloc(sim_action_func sim_action, sim_poll_func sim_poll,
 	sim->max_tagged_dev_openings = max_tagged_dev_transactions;
 	sim->max_dev_openings = max_dev_transactions;
 	sim->flags = 0;
+	sim->refcount = 1;
 	sim->devq = queue;
 	sim->mtx = mtx;
 	if (mtx == &Giant) {
@@ -103,12 +104,40 @@ cam_sim_alloc(sim_action_func sim_action, sim_poll_func sim_poll,
 void
 cam_sim_free(struct cam_sim *sim, int free_devq)
 {
+	int error;
+
+	sim->refcount--;
+	if (sim->refcount > 0) {
+		error = msleep(sim, sim->mtx, PRIBIO, "simfree", 0);
+		KASSERT(error == 0, ("invalid error value for msleep(9)"));
+	}
+
+	KASSERT(sim->refcount == 0, ("sim->refcount == 0"));
+
 	if (free_devq)
 		cam_simq_free(sim->devq);
 	free(sim, M_CAMSIM);
 }
 
 void
+cam_sim_release(struct cam_sim *sim)
+{
+	KASSERT(sim->refcount >= 1, ("sim->refcount >= 1"));
+
+	sim->refcount--;
+	if (sim->refcount <= 1)
+		wakeup(sim);
+}
+
+void
+cam_sim_hold(struct cam_sim *sim)
+{
+	KASSERT(sim->refcount >= 1, ("sim->refcount >= 1"));
+
+	sim->refcount++;
+}
+
+void
 cam_sim_set_path(struct cam_sim *sim, u_int32_t path_id)
 {
 	sim->path_id = path_id;
diff --git a/sys/cam/cam_sim.h b/sys/cam/cam_sim.h
index c2d3c5f..f37c5d5 100644
--- a/sys/cam/cam_sim.h
+++ b/sys/cam/cam_sim.h
@@ -61,6 +61,8 @@ struct cam_sim *  cam_sim_alloc(sim_action_func sim_action,
 				int max_tagged_dev_transactions,
 				struct cam_devq *queue);
 void		  cam_sim_free(struct cam_sim *sim, int free_devq);
+void		  cam_sim_hold(struct cam_sim *sim);
+void		  cam_sim_release(struct cam_sim *sim);
 
 /* Optional sim attributes may be set with these. */
 void	cam_sim_set_path(struct cam_sim *sim, u_int32_t path_id);
@@ -105,6 +107,7 @@ struct cam_sim {
 #define CAM_SIM_ON_DONEQ		0x04
 	struct callout		callout;
 	struct cam_devq 	*devq;	/* Device Queue to use for this SIM */
+	int			refcount; /* References to the SIM. */
 
 	/* "Pool" of inactive ccbs managed by xpt_alloc_ccb and xpt_free_ccb */
 	SLIST_HEAD(,ccb_hdr)	ccb_freeq;
diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index 60bfb99..cf3fa2c 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -4304,6 +4304,7 @@ xpt_bus_register(struct cam_sim *sim, device_t parent, u_int32_t bus)
 
 	TAILQ_INIT(&new_bus->et_entries);
 	new_bus->path_id = sim->path_id;
+	cam_sim_hold(sim);
 	new_bus->sim = sim;
 	timevalclear(&new_bus->last_reset);
 	new_bus->flags = 0;
@@ -4846,6 +4847,7 @@ xpt_release_bus(struct cam_eb *bus)
 		TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
 		xsoftc.bus_generation++;
 		mtx_unlock(&xsoftc.xpt_topo_lock);
+		cam_sim_release(bus->sim);
 		free(bus, M_CAMXPT);
 	}
 }