MFC r312603: Add initial support for CTL module unloading.

It is only a first step and not perfect, but better then nothing.
The main blocker is CAM target frontend, that can not be unloaded,
since CAM does not have mechanism to unregister periph driver now.

13 files changed, 264 insertions, 196 deletions

diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c
index 0ef7f6c..1b3a57f 100644
--- a/sys/cam/ctl/ctl.c
+++ b/sys/cam/ctl/ctl.c
@@ -426,7 +426,7 @@ static void ctl_isc_event_handler(ctl_ha_channel chanel, ctl_ha_event event,
 static void ctl_copy_sense_data(union ctl_ha_msg *src, union ctl_io *dest);
 static void ctl_copy_sense_data_back(union ctl_io *src, union ctl_ha_msg *dest);
 static int ctl_init(void);
-void ctl_shutdown(void);
+static int ctl_shutdown(void);
 static int ctl_open(struct cdev *dev, int flags, int fmt, struct thread *td);
 static int ctl_close(struct cdev *dev, int flags, int fmt, struct thread *td);
 static void ctl_serialize_other_sc_cmd(struct ctl_scsiio *ctsio);
@@ -522,6 +522,8 @@ static const struct ctl_cmd_entry *
     ctl_validate_command(struct ctl_scsiio *ctsio);
 static int ctl_cmd_applicable(uint8_t lun_type,
     const struct ctl_cmd_entry *entry);
+static int ctl_ha_init(void);
+static int ctl_ha_shutdown(void);
 
 static uint64_t ctl_get_prkey(struct ctl_lun *lun, uint32_t residx);
 static void ctl_clr_prkey(struct ctl_lun *lun, uint32_t residx);
@@ -563,6 +565,49 @@ MODULE_VERSION(ctl, 1);
 static struct ctl_frontend ha_frontend =
 {
 	.name = "ha",
+	.init = ctl_ha_init,
+	.shutdown = ctl_ha_shutdown,
+};
+
+static int
+ctl_ha_init(void)
+{
+	struct ctl_softc *softc = control_softc;
+
+	if (ctl_pool_create(softc, "othersc", CTL_POOL_ENTRIES_OTHER_SC,
+	                    &softc->othersc_pool) != 0)
+		return (ENOMEM);
+	if (ctl_ha_msg_init(softc) != CTL_HA_STATUS_SUCCESS) {
+		ctl_pool_free(softc->othersc_pool);
+		return (EIO);
+	}
+	if (ctl_ha_msg_register(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
+	    != CTL_HA_STATUS_SUCCESS) {
+		ctl_ha_msg_destroy(softc);
+		ctl_pool_free(softc->othersc_pool);
+		return (EIO);
+	}
+	return (0);
+};
+
+static int
+ctl_ha_shutdown(void)
+{
+	struct ctl_softc *softc = control_softc;
+	struct ctl_port *port;
+
+	ctl_ha_msg_shutdown(softc);
+	if (ctl_ha_msg_deregister(CTL_HA_CHAN_CTL) != CTL_HA_STATUS_SUCCESS)
+		return (EIO);
+	if (ctl_ha_msg_destroy(softc) != CTL_HA_STATUS_SUCCESS)
+		return (EIO);
+	ctl_pool_free(softc->othersc_pool);
+	while ((port = STAILQ_FIRST(&ha_frontend.port_list)) != NULL) {
+		ctl_port_deregister(port);
+		free(port->port_name, M_CTL);
+		free(port, M_CTL);
+	}
+	return (0);
 };
 
 static void
@@ -1784,7 +1829,6 @@ ctl_init(void)
 {
 	struct make_dev_args args;
 	struct ctl_softc *softc;
-	void *other_pool;
 	int i, error;
 
 	softc = control_softc = malloc(sizeof(*control_softc), M_DEVBUF,
@@ -1859,15 +1903,6 @@ ctl_init(void)
 	STAILQ_INIT(&softc->be_list);
 	ctl_tpc_init(softc);
 
-	if (ctl_pool_create(softc, "othersc", CTL_POOL_ENTRIES_OTHER_SC,
-	                    &other_pool) != 0)
-	{
-		printf("ctl: can't allocate %d entry other SC pool, "
-		       "exiting\n", CTL_POOL_ENTRIES_OTHER_SC);
-		return (ENOMEM);
-	}
-	softc->othersc_pool = other_pool;
-
 	if (worker_threads <= 0)
 		worker_threads = max(1, mp_ncpus / 4);
 	if (worker_threads > CTL_MAX_THREADS)
@@ -1887,22 +1922,19 @@ ctl_init(void)
 		    &softc->ctl_proc, &thr->thread, 0, 0, "ctl", "work%d", i);
 		if (error != 0) {
 			printf("error creating CTL work thread!\n");
-			ctl_pool_free(other_pool);
 			return (error);
 		}
 	}
 	error = kproc_kthread_add(ctl_lun_thread, softc,
-	    &softc->ctl_proc, NULL, 0, 0, "ctl", "lun");
+	    &softc->ctl_proc, &softc->lun_thread, 0, 0, "ctl", "lun");
 	if (error != 0) {
 		printf("error creating CTL lun thread!\n");
-		ctl_pool_free(other_pool);
 		return (error);
 	}
 	error = kproc_kthread_add(ctl_thresh_thread, softc,
-	    &softc->ctl_proc, NULL, 0, 0, "ctl", "thresh");
+	    &softc->ctl_proc, &softc->thresh_thread, 0, 0, "ctl", "thresh");
 	if (error != 0) {
 		printf("error creating CTL threshold thread!\n");
-		ctl_pool_free(other_pool);
 		return (error);
 	}
 
@@ -1911,58 +1943,54 @@ ctl_init(void)
 	    softc, 0, ctl_ha_role_sysctl, "I", "HA role for this head");
 
 	if (softc->is_single == 0) {
-		ctl_frontend_register(&ha_frontend);
-		if (ctl_ha_msg_init(softc) != CTL_HA_STATUS_SUCCESS) {
-			printf("ctl_init: ctl_ha_msg_init failed.\n");
+		if (ctl_frontend_register(&ha_frontend) != 0)
 			softc->is_single = 1;
-		} else
-		if (ctl_ha_msg_register(CTL_HA_CHAN_CTL, ctl_isc_event_handler)
-		    != CTL_HA_STATUS_SUCCESS) {
-			printf("ctl_init: ctl_ha_msg_register failed.\n");
-			softc->is_single = 1;
-		}
 	}
 	return (0);
 }
 
-void
+static int
 ctl_shutdown(void)
 {
 	struct ctl_softc *softc = control_softc;
-	struct ctl_lun *lun, *next_lun;
+	int i;
 
-	if (softc->is_single == 0) {
-		ctl_ha_msg_shutdown(softc);
-		if (ctl_ha_msg_deregister(CTL_HA_CHAN_CTL)
-		    != CTL_HA_STATUS_SUCCESS)
-			printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
-		if (ctl_ha_msg_destroy(softc) != CTL_HA_STATUS_SUCCESS)
-			printf("%s: ctl_ha_msg_destroy failed.\n", __func__);
+	if (softc->is_single == 0)
 		ctl_frontend_deregister(&ha_frontend);
-	}
 
-	mtx_lock(&softc->ctl_lock);
-
-	STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun)
-		ctl_free_lun(lun);
-
-	mtx_unlock(&softc->ctl_lock);
+	destroy_dev(softc->dev);
 
-#if 0
-	ctl_shutdown_thread(softc->work_thread);
-	mtx_destroy(&softc->queue_lock);
-#endif
+	/* Shutdown CTL threads. */
+	softc->shutdown = 1;
+	for (i = 0; i < worker_threads; i++) {
+		struct ctl_thread *thr = &softc->threads[i];
+		while (thr->thread != NULL) {
+			wakeup(thr);
+			if (thr->thread != NULL)
+				pause("CTL thr shutdown", 1);
+		}
+		mtx_destroy(&thr->queue_lock);
+	}
+	while (softc->lun_thread != NULL) {
+		wakeup(&softc->pending_lun_queue);
+		if (softc->lun_thread != NULL)
+			pause("CTL thr shutdown", 1);
+	}
+	while (softc->thresh_thread != NULL) {
+		wakeup(softc->thresh_thread);
+		if (softc->thresh_thread != NULL)
+			pause("CTL thr shutdown", 1);
+	}
 
 	ctl_tpc_shutdown(softc);
 	uma_zdestroy(softc->io_zone);
 	mtx_destroy(&softc->ctl_lock);
 
-	destroy_dev(softc->dev);
-
 	sysctl_ctx_free(&softc->sysctl_ctx);
 
 	free(softc, M_DEVBUF);
 	control_softc = NULL;
+	return (0);
 }
 
 static int
@@ -1973,7 +2001,7 @@ ctl_module_event_handler(module_t mod, int what, void *arg)
 	case MOD_LOAD:
 		return (ctl_init());
 	case MOD_UNLOAD:
-		return (EBUSY);
+		return (ctl_shutdown());
 	default:
 		return (EOPNOTSUPP);
 	}
@@ -13259,7 +13287,7 @@ ctl_work_thread(void *arg)
 
 	CTL_DEBUG_PRINT(("ctl_work_thread starting\n"));
 
-	for (;;) {
+	while (!softc->shutdown) {
 		/*
 		 * We handle the queues in this order:
 		 * - ISC
@@ -13309,6 +13337,8 @@ ctl_work_thread(void *arg)
 		/* Sleep until we have something to do. */
 		mtx_sleep(thr, &thr->queue_lock, PDROP | PRIBIO, "-", 0);
 	}
+	thr->thread = NULL;
+	kthread_exit();
 }
 
 static void
@@ -13319,7 +13349,7 @@ ctl_lun_thread(void *arg)
 
 	CTL_DEBUG_PRINT(("ctl_lun_thread starting\n"));
 
-	for (;;) {
+	while (!softc->shutdown) {
 		mtx_lock(&softc->ctl_lock);
 		be_lun = STAILQ_FIRST(&softc->pending_lun_queue);
 		if (be_lun != NULL) {
@@ -13333,6 +13363,8 @@ ctl_lun_thread(void *arg)
 		mtx_sleep(&softc->pending_lun_queue, &softc->ctl_lock,
 		    PDROP | PRIBIO, "-", 0);
 	}
+	softc->lun_thread = NULL;
+	kthread_exit();
 }
 
 static void
@@ -13348,7 +13380,7 @@ ctl_thresh_thread(void *arg)
 
 	CTL_DEBUG_PRINT(("ctl_thresh_thread starting\n"));
 
-	for (;;) {
+	while (!softc->shutdown) {
 		mtx_lock(&softc->ctl_lock);
 		STAILQ_FOREACH(lun, &softc->lun_list, links) {
 			if ((lun->flags & CTL_LUN_DISABLED) ||
@@ -13433,9 +13465,11 @@ ctl_thresh_thread(void *arg)
 				mtx_lock(&softc->ctl_lock);
 			}
 		}
-		mtx_unlock(&softc->ctl_lock);
-		pause("-", CTL_LBP_PERIOD * hz);
+		mtx_sleep(&softc->thresh_thread, &softc->ctl_lock,
+		    PDROP | PRIBIO, "-", CTL_LBP_PERIOD * hz);
 	}
+	softc->thresh_thread = NULL;
+	kthread_exit();
 }
 
 static void
diff --git a/sys/cam/ctl/ctl_backend.c b/sys/cam/ctl/ctl_backend.c
index 86f7d3c..bac7e85 100644
--- a/sys/cam/ctl/ctl_backend.c
+++ b/sys/cam/ctl/ctl_backend.c
@@ -67,11 +67,10 @@ ctl_backend_register(struct ctl_backend_driver *be)
 {
 	struct ctl_softc *softc = control_softc;
 	struct ctl_backend_driver *be_tmp;
+	int error;
 
+	/* Sanity check, make sure this isn't a duplicate registration. */
 	mtx_lock(&softc->ctl_lock);
-	/*
-	 * Sanity check, make sure this isn't a duplicate registration.
-	 */
 	STAILQ_FOREACH(be_tmp, &softc->be_list, links) {
 		if (strcmp(be_tmp->name, be->name) == 0) {
 			mtx_unlock(&softc->ctl_lock);
@@ -79,39 +78,24 @@ ctl_backend_register(struct ctl_backend_driver *be)
 		}
 	}
 	mtx_unlock(&softc->ctl_lock);
-
-	/*
-	 * Call the backend's initialization routine.
-	 */
-	be->init();
-
-	mtx_lock(&softc->ctl_lock);
-	
-	STAILQ_INSERT_TAIL(&softc->be_list, be, links);
-
-	softc->num_backends++;
-
-	/*
-	 * Don't want to increment the usage count for internal consumers,
-	 * we won't be able to unload otherwise.
-	 */
-	/* XXX KDM find a substitute for this? */
-#if 0
-	if ((be->flags & CTL_BE_FLAG_INTERNAL) == 0)
-		MOD_INC_USE_COUNT;
-#endif
-
 #ifdef CS_BE_CONFIG_MOVE_DONE_IS_NOT_USED
 	be->config_move_done = ctl_config_move_done;
 #endif
-	/* XXX KDM fix this! */
 	be->num_luns = 0;
-#if 0
-	atomic_set(&be->num_luns, 0);
-#endif
 
-	mtx_unlock(&softc->ctl_lock);
+	/* Call the backend's initialization routine. */
+	if (be->init != NULL) {
+		if ((error = be->init()) != 0) {
+			printf("%s backend init error: %d\n",
+			    be->name, error);
+			return (error);
+		}
+	}
 
+	mtx_lock(&softc->ctl_lock);
+	STAILQ_INSERT_TAIL(&softc->be_list, be, links);
+	softc->num_backends++;
+	mtx_unlock(&softc->ctl_lock);
 	return (0);
 }
 
@@ -119,30 +103,21 @@ int
 ctl_backend_deregister(struct ctl_backend_driver *be)
 {
 	struct ctl_softc *softc = control_softc;
-
-	mtx_lock(&softc->ctl_lock);
-
-#if 0
-	if (atomic_read(&be->num_luns) != 0) {
-#endif
-	/* XXX KDM fix this! */
-	if (be->num_luns != 0) {
-		mtx_unlock(&softc->ctl_lock);
-		return (-1);
+	int error;
+
+	/* Call the backend's shutdown routine. */
+	if (be->shutdown != NULL) {
+		if ((error = be->shutdown()) != 0) {
+			printf("%s backend shutdown error: %d\n",
+			    be->name, error);
+			return (error);
+		}
 	}
 
+	mtx_lock(&softc->ctl_lock);
 	STAILQ_REMOVE(&softc->be_list, be, ctl_backend_driver, links);
-
 	softc->num_backends--;
-
-	/* XXX KDM find a substitute for this? */
-#if 0
-	if ((be->flags & CTL_BE_FLAG_INTERNAL) == 0)
-		MOD_DEC_USE_COUNT;
-#endif
-
 	mtx_unlock(&softc->ctl_lock);
-
 	return (0);
 }
 
diff --git a/sys/cam/ctl/ctl_backend.h b/sys/cam/ctl/ctl_backend.h
index d8ada83..4202efc 100644
--- a/sys/cam/ctl/ctl_backend.h
+++ b/sys/cam/ctl/ctl_backend.h
@@ -55,12 +55,13 @@ typedef enum {
 	{ \
 		switch (type) { \
 		case MOD_LOAD: \
-			ctl_backend_register( \
-				(struct ctl_backend_driver *)data); \
+			return (ctl_backend_register( \
+				(struct ctl_backend_driver *)data)); \
 			break; \
 		case MOD_UNLOAD: \
-			printf(#name " module unload - not possible for this module type\n"); \
-			return EINVAL; \
+			return (ctl_backend_deregister( \
+				(struct ctl_backend_driver *)data)); \
+			break; \
 		default: \
 			return EOPNOTSUPP; \
 		} \
@@ -179,10 +180,10 @@ struct ctl_be_lun {
 typedef enum {
 	CTL_BE_FLAG_NONE	= 0x00,	/* no flags */
 	CTL_BE_FLAG_HAS_CONFIG	= 0x01,	/* can do config reads, writes */
-	CTL_BE_FLAG_INTERNAL	= 0x02	/* don't inc mod refcount */
 } ctl_backend_flags;
 
 typedef int (*be_init_t)(void);
+typedef int (*be_shutdown_t)(void);
 typedef int (*be_func_t)(union ctl_io *io);
 typedef void (*be_vfunc_t)(union ctl_io *io);
 typedef int (*be_ioctl_t)(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
@@ -194,6 +195,7 @@ struct ctl_backend_driver {
 	char		  name[CTL_BE_NAME_LEN]; /* passed to CTL */
 	ctl_backend_flags flags;	         /* passed to CTL */
 	be_init_t	  init;			 /* passed to CTL */
+	be_shutdown_t	  shutdown;		 /* passed to CTL */
 	be_func_t	  data_submit;		 /* passed to CTL */
 	be_func_t	  data_move_done;	 /* passed to CTL */
 	be_func_t	  config_read;		 /* passed to CTL */
diff --git a/sys/cam/ctl/ctl_backend_block.c b/sys/cam/ctl/ctl_backend_block.c
index 8f8fa21..2ac0cdd 100644
--- a/sys/cam/ctl/ctl_backend_block.c
+++ b/sys/cam/ctl/ctl_backend_block.c
@@ -185,6 +185,7 @@ struct ctl_be_block_lun {
  */
 struct ctl_be_block_softc {
 	struct mtx			 lock;
+	uma_zone_t			 beio_zone;
 	int				 num_luns;
 	STAILQ_HEAD(, ctl_be_block_lun)	 lun_list;
 };
@@ -276,13 +277,15 @@ static int ctl_be_block_config_write(union ctl_io *io);
 static int ctl_be_block_config_read(union ctl_io *io);
 static int ctl_be_block_lun_info(void *be_lun, struct sbuf *sb);
 static uint64_t ctl_be_block_lun_attr(void *be_lun, const char *attrname);
-int ctl_be_block_init(void);
+static int ctl_be_block_init(void);
+static int ctl_be_block_shutdown(void);
 
 static struct ctl_backend_driver ctl_be_block_driver = 
 {
 	.name = "block",
 	.flags = CTL_BE_FLAG_HAS_CONFIG,
 	.init = ctl_be_block_init,
+	.shutdown = ctl_be_block_shutdown,
 	.data_submit = ctl_be_block_submit,
 	.data_move_done = ctl_be_block_move_done,
 	.config_read = ctl_be_block_config_read,
@@ -295,14 +298,12 @@ static struct ctl_backend_driver ctl_be_block_driver =
 MALLOC_DEFINE(M_CTLBLK, "ctlblk", "Memory used for CTL block backend");
 CTL_BACKEND_DECLARE(cbb, ctl_be_block_driver);
 
-static uma_zone_t beio_zone;
-
 static struct ctl_be_block_io *
 ctl_alloc_beio(struct ctl_be_block_softc *softc)
 {
 	struct ctl_be_block_io *beio;
 
-	beio = uma_zalloc(beio_zone, M_WAITOK | M_ZERO);
+	beio = uma_zalloc(softc->beio_zone, M_WAITOK | M_ZERO);
 	beio->softc = softc;
 	return (beio);
 }
@@ -335,7 +336,7 @@ ctl_free_beio(struct ctl_be_block_io *beio)
 		       duplicate_free, beio->num_segs);
 	}
 
-	uma_zfree(beio_zone, beio);
+	uma_zfree(beio->softc->beio_zone, beio);
 }
 
 static void
@@ -2873,19 +2874,40 @@ ctl_be_block_lun_attr(void *be_lun, const char *attrname)
 	return (lun->getattr(lun, attrname));
 }
 
-int
+static int
 ctl_be_block_init(void)
 {
-	struct ctl_be_block_softc *softc;
-	int retval;
-
-	softc = &backend_block_softc;
-	retval = 0;
+	struct ctl_be_block_softc *softc = &backend_block_softc;
 
 	mtx_init(&softc->lock, "ctlblock", NULL, MTX_DEF);
-	beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io),
+	softc->beio_zone = uma_zcreate("beio", sizeof(struct ctl_be_block_io),
 	    NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
 	STAILQ_INIT(&softc->lun_list);
+	return (0);
+}
 
-	return (retval);
+
+static int
+ctl_be_block_shutdown(void)
+{
+	struct ctl_be_block_softc *softc = &backend_block_softc;
+	struct ctl_be_block_lun *lun, *next_lun;
+
+	mtx_lock(&softc->lock);
+	STAILQ_FOREACH_SAFE(lun, &softc->lun_list, links, next_lun) {
+		/*
+		 * Drop our lock here.  Since ctl_invalidate_lun() can call
+		 * back into us, this could potentially lead to a recursive
+		 * lock of the same mutex, which would cause a hang.
+		 */
+		mtx_unlock(&softc->lock);
+		ctl_disable_lun(&lun->cbe_lun);
+		ctl_invalidate_lun(&lun->cbe_lun);
+		mtx_lock(&softc->lock);
+	}
+	mtx_unlock(&softc->lock);
+
+	uma_zdestroy(softc->beio_zone);
+	mtx_destroy(&softc->lock);
+	return (0);
 }
diff --git a/sys/cam/ctl/ctl_backend_ramdisk.c b/sys/cam/ctl/ctl_backend_ramdisk.c
index 71ed07a..5b85d9c 100644
--- a/sys/cam/ctl/ctl_backend_ramdisk.c
+++ b/sys/cam/ctl/ctl_backend_ramdisk.c
@@ -108,8 +108,8 @@ struct ctl_be_ramdisk_softc {
 static struct ctl_be_ramdisk_softc rd_softc;
 extern struct ctl_softc *control_softc;
 
-int ctl_backend_ramdisk_init(void);
-void ctl_backend_ramdisk_shutdown(void);
+static int ctl_backend_ramdisk_init(void);
+static int ctl_backend_ramdisk_shutdown(void);
 static int ctl_backend_ramdisk_move_done(union ctl_io *io);
 static int ctl_backend_ramdisk_submit(union ctl_io *io);
 static void ctl_backend_ramdisk_continue(union ctl_io *io);
@@ -133,6 +133,7 @@ static struct ctl_backend_driver ctl_be_ramdisk_driver =
 	.name = "ramdisk",
 	.flags = CTL_BE_FLAG_HAS_CONFIG,
 	.init = ctl_backend_ramdisk_init,
+	.shutdown = ctl_backend_ramdisk_shutdown,
 	.data_submit = ctl_backend_ramdisk_submit,
 	.data_move_done = ctl_backend_ramdisk_move_done,
 	.config_read = ctl_backend_ramdisk_config_read,
@@ -170,7 +171,7 @@ ctl_backend_ramdisk_init(void)
 	return (0);
 }
 
-void
+static int
 ctl_backend_ramdisk_shutdown(void)
 {
 	struct ctl_be_ramdisk_softc *softc = &rd_softc;
@@ -192,20 +193,16 @@ ctl_backend_ramdisk_shutdown(void)
 		mtx_lock(&softc->lock);
 	}
 	mtx_unlock(&softc->lock);
-	
+
 #ifdef CTL_RAMDISK_PAGES
 	for (i = 0; i < softc->num_pages; i++)
 		free(softc->ramdisk_pages[i], M_RAMDISK);
-
 	free(softc->ramdisk_pages, M_RAMDISK);
 #else
 	free(softc->ramdisk_buffer, M_RAMDISK);
 #endif
-
-	if (ctl_backend_deregister(&ctl_be_ramdisk_driver) != 0) {
-		printf("ctl_backend_ramdisk_shutdown: "
-		       "ctl_backend_deregister() failed!\n");
-	}
+	mtx_destroy(&softc->lock);
+	return (0);
 }
 
 static int
diff --git a/sys/cam/ctl/ctl_frontend.c b/sys/cam/ctl/ctl_frontend.c
index 03ee9da..765a31d 100644
--- a/sys/cam/ctl/ctl_frontend.c
+++ b/sys/cam/ctl/ctl_frontend.c
@@ -70,12 +70,11 @@ ctl_frontend_register(struct ctl_frontend *fe)
 {
 	struct ctl_softc *softc = control_softc;
 	struct ctl_frontend *fe_tmp;
+	int error;
 
 	KASSERT(softc != NULL, ("CTL is not initialized"));
 
-	/*
-	 * Sanity check, make sure this isn't a duplicate registration.
-	 */
+	/* Sanity check, make sure this isn't a duplicate registration. */
 	mtx_lock(&softc->ctl_lock);
 	STAILQ_FOREACH(fe_tmp, &softc->fe_list, links) {
 		if (strcmp(fe_tmp->name, fe->name) == 0) {
@@ -86,11 +85,14 @@ ctl_frontend_register(struct ctl_frontend *fe)
 	mtx_unlock(&softc->ctl_lock);
 	STAILQ_INIT(&fe->port_list);
 
-	/*
-	 * Call the frontend's initialization routine.
-	 */
-	if (fe->init != NULL)
-		fe->init();
+	/* Call the frontend's initialization routine. */
+	if (fe->init != NULL) {
+		if ((error = fe->init()) != 0) {
+			printf("%s frontend init error: %d\n",
+			    fe->name, error);
+			return (error);
+		}
+	}
 
 	mtx_lock(&softc->ctl_lock);
 	softc->num_frontends++;
@@ -103,20 +105,21 @@ int
 ctl_frontend_deregister(struct ctl_frontend *fe)
 {
 	struct ctl_softc *softc = control_softc;
-
-	if (!STAILQ_EMPTY(&fe->port_list))
-		return (-1);
+	int error;
+
+	/* Call the frontend's shutdown routine.*/
+	if (fe->shutdown != NULL) {
+		if ((error = fe->shutdown()) != 0) {
+			printf("%s frontend shutdown error: %d\n",
+			    fe->name, error);
+			return (error);
+		}
+	}
 
 	mtx_lock(&softc->ctl_lock);
 	STAILQ_REMOVE(&softc->fe_list, fe, ctl_frontend, links);
 	softc->num_frontends--;
 	mtx_unlock(&softc->ctl_lock);
-
-	/*
-	 * Call the frontend's shutdown routine.
-	 */
-	if (fe->shutdown != NULL)
-		fe->shutdown();
 	return (0);
 }
 
diff --git a/sys/cam/ctl/ctl_frontend.h b/sys/cam/ctl/ctl_frontend.h
index 37e6824..38eb863 100644
--- a/sys/cam/ctl/ctl_frontend.h
+++ b/sys/cam/ctl/ctl_frontend.h
@@ -49,7 +49,7 @@ typedef enum {
 } ctl_port_status;
 
 typedef int (*fe_init_t)(void);
-typedef void (*fe_shutdown_t)(void);
+typedef int (*fe_shutdown_t)(void);
 typedef void (*port_func_t)(void *onoff_arg);
 typedef int (*port_info_func_t)(void *onoff_arg, struct sbuf *sb);
 typedef	int (*lun_func_t)(void *arg, int lun_id);
@@ -61,12 +61,13 @@ typedef int (*fe_ioctl_t)(struct cdev *dev, u_long cmd, caddr_t addr, int flag,
 	{ \
 		switch (type) { \
 		case MOD_LOAD: \
-			ctl_frontend_register( \
-				(struct ctl_frontend *)data); \
+			return (ctl_frontend_register( \
+				(struct ctl_frontend *)data)); \
 			break; \
 		case MOD_UNLOAD: \
-			printf(#name " module unload - not possible for this module type\n"); \
-			return EINVAL; \
+			return (ctl_frontend_deregister( \
+				(struct ctl_frontend *)data)); \
+			break; \
 		default: \
 			return EOPNOTSUPP; \
 		} \
diff --git a/sys/cam/ctl/ctl_frontend_cam_sim.c b/sys/cam/ctl/ctl_frontend_cam_sim.c
index 0b76455..9a0ca9a 100644
--- a/sys/cam/ctl/ctl_frontend_cam_sim.c
+++ b/sys/cam/ctl/ctl_frontend_cam_sim.c
@@ -94,15 +94,14 @@ struct cfcs_softc {
 	CAM_SNS_BUF_PHYS | CAM_CDB_PHYS | CAM_SENSE_PTR |		\
 	CAM_SENSE_PHYS)
 
-int cfcs_init(void);
+static int cfcs_init(void);
+static int cfcs_shutdown(void);
 static void cfcs_poll(struct cam_sim *sim);
 static void cfcs_online(void *arg);
 static void cfcs_offline(void *arg);
 static void cfcs_datamove(union ctl_io *io);
 static void cfcs_done(union ctl_io *io);
 void cfcs_action(struct cam_sim *sim, union ccb *ccb);
-static void cfcs_async(void *callback_arg, uint32_t code,
-		       struct cam_path *path, void *arg);
 
 struct cfcs_softc cfcs_softc;
 /*
@@ -121,14 +120,14 @@ static struct ctl_frontend cfcs_frontend =
 {
 	.name = "camsim",
 	.init = cfcs_init,
+	.shutdown = cfcs_shutdown,
 };
 CTL_FRONTEND_DECLARE(ctlcfcs, cfcs_frontend);
 
-int
+static int
 cfcs_init(void)
 {
 	struct cfcs_softc *softc;
-	struct ccb_setasync csa;
 	struct ctl_port *port;
 	int retval;
 
@@ -214,13 +213,6 @@ cfcs_init(void)
 		goto bailout;
 	}
 
-	xpt_setup_ccb(&csa.ccb_h, softc->path, CAM_PRIORITY_NONE);
-	csa.ccb_h.func_code = XPT_SASYNC_CB;
-	csa.event_enable = AC_LOST_DEVICE;
-	csa.callback = cfcs_async;
-        csa.callback_arg = softc->sim;
-        xpt_action((union ccb *)&csa);
-
 	mtx_unlock(&softc->lock);
 
 	return (retval);
@@ -236,6 +228,27 @@ bailout:
 	return (retval);
 }
 
+static int
+cfcs_shutdown(void)
+{
+	struct cfcs_softc *softc = &cfcs_softc;
+	struct ctl_port *port = &softc->port;
+	int error;
+
+	ctl_port_offline(port);
+
+	mtx_lock(&softc->lock);
+	xpt_free_path(softc->path);
+	xpt_bus_deregister(cam_sim_path(softc->sim));
+	cam_sim_free(softc->sim, /*free_devq*/ TRUE);
+	mtx_unlock(&softc->lock);
+	mtx_destroy(&softc->lock);
+
+	if ((error = ctl_port_deregister(port)) != 0)
+		printf("%s: cam_sim port deregistration failed\n", __func__);
+	return (error);
+}
+
 static void
 cfcs_poll(struct cam_sim *sim)
 {
@@ -801,9 +814,3 @@ cfcs_action(struct cam_sim *sim, union ccb *ccb)
 		break;
 	}
 }
-
-static void
-cfcs_async(void *callback_arg, uint32_t code, struct cam_path *path, void *arg)
-{
-
-}
diff --git a/sys/cam/ctl/ctl_frontend_ioctl.c b/sys/cam/ctl/ctl_frontend_ioctl.c
index 3f5c23c..4063c97 100644
--- a/sys/cam/ctl/ctl_frontend_ioctl.c
+++ b/sys/cam/ctl/ctl_frontend_ioctl.c
@@ -76,7 +76,7 @@ struct cfi_softc {
 static struct cfi_softc cfi_softc;
 
 static int cfi_init(void);
-static void cfi_shutdown(void);
+static int cfi_shutdown(void);
 static void cfi_datamove(union ctl_io *io);
 static void cfi_done(union ctl_io *io);
 
@@ -93,6 +93,7 @@ cfi_init(void)
 {
 	struct cfi_softc *isoftc = &cfi_softc;
 	struct ctl_port *port;
+	int error = 0;
 
 	memset(isoftc, 0, sizeof(*isoftc));
 
@@ -108,24 +109,25 @@ cfi_init(void)
 	port->targ_port = -1;
 	port->max_initiators = 1;
 
-	if (ctl_port_register(port) != 0) {
+	if ((error = ctl_port_register(port)) != 0) {
 		printf("%s: ioctl port registration failed\n", __func__);
-		return (0);
+		return (error);
 	}
 	ctl_port_online(port);
 	return (0);
 }
 
-void
+static int
 cfi_shutdown(void)
 {
 	struct cfi_softc *isoftc = &cfi_softc;
-	struct ctl_port *port;
+	struct ctl_port *port = &isoftc->port;
+	int error = 0;
 
-	port = &isoftc->port;
 	ctl_port_offline(port);
-	if (ctl_port_deregister(&isoftc->port) != 0)
-		printf("%s: ctl_frontend_deregister() failed\n", __func__);
+	if ((error = ctl_port_deregister(port)) != 0)
+		printf("%s: ioctl port deregistration failed\n", __func__);
+	return (error);
 }
 
 /*
diff --git a/sys/cam/ctl/ctl_frontend_iscsi.c b/sys/cam/ctl/ctl_frontend_iscsi.c
index 1f96f8e..8a47513 100644
--- a/sys/cam/ctl/ctl_frontend_iscsi.c
+++ b/sys/cam/ctl/ctl_frontend_iscsi.c
@@ -146,7 +146,8 @@ SYSCTL_INT(_kern_cam_ctl_iscsi, OID_AUTO, maxcmdsn_delta, CTLFLAG_RWTUN,
 #define	PDU_TOTAL_TRANSFER_LEN(X)	(X)->ip_prv1
 #define	PDU_R2TSN(X)			(X)->ip_prv2
 
-int		cfiscsi_init(void);
+static int	cfiscsi_init(void);
+static int	cfiscsi_shutdown(void);
 static void	cfiscsi_online(void *arg);
 static void	cfiscsi_offline(void *arg);
 static int	cfiscsi_info(void *arg, struct sbuf *sb);
@@ -178,6 +179,7 @@ static struct ctl_frontend cfiscsi_frontend =
 	.name = "iscsi",
 	.init = cfiscsi_init,
 	.ioctl = cfiscsi_ioctl,
+	.shutdown = cfiscsi_shutdown,
 };
 CTL_FRONTEND_DECLARE(ctlcfiscsi, cfiscsi_frontend);
 MODULE_DEPEND(ctlcfiscsi, icl, 1, 1, 1);
@@ -1274,7 +1276,7 @@ cfiscsi_session_delete(struct cfiscsi_session *cs)
 	free(cs, M_CFISCSI);
 }
 
-int
+static int
 cfiscsi_init(void)
 {
 	struct cfiscsi_softc *softc;
@@ -1297,6 +1299,23 @@ cfiscsi_init(void)
 	return (0);
 }
 
+static int
+cfiscsi_shutdown(void)
+{
+	struct cfiscsi_softc *softc = &cfiscsi_softc;
+
+	if (!TAILQ_EMPTY(&softc->sessions) || !TAILQ_EMPTY(&softc->targets))
+		return (EBUSY);
+
+	uma_zdestroy(cfiscsi_data_wait_zone);
+#ifdef ICL_KERNEL_PROXY
+	cv_destroy(&softc->accept_cv);
+#endif
+	cv_destroy(&softc->sessions_cv);
+	mtx_destroy(&softc->lock);
+	return (0);
+}
+
 #ifdef ICL_KERNEL_PROXY
 static void
 cfiscsi_accept(struct socket *so, struct sockaddr *sa, int portal_id)
diff --git a/sys/cam/ctl/ctl_private.h b/sys/cam/ctl/ctl_private.h
index 3f46769..40f0e61 100644
--- a/sys/cam/ctl/ctl_private.h
+++ b/sys/cam/ctl/ctl_private.h
@@ -470,7 +470,10 @@ struct ctl_softc {
 	STAILQ_HEAD(, ctl_backend_driver) be_list;
 	struct uma_zone *io_zone;
 	uint32_t cur_pool_id;
+	int shutdown;
 	struct ctl_thread threads[CTL_MAX_THREADS];
+	struct thread *lun_thread;
+	struct thread *thresh_thread;
 	TAILQ_HEAD(tpc_tokens, tpc_token) tpc_tokens;
 	struct callout tpc_timeout;
 	struct mtx tpc_lock;
diff --git a/sys/cam/ctl/ctl_tpc_local.c b/sys/cam/ctl/ctl_tpc_local.c
index 47b7ed6..e5e77b4 100644
--- a/sys/cam/ctl/ctl_tpc_local.c
+++ b/sys/cam/ctl/ctl_tpc_local.c
@@ -65,7 +65,7 @@ struct tpcl_softc {
 static struct tpcl_softc tpcl_softc;
 
 static int tpcl_init(void);
-static void tpcl_shutdown(void);
+static int tpcl_shutdown(void);
 static void tpcl_datamove(union ctl_io *io);
 static void tpcl_done(union ctl_io *io);
 
@@ -84,7 +84,7 @@ tpcl_init(void)
 	struct tpcl_softc *tsoftc = &tpcl_softc;
 	struct ctl_port *port;
 	struct scsi_transportid_spi *tid;
-	int len;
+	int error, len;
 
 	memset(tsoftc, 0, sizeof(*tsoftc));
 
@@ -100,9 +100,9 @@ tpcl_init(void)
 	port->targ_port = -1;
 	port->max_initiators = 1;
 
-	if (ctl_port_register(port) != 0) {
-		printf("%s: ctl_port_register() failed with error\n", __func__);
-		return (0);
+	if ((error = ctl_port_register(port)) != 0) {
+		printf("%s: tpc port registration failed\n", __func__);
+		return (error);
 	}
 
 	len = sizeof(struct scsi_transportid_spi);
@@ -118,16 +118,17 @@ tpcl_init(void)
 	return (0);
 }
 
-void
+static int
 tpcl_shutdown(void)
 {
 	struct tpcl_softc *tsoftc = &tpcl_softc;
-	struct ctl_port *port;
+	struct ctl_port *port = &tsoftc->port;
+	int error;
 
-	port = &tsoftc->port;
 	ctl_port_offline(port);
-	if (ctl_port_deregister(&tsoftc->port) != 0)
-		printf("%s: ctl_frontend_deregister() failed\n", __func__);
+	if ((error = ctl_port_deregister(port)) != 0)
+		printf("%s: tpc port deregistration failed\n", __func__);
+	return (error);
 }
 
 static void
diff --git a/sys/cam/ctl/scsi_ctl.c b/sys/cam/ctl/scsi_ctl.c
index 0ff7b30..dc82e91 100644
--- a/sys/cam/ctl/scsi_ctl.c
+++ b/sys/cam/ctl/scsi_ctl.c
@@ -188,8 +188,8 @@ MALLOC_DEFINE(M_CTLFE, "CAM CTL FE", "CAM CTL FE interface");
 #define PRIV_CCB(io)	((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[0])
 #define PRIV_INFO(io)	((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptrs[1])
 
-int			ctlfeinitialize(void);
-void			ctlfeshutdown(void);
+static int		ctlfeinitialize(void);
+static int		ctlfeshutdown(void);
 static periph_init_t	ctlfeperiphinit;
 static void		ctlfeasync(void *callback_arg, uint32_t code,
 				   struct cam_path *path, void *arg);
@@ -227,13 +227,15 @@ static struct ctl_frontend ctlfe_frontend =
 };
 CTL_FRONTEND_DECLARE(ctlfe, ctlfe_frontend);
 
-void
+static int
 ctlfeshutdown(void)
 {
-	return;
+
+	/* CAM does not support periph driver unregister now. */
+	return (EBUSY);
 }
 
-int
+static int
 ctlfeinitialize(void)
 {
 
@@ -243,7 +245,7 @@ ctlfeinitialize(void)
 	return (0);
 }
 
-void
+static void
 ctlfeperiphinit(void)
 {
 	cam_status status;