Move dynamic sysctl(8) variable creation for the cd(4) and da(4) drivers

out of cdregister() and daregister(), which are run from interrupt context.

The sysctl code does blocking mallocs (M_WAITOK), which causes problems
if malloc(9) actually needs to sleep.

The eventual fix for this issue will involve moving the CAM probe process
inside a kernel thread.  For now, though, I have fixed the issue by moving
dynamic sysctl variable creation for these two drivers to a task queue
running in a kernel thread.

The existing task queues (taskqueue_swi and taskqueue_swi_giant) run in
software interrupt handlers, which wouldn't fix the problem at hand.  So I
have created a new task queue, taskqueue_thread, that runs inside a kernel
thread.  (It also runs outside of Giant -- clients must explicitly acquire
and release Giant in their taskqueue functions.)

scsi_cd.c:	Remove sysctl variable creation code from cdregister(), and
		move it to a new function, cdsysctlinit().  Queue
		cdsysctlinit() to the taskqueue_thread taskqueue once we
		have fully registered the cd(4) driver instance.

scsi_da.c:	Remove sysctl variable creation code from daregister(), and
		move it to move it to a new function, dasysctlinit().
		Queue dasysctlinit() to the taskqueue_thread taskqueue once
		we have fully registered the da(4) instance.

taskqueue.h:	Declare the new taskqueue_thread taskqueue, update some
		comments.

subr_taskqueue.c:
		Create the new kernel thread taskqueue.  This taskqueue
		runs outside of Giant, so any functions queued to it would
		need to explicitly acquire/release Giant if they need it.

cd.4:		Update the cd(4) man page to talk about the minimum command
		size sysctl/loader tunable.  Also note that the changer
		variables are available as loader tunables as well.

da.4:		Update the da(4) man page to cover the retry_count,
		default_timeout and minimum_cmd_size sysctl variables/loader
		tunables.  Remove references to /dev/r???, they aren't used
		any longer.

cd.9:		Update the cd(9) man page to describe the CD_Q_10_BYTE_ONLY
		quirk.

taskqueue.9:	Update the taskqueue(9) man page to describe the new thread
		task queue, and the taskqueue_swi_giant queue.

MFC after:	3 days

2 files changed, 93 insertions, 43 deletions

diff --git a/sys/cam/scsi/scsi_cd.c b/sys/cam/scsi/scsi_cd.c
index a887243..39c2562 100644
--- a/sys/cam/scsi/scsi_cd.c
+++ b/sys/cam/scsi/scsi_cd.c
@@ -62,6 +62,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/dvdio.h>
 #include <sys/devicestat.h>
 #include <sys/sysctl.h>
+#include <sys/taskqueue.h>
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
@@ -154,6 +155,7 @@ struct cd_softc {
 	eventhandler_tag	clonetag;
 	int			minimum_command_size;
 	int			outstanding_cmds;
+	struct task		sysctl_task;
 	struct sysctl_ctx_list	sysctl_ctx;
 	struct sysctl_oid	*sysctl_tree;
 	STAILQ_HEAD(, cd_mode_params)	mode_queue;
@@ -598,6 +600,43 @@ cdasync(void *callback_arg, u_int32_t code,
 	}
 }
 
+static void
+cdsysctlinit(void *context, int pending)
+{
+	struct cam_periph *periph;
+	struct cd_softc *softc;
+	char tmpstr[80], tmpstr2[80];
+
+	periph = (struct cam_periph *)context;
+	softc = (struct cd_softc *)periph->softc;
+
+	snprintf(tmpstr, sizeof(tmpstr), "CAM CD unit %d", periph->unit_number);
+	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
+
+	mtx_lock(&Giant);
+
+	sysctl_ctx_init(&softc->sysctl_ctx);
+	softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
+		SYSCTL_STATIC_CHILDREN(_kern_cam_cd), OID_AUTO,
+		tmpstr2, CTLFLAG_RD, 0, tmpstr);
+
+	if (softc->sysctl_tree == NULL) {
+		printf("cdsysctlinit: unable to allocate sysctl tree\n");
+		return;
+	}
+
+	/*
+	 * Now register the sysctl handler, so the user can the value on
+	 * the fly.
+	 */
+	SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+		OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW,
+		&softc->minimum_command_size, 0, cdcmdsizesysctl, "I",
+		"Minimum CDB size");
+
+	mtx_unlock(&Giant);
+}
+
 /*
  * We have a handler function for this so we can check the values when the
  * user sets them, instead of every time we look at them.
@@ -642,7 +681,7 @@ cdregister(struct cam_periph *periph, void *arg)
 	struct ccb_setasync csa;
 	struct ccb_pathinq cpi;
 	struct ccb_getdev *cgd;
-	char tmpstr[80], tmpstr2[80];
+	char tmpstr[80];
 	caddr_t match;
 
 	cgd = (struct ccb_getdev *)arg;
@@ -696,17 +735,7 @@ cdregister(struct cam_periph *periph, void *arg)
 	if (cpi.ccb_h.status == CAM_REQ_CMP && (cpi.hba_misc & PIM_NO_6_BYTE))
 		softc->quirks |= CD_Q_10_BYTE_ONLY;
 
-	snprintf(tmpstr, sizeof(tmpstr), "CAM CD unit %d", periph->unit_number);
-	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
-	sysctl_ctx_init(&softc->sysctl_ctx);
-	softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
-		SYSCTL_STATIC_CHILDREN(_kern_cam_cd), OID_AUTO,
-		tmpstr2, CTLFLAG_RD, 0, tmpstr);
-	if (softc->sysctl_tree == NULL) {
-		printf("cdregister: unable to allocate sysctl tree\n");
-		free(softc, M_DEVBUF);
-		return (CAM_REQ_CMP_ERR);
-	}
+	TASK_INIT(&softc->sysctl_task, 0, cdsysctlinit, periph);
 
 	/* The default is 6 byte commands, unless quirked otherwise */
 	if (softc->quirks & CD_Q_10_BYTE_ONLY)
@@ -728,15 +757,6 @@ cdregister(struct cam_periph *periph, void *arg)
 		softc->minimum_command_size = 10;
 
 	/*
-	 * Now register the sysctl handler, so the user can the value on
-	 * the fly.
-	 */
-	SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
-		OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW,
-		&softc->minimum_command_size, 0, cdcmdsizesysctl, "I",
-		"Minimum CDB size");
-
-	/*
 	 * We need to register the statistics structure for this device,
 	 * but we don't have the blocksize yet for it.  So, we register
 	 * the structure and indicate that we don't have the blocksize
@@ -1847,6 +1867,11 @@ cddone(struct cam_periph *periph, union ccb *done_ccb)
 			xpt_announce_periph(periph, announce_buf);
 			if (softc->flags & CD_FLAG_CHANGER)
 				cdchangerschedule(softc);
+			/*
+			 * Create our sysctl variables, now that we know
+			 * we have successfully attached.
+			 */
+			taskqueue_enqueue(taskqueue_thread,&softc->sysctl_task);
 		}
 		softc->state = CD_STATE_NORMAL;		
 		/*
diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index e2edbcf..ce2bfff 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/eventhandler.h>
 #include <sys/malloc.h>
 #include <sys/cons.h>
+#include <sys/taskqueue.h>
 
 #include <machine/md_var.h>
 
@@ -133,6 +134,7 @@ struct da_softc {
 	struct	 disk_params params;
 	struct	 disk disk;
 	union	 ccb saved_ccb;
+	struct task		sysctl_task;
 	struct sysctl_ctx_list	sysctl_ctx;
 	struct sysctl_oid	*sysctl_tree;
 };
@@ -388,6 +390,7 @@ static	dumper_t	dadump;
 static	periph_init_t	dainit;
 static	void		daasync(void *callback_arg, u_int32_t code,
 				struct cam_path *path, void *arg);
+static	void		dasysctlinit(void *context, int pending);
 static	int		dacmdsizesysctl(SYSCTL_HANDLER_ARGS);
 static	periph_ctor_t	daregister;
 static	periph_dtor_t	dacleanup;
@@ -915,6 +918,41 @@ daasync(void *callback_arg, u_int32_t code,
 	}
 }
 
+static void
+dasysctlinit(void *context, int pending)
+{
+	struct cam_periph *periph;
+	struct da_softc *softc;
+	char tmpstr[80], tmpstr2[80];
+
+	periph = (struct cam_periph *)context;
+	softc = (struct da_softc *)periph->softc;
+
+	snprintf(tmpstr, sizeof(tmpstr), "CAM DA unit %d", periph->unit_number);
+	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
+
+	mtx_lock(&Giant);
+	sysctl_ctx_init(&softc->sysctl_ctx);
+	softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
+		SYSCTL_STATIC_CHILDREN(_kern_cam_da), OID_AUTO, tmpstr2,
+		CTLFLAG_RD, 0, tmpstr);
+	if (softc->sysctl_tree == NULL) {
+		printf("dasysctlinit: unable to allocate sysctl tree\n");
+		return;
+	}
+
+	/*
+	 * Now register the sysctl handler, so the user can the value on
+	 * the fly.
+	 */
+	SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
+		OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW,
+		&softc->minimum_cmd_size, 0, dacmdsizesysctl, "I",
+		"Minimum CDB size");
+
+	mtx_unlock(&Giant);
+}
+
 static int
 dacmdsizesysctl(SYSCTL_HANDLER_ARGS)
 {
@@ -955,7 +993,7 @@ daregister(struct cam_periph *periph, void *arg)
 	struct ccb_setasync csa;
 	struct ccb_pathinq cpi;
 	struct ccb_getdev *cgd;
-	char tmpstr[80], tmpstr2[80];
+	char tmpstr[80];
 	caddr_t match;
 
 	cgd = (struct ccb_getdev *)arg;
@@ -1008,17 +1046,7 @@ daregister(struct cam_periph *periph, void *arg)
 	if (cpi.ccb_h.status == CAM_REQ_CMP && (cpi.hba_misc & PIM_NO_6_BYTE))
 		softc->quirks |= DA_Q_NO_6_BYTE;
 
-	snprintf(tmpstr, sizeof(tmpstr), "CAM DA unit %d", periph->unit_number);
-	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
-	sysctl_ctx_init(&softc->sysctl_ctx);
-	softc->sysctl_tree = SYSCTL_ADD_NODE(&softc->sysctl_ctx,
-		SYSCTL_STATIC_CHILDREN(_kern_cam_da), OID_AUTO, tmpstr2,
-		CTLFLAG_RD, 0, tmpstr);
-	if (softc->sysctl_tree == NULL) {
-		printf("daregister: unable to allocate sysctl tree\n");
-		free(softc, M_DEVBUF);
-		return (CAM_REQ_CMP_ERR);
-	}
+	TASK_INIT(&softc->sysctl_task, 0, dasysctlinit, periph);
 
 	/*
 	 * RBC devices don't have to support READ(6), only READ(10).
@@ -1050,15 +1078,6 @@ daregister(struct cam_periph *periph, void *arg)
 		softc->minimum_cmd_size = 16;
 
 	/*
-	 * Now register the sysctl handler, so the user can the value on
-	 * the fly.
-	 */
-	SYSCTL_ADD_PROC(&softc->sysctl_ctx,SYSCTL_CHILDREN(softc->sysctl_tree),
-		OID_AUTO, "minimum_cmd_size", CTLTYPE_INT | CTLFLAG_RW,
-		&softc->minimum_cmd_size, 0, dacmdsizesysctl, "I",
-		"Minimum CDB size");
-
-	/*
 	 * Block our timeout handler while we
 	 * add this softc to the dev list.
 	 */
@@ -1539,8 +1558,14 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 			}
 		}
 		free(csio->data_ptr, M_TEMP);
-		if (announce_buf[0] != '\0')
+		if (announce_buf[0] != '\0') {
 			xpt_announce_periph(periph, announce_buf);
+			/*
+			 * Create our sysctl variables, now that we know
+			 * we have successfully attached.
+			 */
+			taskqueue_enqueue(taskqueue_thread,&softc->sysctl_task);
+		}
 		softc->state = DA_STATE_NORMAL;	
 		/*
 		 * Since our peripheral may be invalidated by an error