MFC r286773:

Improve the realtime properties of USB transfers for embedded systems
like RPI-B and RPI-2.

8 files changed, 73 insertions, 30 deletions

diff --git a/sys/boot/usb/bsd_kernel.h b/sys/boot/usb/bsd_kernel.h
index cb947dd..74bccd6 100644
--- a/sys/boot/usb/bsd_kernel.h
+++ b/sys/boot/usb/bsd_kernel.h
@@ -42,7 +42,8 @@
 #define	M_USBDEV 0
 #define	USB_PROC_MAX 3
 #define	USB_BUS_GIANT_PROC(bus) (usb_process + 2)
-#define	USB_BUS_NON_GIANT_PROC(bus) (usb_process + 2)
+#define	USB_BUS_NON_GIANT_BULK_PROC(bus) (usb_process + 2)
+#define	USB_BUS_NON_GIANT_ISOC_PROC(bus) (usb_process + 2)
 #define	USB_BUS_EXPLORE_PROC(bus) (usb_process + 0)
 #define	USB_BUS_CONTROL_XFER_PROC(bus) (usb_process + 1)
 #define	SYSCTL_DECL(...)
diff --git a/sys/dev/usb/controller/usb_controller.c b/sys/dev/usb/controller/usb_controller.c
index f67c94d..1852467 100644
--- a/sys/dev/usb/controller/usb_controller.c
+++ b/sys/dev/usb/controller/usb_controller.c
@@ -233,7 +233,8 @@ usb_detach(device_t dev)
 	/* Get rid of USB callback processes */
 
 	usb_proc_free(USB_BUS_GIANT_PROC(bus));
-	usb_proc_free(USB_BUS_NON_GIANT_PROC(bus));
+	usb_proc_free(USB_BUS_NON_GIANT_ISOC_PROC(bus));
+	usb_proc_free(USB_BUS_NON_GIANT_BULK_PROC(bus));
 
 	/* Get rid of USB explore process */
 
@@ -397,7 +398,8 @@ usb_bus_explore(struct usb_proc_msg *pm)
 		 */
 		usb_proc_rewakeup(USB_BUS_CONTROL_XFER_PROC(bus));
 		usb_proc_rewakeup(USB_BUS_GIANT_PROC(bus));
-		usb_proc_rewakeup(USB_BUS_NON_GIANT_PROC(bus));
+		usb_proc_rewakeup(USB_BUS_NON_GIANT_ISOC_PROC(bus));
+		usb_proc_rewakeup(USB_BUS_NON_GIANT_BULK_PROC(bus));
 #endif
 
 		USB_BUS_UNLOCK(bus);
@@ -862,9 +864,13 @@ usb_attach_sub(device_t dev, struct usb_bus *bus)
 	    &bus->bus_mtx, device_get_nameunit(dev), USB_PRI_MED)) {
 		device_printf(dev, "WARNING: Creation of USB Giant "
 		    "callback process failed.\n");
-	} else if (usb_proc_create(USB_BUS_NON_GIANT_PROC(bus),
+	} else if (usb_proc_create(USB_BUS_NON_GIANT_ISOC_PROC(bus),
+	    &bus->bus_mtx, device_get_nameunit(dev), USB_PRI_HIGHEST)) {
+		device_printf(dev, "WARNING: Creation of USB non-Giant ISOC "
+		    "callback process failed.\n");
+	} else if (usb_proc_create(USB_BUS_NON_GIANT_BULK_PROC(bus),
 	    &bus->bus_mtx, device_get_nameunit(dev), USB_PRI_HIGH)) {
-		device_printf(dev, "WARNING: Creation of USB non-Giant "
+		device_printf(dev, "WARNING: Creation of USB non-Giant BULK "
 		    "callback process failed.\n");
 	} else if (usb_proc_create(USB_BUS_EXPLORE_PROC(bus),
 	    &bus->bus_mtx, device_get_nameunit(dev), USB_PRI_MED)) {
diff --git a/sys/dev/usb/usb_bus.h b/sys/dev/usb/usb_bus.h
index afc20f4..e9d4048 100644
--- a/sys/dev/usb/usb_bus.h
+++ b/sys/dev/usb/usb_bus.h
@@ -57,19 +57,26 @@ struct usb_bus {
 	struct root_hold_token *bus_roothold;
 #endif
 
+/* convenience macros */
+#define	USB_BUS_TT_PROC(bus) USB_BUS_NON_GIANT_ISOC_PROC(bus)
+#define	USB_BUS_CS_PROC(bus) USB_BUS_NON_GIANT_ISOC_PROC(bus)
+  
 #if USB_HAVE_PER_BUS_PROCESS
 #define	USB_BUS_GIANT_PROC(bus) (&(bus)->giant_callback_proc)
-#define	USB_BUS_NON_GIANT_PROC(bus) (&(bus)->non_giant_callback_proc)
+#define	USB_BUS_NON_GIANT_ISOC_PROC(bus) (&(bus)->non_giant_isoc_callback_proc)
+#define	USB_BUS_NON_GIANT_BULK_PROC(bus) (&(bus)->non_giant_bulk_callback_proc)
 #define	USB_BUS_EXPLORE_PROC(bus) (&(bus)->explore_proc)
 #define	USB_BUS_CONTROL_XFER_PROC(bus) (&(bus)->control_xfer_proc)
-
 	/*
-	 * There are two callback processes. One for Giant locked
-	 * callbacks. One for non-Giant locked callbacks. This should
-	 * avoid congestion and reduce response time in most cases.
+	 * There are three callback processes. One for Giant locked
+	 * callbacks. One for non-Giant locked non-periodic callbacks
+	 * and one for non-Giant locked periodic callbacks. This
+	 * should avoid congestion and reduce response time in most
+	 * cases.
 	 */
 	struct usb_process giant_callback_proc;
-	struct usb_process non_giant_callback_proc;
+	struct usb_process non_giant_isoc_callback_proc;
+	struct usb_process non_giant_bulk_callback_proc;
 
 	/* Explore process */
 	struct usb_process explore_proc;
diff --git a/sys/dev/usb/usb_device.c b/sys/dev/usb/usb_device.c
index c3936f6..3e29aa4 100644
--- a/sys/dev/usb/usb_device.c
+++ b/sys/dev/usb/usb_device.c
@@ -2184,7 +2184,7 @@ usb_free_device(struct usb_device *udev, uint8_t flag)
 	 * anywhere:
 	 */
 	USB_BUS_LOCK(udev->bus);
-	usb_proc_mwait(USB_BUS_NON_GIANT_PROC(udev->bus),
+	usb_proc_mwait(USB_BUS_CS_PROC(udev->bus),
 	    &udev->cs_msg[0], &udev->cs_msg[1]);
 	USB_BUS_UNLOCK(udev->bus);
 
diff --git a/sys/dev/usb/usb_hub.c b/sys/dev/usb/usb_hub.c
index 1c8682b..354e62d 100644
--- a/sys/dev/usb/usb_hub.c
+++ b/sys/dev/usb/usb_hub.c
@@ -349,7 +349,7 @@ uhub_tt_buffer_reset_async_locked(struct usb_device *child, struct usb_endpoint
 	}
 	up->req_reset_tt = req;
 	/* get reset transfer started */
-	usb_proc_msignal(USB_BUS_NON_GIANT_PROC(udev->bus),
+	usb_proc_msignal(USB_BUS_TT_PROC(udev->bus),
 	    &hub->tt_msg[0], &hub->tt_msg[1]);
 }
 #endif
@@ -1592,7 +1592,7 @@ uhub_detach(device_t dev)
 #if USB_HAVE_TT_SUPPORT
 	/* Make sure our TT messages are not queued anywhere */
 	USB_BUS_LOCK(bus);
-	usb_proc_mwait(USB_BUS_NON_GIANT_PROC(bus),
+	usb_proc_mwait(USB_BUS_TT_PROC(bus),
 	    &hub->tt_msg[0], &hub->tt_msg[1]);
 	USB_BUS_UNLOCK(bus);
 #endif
diff --git a/sys/dev/usb/usb_process.h b/sys/dev/usb/usb_process.h
index c12cdc4..dd20afd 100644
--- a/sys/dev/usb/usb_process.h
+++ b/sys/dev/usb/usb_process.h
@@ -34,6 +34,7 @@
 #endif
 
 /* defines */
+#define	USB_PRI_HIGHEST	PI_SWI(SWI_TTY)
 #define	USB_PRI_HIGH	PI_SWI(SWI_NET)
 #define	USB_PRI_MED	PI_SWI(SWI_CAMBIO)
 
diff --git a/sys/dev/usb/usb_transfer.c b/sys/dev/usb/usb_transfer.c
index 6c2e1440..53face6 100644
--- a/sys/dev/usb/usb_transfer.c
+++ b/sys/dev/usb/usb_transfer.c
@@ -872,6 +872,19 @@ done:
 	}
 }
 
+static uint8_t
+usbd_transfer_setup_has_bulk(const struct usb_config *setup_start,
+    uint16_t n_setup)
+{
+	while (n_setup--) {
+		uint8_t type = setup_start[n_setup].type;
+		if (type == UE_BULK || type == UE_BULK_INTR ||
+		    type == UE_TYPE_ANY)
+			return (1);
+	}
+	return (0);
+}
+
 /*------------------------------------------------------------------------*
  *	usbd_transfer_setup - setup an array of USB transfers
  *
@@ -1013,9 +1026,12 @@ usbd_transfer_setup(struct usb_device *udev,
 			else if (xfer_mtx == &Giant)
 				info->done_p =
 				    USB_BUS_GIANT_PROC(udev->bus);
+			else if (usbd_transfer_setup_has_bulk(setup_start, n_setup))
+				info->done_p =
+				    USB_BUS_NON_GIANT_BULK_PROC(udev->bus);
 			else
 				info->done_p =
-				    USB_BUS_NON_GIANT_PROC(udev->bus);
+				    USB_BUS_NON_GIANT_ISOC_PROC(udev->bus);
 		}
 		/* reset sizes */
 
@@ -2280,10 +2296,8 @@ usbd_callback_ss_done_defer(struct usb_xfer *xfer)
 	         * will have a Lock Order Reversal, LOR, if we try to
 	         * proceed !
 	         */
-		if (usb_proc_msignal(info->done_p,
-		    &info->done_m[0], &info->done_m[1])) {
-			/* ignore */
-		}
+		(void) usb_proc_msignal(info->done_p,
+		    &info->done_m[0], &info->done_m[1]);
 	} else {
 		/* clear second recurse flag */
 		pq->recurse_2 = 0;
@@ -2307,23 +2321,26 @@ usbd_callback_wrapper(struct usb_xfer_queue *pq)
 	struct usb_xfer_root *info = xfer->xroot;
 
 	USB_BUS_LOCK_ASSERT(info->bus, MA_OWNED);
-	if (!mtx_owned(info->xfer_mtx) && !SCHEDULER_STOPPED()) {
+	if ((pq->recurse_3 != 0 || mtx_owned(info->xfer_mtx) == 0) &&
+	    SCHEDULER_STOPPED() == 0) {
 		/*
 	       	 * Cases that end up here:
 		 *
 		 * 5) HW interrupt done callback or other source.
+		 * 6) HW completed transfer during callback
 		 */
-		DPRINTFN(3, "case 5\n");
+		DPRINTFN(3, "case 5 and 6\n");
 
 		/*
 	         * We have to postpone the callback due to the fact we
 	         * will have a Lock Order Reversal, LOR, if we try to
-	         * proceed !
+	         * proceed!
+		 *
+		 * Postponing the callback also ensures that other USB
+		 * transfer queues get a chance.
 	         */
-		if (usb_proc_msignal(info->done_p,
-		    &info->done_m[0], &info->done_m[1])) {
-			/* ignore */
-		}
+		(void) usb_proc_msignal(info->done_p,
+		    &info->done_m[0], &info->done_m[1]);
 		return;
 	}
 	/*
@@ -2697,7 +2714,7 @@ usbd_pipe_start(struct usb_xfer_queue *pq)
 			} else if (udev->ctrl_xfer[1]) {
 				info = udev->ctrl_xfer[1]->xroot;
 				usb_proc_msignal(
-				    USB_BUS_NON_GIANT_PROC(info->bus),
+				    USB_BUS_CS_PROC(info->bus),
 				    &udev->cs_msg[0], &udev->cs_msg[1]);
 			} else {
 				/* should not happen */
@@ -3022,9 +3039,11 @@ usb_command_wrapper(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
 
 	if (!pq->recurse_1) {
 
-		do {
+		/* clear third recurse flag */
+		pq->recurse_3 = 0;
 
-			/* set both recurse flags */
+		do {
+			/* set two first recurse flags */
 			pq->recurse_1 = 1;
 			pq->recurse_2 = 1;
 
@@ -3043,6 +3062,12 @@ usb_command_wrapper(struct usb_xfer_queue *pq, struct usb_xfer *xfer)
 			(pq->command) (pq);
 			DPRINTFN(6, "cb %p (leave)\n", pq->curr);
 
+			/*
+			 * Set third recurse flag to indicate
+			 * recursion happened:
+			 */
+			pq->recurse_3 = 1;
+
 		} while (!pq->recurse_2);
 
 		/* clear first recurse flag */
@@ -3318,7 +3343,8 @@ usbd_transfer_poll(struct usb_xfer **ppxfer, uint16_t max)
 		USB_BUS_CONTROL_XFER_PROC(udev->bus)->up_msleep = 0;
 		USB_BUS_EXPLORE_PROC(udev->bus)->up_msleep = 0;
 		USB_BUS_GIANT_PROC(udev->bus)->up_msleep = 0;
-		USB_BUS_NON_GIANT_PROC(udev->bus)->up_msleep = 0;
+		USB_BUS_NON_GIANT_ISOC_PROC(udev->bus)->up_msleep = 0;
+		USB_BUS_NON_GIANT_BULK_PROC(udev->bus)->up_msleep = 0;
 
 		/* poll USB hardware */
 		(udev->bus->methods->xfer_poll) (udev->bus);
diff --git a/sys/dev/usb/usbdi.h b/sys/dev/usb/usbdi.h
index f3930ff..a125184 100644
--- a/sys/dev/usb/usbdi.h
+++ b/sys/dev/usb/usbdi.h
@@ -128,6 +128,8 @@ struct usb_xfer_queue {
 	void    (*command) (struct usb_xfer_queue *pq);
 	uint8_t	recurse_1:1;
 	uint8_t	recurse_2:1;
+	uint8_t	recurse_3:1;
+	uint8_t	reserved:5;
 };
 
 /*