Flush my local cache of cryto subsystem fixes:

o add a ``done'' flag for crypto operations; this is set when the operation
  completes and is intended for callers to check operations that may complete
  ``prematurely'' because of direct callbacks
o close a race for operations where the crypto driver returns ERESTART: we
  need to hold the q lock to insure the blocked state for the driver and any
  driver-private state is consistent; otherwise drivers may take an interrupt
  and notify the crypto subsystem that it can unblock the driver but operations
  will be left queued and never be processed
o close a race in /dev/crypto where operations can complete before the caller
  can sleep waiting for the callback: use a per-session mutex and the new done
  flag to handle this
o correct crypto_dispatch's handling of operations where the driver returns
  ERESTART: the return value must be zero and not ERESTART, otherwise the
  caller may free the crypto request despite it being queued for later handling
  (this typically results in a later panic)
o change crypto mutex ``names'' so witness printouts and the like are more
  meaningful

1 files changed, 26 insertions, 6 deletions

diff --git a/sys/opencrypto/cryptodev.c b/sys/opencrypto/cryptodev.c
index a40c859..e9a1129 100644
--- a/sys/opencrypto/cryptodev.c
+++ b/sys/opencrypto/cryptodev.c
@@ -56,6 +56,7 @@ struct csession {
 	TAILQ_ENTRY(csession) next;
 	u_int64_t	sid;
 	u_int32_t	ses;
+	struct mtx	lock;		/* for op submission */
 
 	u_int32_t	cipher;
 	struct enc_xform *txform;
@@ -419,12 +420,21 @@ cryptodev_op(
 		crp->crp_mac=cse->tmp_mac;
 	}
 
-	crypto_dispatch(crp);
-	error = tsleep(cse, PSOCK, "crydev", 0);
-	if (error) {
-		/* XXX can this happen?  if so, how do we recover? */
+	/*
+	 * Let the dispatch run unlocked, then, interlock against the
+	 * callback before checking if the operation completed and going
+	 * to sleep.  This insures drivers don't inherit our lock which
+	 * results in a lock order reversal between crypto_dispatch forced
+	 * entry and the crypto_done callback into us.
+	 */
+	error = crypto_dispatch(crp);
+	mtx_lock(&cse->lock);
+	if (error == 0 && (crp->crp_flags & CRYPTO_F_DONE) == 0)
+		error = msleep(crp, &cse->lock, PWAIT, "crydev", 0);
+	mtx_unlock(&cse->lock);
+
+	if (error != 0)
 		goto bail;
-	}
 
 	if (crp->crp_etype != 0) {
 		error = crp->crp_etype;
@@ -462,7 +472,9 @@ cryptodev_cb(void *op)
 	cse->error = crp->crp_etype;
 	if (crp->crp_etype == EAGAIN)
 		return crypto_dispatch(crp);
-	wakeup(cse);
+	mtx_lock(&cse->lock);
+	wakeup_one(crp);
+	mtx_unlock(&cse->lock);
 	return (0);
 }
 
@@ -661,10 +673,17 @@ csecreate(struct fcrypt *fcr, u_int64_t sid, caddr_t key, u_int64_t keylen,
 {
 	struct csession *cse;
 
+#ifdef INVARIANTS
+	/* NB: required when mtx_init is built with INVARIANTS */
+	MALLOC(cse, struct csession *, sizeof(struct csession),
+	    M_XDATA, M_NOWAIT | M_ZERO);
+#else
 	MALLOC(cse, struct csession *, sizeof(struct csession),
 	    M_XDATA, M_NOWAIT);
+#endif
 	if (cse == NULL)
 		return NULL;
+	mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF);
 	cse->key = key;
 	cse->keylen = keylen/8;
 	cse->mackey = mackey;
@@ -684,6 +703,7 @@ csefree(struct csession *cse)
 	int error;
 
 	error = crypto_freesession(cse->sid);
+	mtx_destroy(&cse->lock);
 	if (cse->key)
 		FREE(cse->key, M_XDATA);
 	if (cse->mackey)