From 9da403e48c25f4100a1febdffa6b0aa4c372c7fa Mon Sep 17 00:00:00 2001 From: sam Date: Sun, 23 Feb 2003 07:25:48 +0000 Subject: o add a CRYPTO_F_CBIMM flag to symmetric ops to indicate the callback should be done in crypto_done rather than in the callback thread o use this flag to mark operations from /dev/crypto since the callback routine just does a wakeup; this eliminates the last unneeded ctx switch o change CRYPTO_F_NODELAY to CRYPTO_F_BATCH with an inverted meaning so "0" becomes the default/desired setting (needed for user-mode compatibility with openbsd) o change crypto_dispatch to honor CRYPTO_F_BATCH instead of always dispatching immediately o remove uses of CRYPTO_F_NODELAY o define COP_F_BATCH for ops submitted through /dev/crypto and pass this on to the op that is submitted Similar changes and more eventually coming for asymmetric ops. MFC if re gives approval. --- sys/opencrypto/crypto.c | 98 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 26 deletions(-) (limited to 'sys/opencrypto/crypto.c') diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index c31b383..7f6c71c 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -655,7 +655,6 @@ int crypto_dispatch(struct cryptop *crp) { u_int32_t hid = SESID2HID(crp->crp_sid); - struct cryptocap *cap; int result; cryptostats.cs_ops++; @@ -665,29 +664,54 @@ crypto_dispatch(struct cryptop *crp) binuptime(&crp->crp_tstamp); #endif - CRYPTO_Q_LOCK(); - cap = crypto_checkdriver(hid); - if (cap && !cap->cc_qblocked) { - result = crypto_invoke(crp, 0); - if (result == ERESTART) { + if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) { + struct cryptocap *cap; + /* + * Caller marked the request to be processed + * immediately; dispatch it directly to the + * driver unless the driver is currently blocked. + */ + cap = crypto_checkdriver(hid); + if (cap && !cap->cc_qblocked) { + result = crypto_invoke(crp, 0); + if (result == ERESTART) { + /* + * The driver ran out of resources, mark the + * driver ``blocked'' for cryptop's and put + * the request on the queue. + */ + CRYPTO_Q_LOCK(); + crypto_drivers[hid].cc_qblocked = 1; + TAILQ_INSERT_HEAD(&crp_q, crp, crp_next); + CRYPTO_Q_UNLOCK(); + cryptostats.cs_blocks++; + } + } else { /* - * The driver ran out of resources, mark the - * driver ``blocked'' for cryptop's and put - * the request on the queue. + * The driver is blocked, just queue the op until + * it unblocks and the kernel thread gets kicked. */ - crypto_drivers[hid].cc_qblocked = 1; - TAILQ_INSERT_HEAD(&crp_q, crp, crp_next); - cryptostats.cs_blocks++; + CRYPTO_Q_LOCK(); + TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); + CRYPTO_Q_UNLOCK(); + result = 0; } } else { + int wasempty; /* - * The driver is blocked, just queue the op until - * it unblocks and the kernel thread gets kicked. + * Caller marked the request as ``ok to delay''; + * queue it for the dispatch thread. This is desirable + * when the operation is low priority and/or suitable + * for batching. */ + CRYPTO_Q_LOCK(); + wasempty = TAILQ_EMPTY(&crp_q); TAILQ_INSERT_TAIL(&crp_q, crp, crp_next); + if (wasempty) + wakeup_one(&crp_q); + CRYPTO_Q_UNLOCK(); result = 0; } - CRYPTO_Q_UNLOCK(); return result; } @@ -818,8 +842,6 @@ crypto_invoke(struct cryptop *crp, int hint) if (crypto_timing) crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp); #endif - mtx_assert(&crypto_q_mtx, MA_OWNED); - /* Sanity checks. */ if (crp == NULL) return EINVAL; @@ -917,21 +939,45 @@ crypto_getreq(int num) void crypto_done(struct cryptop *crp) { - int wasempty; - if (crp->crp_etype != 0) cryptostats.cs_errs++; #ifdef CRYPTO_TIMING if (crypto_timing) crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp); #endif - CRYPTO_RETQ_LOCK(); - wasempty = TAILQ_EMPTY(&crp_ret_q); - TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); + if (crp->crp_flags & CRYPTO_F_CBIMM) { + /* + * Do the callback directly. This is ok when the + * callback routine does very little (e.g. the + * /dev/crypto callback method just does a wakeup). + */ +#ifdef CRYPTO_TIMING + if (crypto_timing) { + /* + * NB: We must copy the timestamp before + * doing the callback as the cryptop is + * likely to be reclaimed. + */ + struct bintime t = crp->crp_tstamp; + crypto_tstat(&cryptostats.cs_cb, &t); + crp->crp_callback(crp); + crypto_tstat(&cryptostats.cs_finis, &t); + } else +#endif + crp->crp_callback(crp); + } else { + int wasempty; + /* + * Normal case; queue the callback for the thread. + */ + CRYPTO_RETQ_LOCK(); + wasempty = TAILQ_EMPTY(&crp_ret_q); + TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next); - if (wasempty) - wakeup_one(&crp_ret_q); /* shared wait channel */ - CRYPTO_RETQ_UNLOCK(); + if (wasempty) + wakeup_one(&crp_ret_q); /* shared wait channel */ + CRYPTO_RETQ_UNLOCK(); + } } /* @@ -1043,7 +1089,7 @@ crypto_proc(void) break; } else { submit = crp; - if (submit->crp_flags & CRYPTO_F_NODELAY) + if ((submit->crp_flags & CRYPTO_F_BATCH) == 0) break; /* keep scanning for more are q'd */ } -- cgit v1.1