Break out the code for completing an nvme_tracker object into a separate

function.

This allows for completions outside the normal completion path, for example
when an ABORT command fails due to the controller reporting the targeted
command does not exist.  This is mainly for protection against a faulty
controller, but we need to clean up our internal request nonetheless.

Sponsored by:	Intel

1 files changed, 59 insertions, 43 deletions

diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c
index 6954830..8abdce4 100644
--- a/sys/dev/nvme/nvme_qpair.c
+++ b/sys/dev/nvme/nvme_qpair.c
@@ -38,7 +38,14 @@ static void	_nvme_qpair_submit_request(struct nvme_qpair *qpair,
 					   struct nvme_request *req);
 
 static boolean_t
-nvme_completion_check_retry(const struct nvme_completion *cpl)
+nvme_completion_is_error(struct nvme_completion *cpl)
+{
+
+	return (cpl->sf_sc != 0 || cpl->sf_sct != 0);
+}
+
+static boolean_t
+nvme_completion_is_retry(const struct nvme_completion *cpl)
 {
 	/*
 	 * TODO: spec is not clear how commands that are aborted due
@@ -96,69 +103,78 @@ nvme_qpair_construct_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
 	tr->qpair = qpair;
 }
 
-void
-nvme_qpair_process_completions(struct nvme_qpair *qpair)
+static void
+nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
+    struct nvme_completion *cpl, boolean_t print_on_error)
 {
-	struct nvme_tracker	*tr;
 	struct nvme_request	*req;
-	struct nvme_completion	*cpl;
 	boolean_t		retry, error;
 
-	qpair->num_intr_handler_calls++;
+	req = tr->req;
+	error = nvme_completion_is_error(cpl);
+	retry = error && nvme_completion_is_retry(cpl);
 
-	while (1) {
-		cpl = &qpair->cpl[qpair->cq_head];
+	if (error && print_on_error) {
+		nvme_dump_completion(cpl);
+		nvme_dump_command(&req->cmd);
+	}
 
-		if (cpl->p != qpair->phase)
-			break;
+	qpair->act_tr[cpl->cid] = NULL;
 
-		tr = qpair->act_tr[cpl->cid];
-		req = tr->req;
+	KASSERT(cpl->cid == req->cmd.cid, ("cpl cid does not match cmd cid\n"));
 
-		KASSERT(tr,
-		    ("completion queue has entries but no active trackers\n"));
+	if (req->cb_fn && !retry)
+		req->cb_fn(req->cb_arg, cpl);
 
-		error = cpl->sf_sc || cpl->sf_sct;
-		retry = error && nvme_completion_check_retry(cpl);
+	mtx_lock(&qpair->lock);
+	callout_stop(&tr->timer);
 
-		if (error) {
-			nvme_dump_completion(cpl);
-			nvme_dump_command(&tr->req->cmd);
-		}
+	if (retry)
+		nvme_qpair_submit_cmd(qpair, tr);
+	else {
+		if (req->payload_size > 0 || req->uio != NULL)
+			bus_dmamap_unload(qpair->dma_tag,
+			    tr->payload_dma_map);
+
+		nvme_free_request(req);
 
-		qpair->act_tr[cpl->cid] = NULL;
+		SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist);
 
-		KASSERT(cpl->cid == req->cmd.cid,
-		    ("cpl cid does not match cmd cid\n"));
+		if (!STAILQ_EMPTY(&qpair->queued_req)) {
+			req = STAILQ_FIRST(&qpair->queued_req);
+			STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
+			_nvme_qpair_submit_request(qpair, req);
+		}
+	}
 
-		if (req->cb_fn && !retry)
-			req->cb_fn(req->cb_arg, cpl);
+	mtx_unlock(&qpair->lock);
+}
 
-		qpair->sq_head = cpl->sqhd;
+void
+nvme_qpair_process_completions(struct nvme_qpair *qpair)
+{
+	struct nvme_tracker	*tr;
+	struct nvme_completion	*cpl;
 
-		mtx_lock(&qpair->lock);
-		callout_stop(&tr->timer);
+	qpair->num_intr_handler_calls++;
 
-		if (retry)
-			nvme_qpair_submit_cmd(qpair, tr);
-		else {
-			if (req->payload_size > 0 || req->uio != NULL)
-				bus_dmamap_unload(qpair->dma_tag,
-				    tr->payload_dma_map);
+	while (1) {
+		cpl = &qpair->cpl[qpair->cq_head];
 
-			nvme_free_request(req);
+		if (cpl->p != qpair->phase)
+			break;
 
-			SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist);
+		tr = qpair->act_tr[cpl->cid];
 
-			if (!STAILQ_EMPTY(&qpair->queued_req)) {
-				req = STAILQ_FIRST(&qpair->queued_req);
-				STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq);
-				_nvme_qpair_submit_request(qpair, req);
-			}
+		if (tr != NULL) {
+			nvme_qpair_complete_tracker(qpair, tr, cpl, TRUE);
+			qpair->sq_head = cpl->sqhd;
+		} else {
+			printf("cpl does not map to outstanding cmd\n");
+			nvme_dump_completion(cpl);
+			KASSERT(0, ("received completion for unknown cmd\n"));
 		}
 
-		mtx_unlock(&qpair->lock);
-
 		if (++qpair->cq_head == qpair->num_entries) {
 			qpair->cq_head = 0;
 			qpair->phase = !qpair->phase;