Add "type" to nvme_request, signifying if its payload is a VADDR, UIO, or

NULL. This simplifies decisions around if/how requests are routed through
busdma.  It also paves the way for supporting unmapped bios.

Sponsored by:	Intel

5 files changed, 63 insertions, 37 deletions

diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c
index d0f9c32..9d10b2ee 100644
--- a/sys/dev/nvme/nvme_ctrlr.c
+++ b/sys/dev/nvme/nvme_ctrlr.c
@@ -701,7 +701,7 @@ nvme_ctrlr_construct_and_submit_aer(struct nvme_controller *ctrlr,
 	struct nvme_request *req;
 
 	aer->ctrlr = ctrlr;
-	req = nvme_allocate_request(NULL, 0, nvme_ctrlr_async_event_cb, aer);
+	req = nvme_allocate_request_null(nvme_ctrlr_async_event_cb, aer);
 	aer->req = req;
 
 	/*
diff --git a/sys/dev/nvme/nvme_ctrlr_cmd.c b/sys/dev/nvme/nvme_ctrlr_cmd.c
index 959e25e..e17b5e9 100644
--- a/sys/dev/nvme/nvme_ctrlr_cmd.c
+++ b/sys/dev/nvme/nvme_ctrlr_cmd.c
@@ -36,7 +36,7 @@ nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, void *payload,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(payload,
+	req = nvme_allocate_request_vaddr(payload,
 	    sizeof(struct nvme_controller_data), cb_fn, cb_arg);
 
 	cmd = &req->cmd;
@@ -58,7 +58,7 @@ nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr, uint16_t nsid,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(payload,
+	req = nvme_allocate_request_vaddr(payload,
 	    sizeof(struct nvme_namespace_data), cb_fn, cb_arg);
 
 	cmd = &req->cmd;
@@ -80,7 +80,7 @@ nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_CREATE_IO_CQ;
@@ -104,7 +104,7 @@ nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_CREATE_IO_SQ;
@@ -128,7 +128,7 @@ nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_DELETE_IO_CQ;
@@ -149,7 +149,7 @@ nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_DELETE_IO_SQ;
@@ -171,7 +171,7 @@ nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint8_t feature,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_SET_FEATURES;
@@ -189,7 +189,7 @@ nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint8_t feature,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_GET_FEATURES;
@@ -256,7 +256,7 @@ nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr, uint8_t log_page,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(payload, payload_size, cb_fn, cb_arg);
+	req = nvme_allocate_request_vaddr(payload, payload_size, cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_GET_LOG_PAGE;
@@ -315,7 +315,7 @@ nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
-	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_ABORT;
diff --git a/sys/dev/nvme/nvme_ns_cmd.c b/sys/dev/nvme/nvme_ns_cmd.c
index c77361a..011ce4d 100644
--- a/sys/dev/nvme/nvme_ns_cmd.c
+++ b/sys/dev/nvme/nvme_ns_cmd.c
@@ -36,7 +36,7 @@ nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba,
 	struct nvme_request	*req;
 	struct nvme_command	*cmd;
 
-	req = nvme_allocate_request(payload, lba_count*512, cb_fn, cb_arg);
+	req = nvme_allocate_request_vaddr(payload, lba_count*512, cb_fn, cb_arg);
 
 	if (req == NULL)
 		return (ENOMEM);
@@ -60,7 +60,8 @@ nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, uint64_t lba,
 	struct nvme_request	*req;
 	struct nvme_command	*cmd;
 
-	req = nvme_allocate_request(payload, lba_count*512, cb_fn, cb_arg);
+	req = nvme_allocate_request_vaddr(payload, lba_count*512, cb_fn,
+	    cb_arg);
 
 	if (req == NULL)
 		return (ENOMEM);
@@ -85,7 +86,7 @@ nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload,
 	struct nvme_request	*req;
 	struct nvme_command	*cmd;
 
-	req = nvme_allocate_request(payload,
+	req = nvme_allocate_request_vaddr(payload,
 	    num_ranges * sizeof(struct nvme_dsm_range), cb_fn, cb_arg);
 
 	if (req == NULL)
@@ -110,7 +111,7 @@ nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn, void *cb_arg)
 	struct nvme_request	*req;
 	struct nvme_command	*cmd;
 
-	req = nvme_allocate_request(NULL, 0, cb_fn, cb_arg);
+	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	if (req == NULL)
 		return (ENOMEM);
diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h
index d1cd4b1..d9946b6 100644
--- a/sys/dev/nvme/nvme_private.h
+++ b/sys/dev/nvme/nvme_private.h
@@ -123,14 +123,21 @@ struct nvme_completion_poll_status {
 	boolean_t		done;
 };
 
+#define NVME_REQUEST_VADDR	1
+#define NVME_REQUEST_NULL	2 /* For requests with no payload. */
+#define NVME_REQUEST_UIO	3
+
 struct nvme_request {
 
 	struct nvme_command		cmd;
 	struct nvme_qpair		*qpair;
-	void				*payload;
+	union {
+		void			*payload;
+		struct uio		*uio;
+	} u;
+	uint32_t			type;
 	uint32_t			payload_size;
 	boolean_t			timeout;
-	struct uio			*uio;
 	nvme_cb_fn_t			cb_fn;
 	void				*cb_arg;
 	int32_t				retries;
@@ -482,27 +489,41 @@ _nvme_allocate_request(nvme_cb_fn_t cb_fn, void *cb_arg)
 }
 
 static __inline struct nvme_request *
-nvme_allocate_request(void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn, 
-		      void *cb_arg)
+nvme_allocate_request_vaddr(void *payload, uint32_t payload_size,
+    nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 
 	req = _nvme_allocate_request(cb_fn, cb_arg);
 	if (req != NULL) {
-		req->payload = payload;
+		req->type = NVME_REQUEST_VADDR;
+		req->u.payload = payload;
 		req->payload_size = payload_size;
 	}
 	return (req);
 }
 
 static __inline struct nvme_request *
-nvme_allocate_request_uio(struct uio *uio, nvme_cb_fn_t cb_fn, void *cb_arg)
+nvme_allocate_request_null(nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 
 	req = _nvme_allocate_request(cb_fn, cb_arg);
 	if (req != NULL)
-		req->uio = uio;
+		req->type = NVME_REQUEST_NULL;
+	return (req);
+}
+
+static __inline struct nvme_request *
+nvme_allocate_request_uio(struct uio *uio, nvme_cb_fn_t cb_fn, void *cb_arg)
+{
+	struct nvme_request *req;
+
+	req = _nvme_allocate_request(cb_fn, cb_arg);
+	if (req != NULL) {
+		req->type = NVME_REQUEST_UIO;
+		req->u.uio = uio;
+	}
 	return (req);
 }
 
diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c
index 2deb4b9..62a2277 100644
--- a/sys/dev/nvme/nvme_qpair.c
+++ b/sys/dev/nvme/nvme_qpair.c
@@ -336,7 +336,7 @@ nvme_qpair_complete_tracker(struct nvme_qpair *qpair, struct nvme_tracker *tr,
 		req->retries++;
 		nvme_qpair_submit_tracker(qpair, tr);
 	} else {
-		if (req->payload_size > 0 || req->uio != NULL)
+		if (req->type != NVME_REQUEST_NULL)
 			bus_dmamap_unload(qpair->dma_tag,
 			    tr->payload_dma_map);
 
@@ -740,22 +740,26 @@ _nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req)
 	TAILQ_INSERT_TAIL(&qpair->outstanding_tr, tr, tailq);
 	tr->req = req;
 
-	if (req->uio == NULL) {
-		if (req->payload_size > 0) {
-			err = bus_dmamap_load(tr->qpair->dma_tag,
-					      tr->payload_dma_map, req->payload,
-					      req->payload_size,
-					      nvme_payload_map, tr, 0);
-			if (err != 0)
-				panic("bus_dmamap_load returned non-zero!\n");
-		} else
-			nvme_qpair_submit_tracker(tr->qpair, tr);
-	} else {
-		err = bus_dmamap_load_uio(tr->qpair->dma_tag,
-					  tr->payload_dma_map, req->uio,
-					  nvme_payload_map_uio, tr, 0);
+	switch (req->type) {
+	case NVME_REQUEST_VADDR:
+		err = bus_dmamap_load(tr->qpair->dma_tag, tr->payload_dma_map,
+		    req->u.payload, req->payload_size, nvme_payload_map, tr, 0);
 		if (err != 0)
 			panic("bus_dmamap_load returned non-zero!\n");
+		break;
+	case NVME_REQUEST_NULL:
+		nvme_qpair_submit_tracker(tr->qpair, tr);
+		break;
+	case NVME_REQUEST_UIO:
+		err = bus_dmamap_load_uio(tr->qpair->dma_tag,
+		    tr->payload_dma_map, req->u.uio, nvme_payload_map_uio,
+		    tr, 0);
+		if (err != 0)
+			panic("bus_dmamap_load_uio returned non-zero!\n");
+		break;
+	default:
+		panic("unknown nvme request type 0x%x\n", req->type);
+		break;
 	}
 }