diff options
| -rw-r--r-- | sys/dev/nvd/nvd.c | 22 | ||||
| -rw-r--r-- | sys/dev/nvme/nvme.h | 3 | ||||
| -rw-r--r-- | sys/dev/nvme/nvme_private.h | 3 | ||||
| -rw-r--r-- | sys/dev/nvme/nvme_qpair.c | 38 |
4 files changed, 34 insertions, 32 deletions
diff --git a/sys/dev/nvd/nvd.c b/sys/dev/nvd/nvd.c index 889401b..d221375 100644 --- a/sys/dev/nvd/nvd.c +++ b/sys/dev/nvd/nvd.c @@ -162,8 +162,7 @@ nvd_done(void *arg, const struct nvme_completion *status) ndisk = bp->bio_disk->d_drv1; - if (atomic_fetchadd_int(&ndisk->cur_depth, -1) == NVME_QD) - taskqueue_enqueue(ndisk->tq, &ndisk->bioqtask); + atomic_add_int(&ndisk->cur_depth, -1); /* * TODO: add more extensive translation of NVMe status codes @@ -187,9 +186,6 @@ nvd_bioq_process(void *arg, int pending) int err; for (;;) { - if (atomic_load_acq_int(&ndisk->cur_depth) >= NVME_QD) - break; - mtx_lock(&ndisk->bioqlock); bp = bioq_takefirst(&ndisk->bioq); mtx_unlock(&ndisk->bioqlock); @@ -210,24 +206,12 @@ nvd_bioq_process(void *arg, int pending) #endif bp->bio_driver1 = NULL; - atomic_add_acq_int(&ndisk->cur_depth, 1); + atomic_add_int(&ndisk->cur_depth, 1); err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done); - /* - * TODO: remove this loop and rely on GEOM's pacing once - * nvme(4) returns ENOMEM only for malloc() failures. - * Currently nvme(4) returns ENOMEM also for cases when - * the submission queue is completely full, and that case - * will be handled more elegantly in a future update. - */ - while (err == ENOMEM) { - pause("nvd enomem", 1); - err = nvme_ns_bio_process(ndisk->ns, bp, nvd_done); - } - if (err) { - atomic_add_acq_int(&ndisk->cur_depth, -1); + atomic_add_int(&ndisk->cur_depth, -1); bp->bio_error = err; bp->bio_flags |= BIO_ERROR; bp->bio_resid = bp->bio_bcount; diff --git a/sys/dev/nvme/nvme.h b/sys/dev/nvme/nvme.h index cc2d599..ed1098a 100644 --- a/sys/dev/nvme/nvme.h +++ b/sys/dev/nvme/nvme.h @@ -689,9 +689,6 @@ enum nvme_io_test_flags { struct bio; -/* TODO: reassess this QD variable - its a workaround for Chatham2 issue */ -#define NVME_QD (200) - struct nvme_namespace; struct nvme_consumer; diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h index 0797b53..4f57f7c 100644 --- a/sys/dev/nvme/nvme_private.h +++ b/sys/dev/nvme/nvme_private.h @@ -115,7 +115,7 @@ struct nvme_request { struct uio *uio; nvme_cb_fn_t cb_fn; void *cb_arg; - SLIST_ENTRY(nvme_request) slist; + STAILQ_ENTRY(nvme_request) stailq; }; struct nvme_tracker { @@ -168,6 +168,7 @@ struct nvme_qpair { uint64_t cpl_bus_addr; SLIST_HEAD(, nvme_tracker) free_tr; + STAILQ_HEAD(, nvme_request) queued_req; struct nvme_tracker **act_tr; diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c index 0abac7c..1eac48d 100644 --- a/sys/dev/nvme/nvme_qpair.c +++ b/sys/dev/nvme/nvme_qpair.c @@ -30,6 +30,8 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/bus.h> +#include <dev/pci/pcivar.h> + #include "nvme_private.h" static boolean_t @@ -142,10 +144,13 @@ nvme_qpair_process_completions(struct nvme_qpair *qpair) nvme_free_request(req); - if (SLIST_EMPTY(&qpair->free_tr)) - wakeup(qpair); - SLIST_INSERT_HEAD(&qpair->free_tr, tr, slist); + + if (!STAILQ_EMPTY(&qpair->queued_req)) { + req = STAILQ_FIRST(&qpair->queued_req); + STAILQ_REMOVE_HEAD(&qpair->queued_req, stailq); + nvme_qpair_submit_request(qpair, req); + } } mtx_unlock(&qpair->lock); @@ -179,6 +184,16 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, qpair->id = id; qpair->vector = vector; qpair->num_entries = num_entries; +#ifdef CHATHAM2 + /* + * Chatham prototype board starts having issues at higher queue + * depths. So use a conservative estimate here of no more than 64 + * outstanding I/O per queue at any one point. + */ + if (pci_get_devid(ctrlr->dev) == CHATHAM_PCI_ID) + num_trackers = min(num_trackers, 64); +#endif + qpair->num_trackers = num_trackers; qpair->max_xfer_size = max_xfer_size; qpair->ctrlr = ctrlr; @@ -217,7 +232,6 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, qpair->num_cmds = 0; qpair->num_intr_handler_calls = 0; - qpair->num_trackers = num_trackers; qpair->sq_head = qpair->sq_tail = qpair->cq_head = 0; /* TODO: error checking on contigmalloc, bus_dmamap_load calls */ @@ -242,8 +256,9 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id, qpair->cq_hdbl_off = nvme_mmio_offsetof(doorbell[id].cq_hdbl); SLIST_INIT(&qpair->free_tr); + STAILQ_INIT(&qpair->queued_req); - for (i = 0; i < num_trackers; i++) { + for (i = 0; i < qpair->num_trackers; i++) { tr = malloc(sizeof(*tr), M_NVME, M_ZERO | M_NOWAIT); if (tr == NULL) { @@ -400,10 +415,14 @@ nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) tr = SLIST_FIRST(&qpair->free_tr); - while (tr == NULL) { - msleep(qpair, &qpair->lock, PRIBIO, "qpair_tr", 0); - printf("msleep\n"); - tr = SLIST_FIRST(&qpair->free_tr); + if (tr == NULL) { + /* + * No tracker is available. Put the request on the qpair's + * request queue to be processed when a tracker frees up + * via a command completion. + */ + STAILQ_INSERT_TAIL(&qpair->queued_req, req, stailq); + goto ret; } SLIST_REMOVE_HEAD(&qpair->free_tr, slist); @@ -427,5 +446,6 @@ nvme_qpair_submit_request(struct nvme_qpair *qpair, struct nvme_request *req) panic("bus_dmamap_load returned non-zero!\n"); } +ret: mtx_unlock(&qpair->lock); } |
