Add the ability to internally mark a controller as failed, if it is unable to

start or reset.  Also add a notifier for NVMe consumers for controller fail
conditions and plumb this notifier for nvd(4) to destroy the associated
GEOM disks when a failure occurs.

This requires a bit of work to cover the races when a consumer is sending
I/O requests to a controller that is transitioning to the failed state.  To
help cover this condition, add a task to defer completion of I/Os submitted
to a failed controller, so that the consumer will still always receive its
completions in a different context than the submission.

Sponsored by:	Intel
Reviewed by:	carl

1 files changed, 21 insertions, 1 deletions

diff --git a/sys/dev/nvd/nvd.c b/sys/dev/nvd/nvd.c
index 76ec5f7..09886b4 100644
--- a/sys/dev/nvd/nvd.c
+++ b/sys/dev/nvd/nvd.c
@@ -49,6 +49,7 @@ static void *nvd_new_disk(struct nvme_namespace *ns, void *ctrlr);
 static void destroy_geom_disk(struct nvd_disk *ndisk);
 
 static void *nvd_new_controller(struct nvme_controller *ctrlr);
+static void nvd_controller_fail(void *ctrlr);
 
 static int nvd_load(void);
 static void nvd_unload(void);
@@ -118,7 +119,7 @@ nvd_load()
 	TAILQ_INIT(&disk_head);
 
 	consumer_handle = nvme_register_consumer(nvd_new_disk,
-	    nvd_new_controller, NULL);
+	    nvd_new_controller, NULL, nvd_controller_fail);
 
 	return (consumer_handle != NULL ? 0 : -1);
 }
@@ -351,3 +352,22 @@ destroy_geom_disk(struct nvd_disk *ndisk)
 
 	mtx_destroy(&ndisk->bioqlock);
 }
+
+static void
+nvd_controller_fail(void *ctrlr_arg)
+{
+	struct nvd_controller	*ctrlr = ctrlr_arg;
+	struct nvd_disk		*disk;
+
+	while (!TAILQ_EMPTY(&ctrlr->disk_head)) {
+		disk = TAILQ_FIRST(&ctrlr->disk_head);
+		TAILQ_REMOVE(&disk_head, disk, global_tailq);
+		TAILQ_REMOVE(&ctrlr->disk_head, disk, ctrlr_tailq);
+		destroy_geom_disk(disk);
+		free(disk, M_NVD);
+	}
+
+	TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq);
+	free(ctrlr, M_NVD);
+}
+