MFC r322036 (by imp):

Make nvd vs nda choice boot-time rather than build-time

Introduce hw.nvme.use_nvd tunable. This tunable allows both nvd and
nda to be installed in the kernel, while allowing only one of them to
create devices. This is an all-or-nothing setting, and you can't
change it after boot-time. However, it will allow easier A/B testing.

6 files changed, 20 insertions, 3 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 0ef670a..62f2f86 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -86,8 +86,8 @@ cam/cam_xpt.c			optional scbus
 cam/ata/ata_all.c		optional scbus
 cam/ata/ata_xpt.c		optional scbus
 cam/ata/ata_pmp.c		optional scbus
-cam/nvme/nvme_all.c		optional scbus nvme 
-cam/nvme/nvme_da.c		optional scbus nvme da !nvd
+cam/nvme/nvme_all.c		optional scbus nvme
+cam/nvme/nvme_da.c		optional scbus nvme da
 cam/nvme/nvme_xpt.c		optional scbus nvme
 cam/scsi/scsi_xpt.c		optional scbus
 cam/scsi/scsi_all.c		optional scbus
diff --git a/sys/conf/files.amd64 b/sys/conf/files.amd64
index 404e6e2..4bbe63c 100644
--- a/sys/conf/files.amd64
+++ b/sys/conf/files.amd64
@@ -336,7 +336,7 @@ dev/nvme/nvme_ctrlr_cmd.c	optional	nvme
 dev/nvme/nvme_ns.c		optional	nvme
 dev/nvme/nvme_ns_cmd.c		optional	nvme
 dev/nvme/nvme_qpair.c		optional	nvme
-dev/nvme/nvme_sim.c		optional	nvme scbus !nvd
+dev/nvme/nvme_sim.c		optional	nvme scbus
 dev/nvme/nvme_sysctl.c		optional	nvme
 dev/nvme/nvme_test.c		optional	nvme
 dev/nvme/nvme_util.c		optional	nvme
diff --git a/sys/dev/nvd/nvd.c b/sys/dev/nvd/nvd.c
index bfb3d26..cebd36d 100644
--- a/sys/dev/nvd/nvd.c
+++ b/sys/dev/nvd/nvd.c
@@ -134,6 +134,8 @@ MODULE_DEPEND(nvd, nvme, 1, 1, 1);
 static int
 nvd_load()
 {
+	if (!nvme_use_nvd)
+		return 0;
 
 	TAILQ_INIT(&ctrlr_head);
 	TAILQ_INIT(&disk_head);
@@ -150,6 +152,9 @@ nvd_unload()
 	struct nvd_controller	*ctrlr;
 	struct nvd_disk		*disk;
 
+	if (!nvme_use_nvd)
+		return;
+
 	while (!TAILQ_EMPTY(&ctrlr_head)) {
 		ctrlr = TAILQ_FIRST(&ctrlr_head);
 		TAILQ_REMOVE(&ctrlr_head, ctrlr, tailq);
diff --git a/sys/dev/nvme/nvme.h b/sys/dev/nvme/nvme.h
index aa640b3..1939afd 100644
--- a/sys/dev/nvme/nvme.h
+++ b/sys/dev/nvme/nvme.h
@@ -1003,6 +1003,8 @@ void	nvme_ns_trim_cmd(struct nvme_command *cmd, uint16_t nsid,
 	cmd->cdw11 = NVME_DSM_ATTR_DEALLOCATE;
 }
 
+extern int nvme_use_nvd;
+
 #endif /* _KERNEL */
 
 #endif /* __NVME_H__ */
diff --git a/sys/dev/nvme/nvme_sim.c b/sys/dev/nvme/nvme_sim.c
index f2b81ec..ee6b025 100644
--- a/sys/dev/nvme/nvme_sim.c
+++ b/sys/dev/nvme/nvme_sim.c
@@ -379,6 +379,8 @@ struct nvme_consumer *consumer_cookie;
 static void
 nvme_sim_init(void)
 {
+	if (nvme_use_nvd)
+		return;
 
 	consumer_cookie = nvme_register_consumer(nvme_sim_new_ns,
 	    nvme_sim_new_controller, NULL, nvme_sim_controller_fail);
@@ -390,6 +392,8 @@ SYSINIT(nvme_sim_register, SI_SUB_DRIVERS, SI_ORDER_ANY,
 static void
 nvme_sim_uninit(void)
 {
+	if (nvme_use_nvd)
+		return;
 	/* XXX Cleanup */
 
 	nvme_unregister_consumer(consumer_cookie);
diff --git a/sys/dev/nvme/nvme_sysctl.c b/sys/dev/nvme/nvme_sysctl.c
index 44b0ab7..b76df11 100644
--- a/sys/dev/nvme/nvme_sysctl.c
+++ b/sys/dev/nvme/nvme_sysctl.c
@@ -33,6 +33,12 @@ __FBSDID("$FreeBSD$");
 
 #include "nvme_private.h"
 
+int nvme_use_nvd = 1;
+
+SYSCTL_NODE(_hw, OID_AUTO, nvme, CTLFLAG_RD, 0, "NVMe sysctl tunables");
+SYSCTL_INT(_hw_nvme, OID_AUTO, use_nvd, CTLFLAG_RDTUN,
+    &nvme_use_nvd, 1, "1 = Create NVD devices, 0 = Create NDA devices");
+
 /*
  * CTLTYPE_S64 and sysctl_handle_64 were added in r217616.  Define these
  *  explicitly here for older kernels that don't include the r217616