MFC r268291:

Create separate CTL port for every iSCSI target (and maybe portal group).

Having single port for all iSCSI connections makes problematic implementing
some more advanced SCSI functionality in CTL, that require proper ports
enumeration and identification.

This change extends CTL iSCSI API, making ctld daemon to control list of
iSCSI ports in CTL.  When new target is defined in config fine, ctld will
create respective port in CTL.  When target is removed -- port will be
also removed after all active commands through that port properly aborted.
This change require ctld to be rebuilt to match the kernel.

As a minor side effect, this allows to have iSCSI targets without LUNs.
While that may look odd and not very useful, that is not incorrect.

3 files changed, 288 insertions, 52 deletions

diff --git a/usr.sbin/ctld/ctld.c b/usr.sbin/ctld/ctld.c
index 2cda316..e2b9ebe 100644
--- a/usr.sbin/ctld/ctld.c
+++ b/usr.sbin/ctld/ctld.c
@@ -1120,7 +1120,6 @@ conf_verify(struct conf *conf)
 		if (!found_lun) {
 			log_warnx("no LUNs defined for target \"%s\"",
 			    targ->t_name);
-			return (1);
 		}
 	}
 	TAILQ_FOREACH(pg, &conf->conf_portal_groups, pg_next) {
@@ -1209,19 +1208,6 @@ conf_apply(struct conf *oldconf, struct conf *newconf)
 		}
 	}
 
-	if (oldconf->conf_kernel_port_on != newconf->conf_kernel_port_on) {
-		if (newconf->conf_kernel_port_on == true) {
-			log_debugx("enabling CTL iSCSI port");
-			error = kernel_port_on();
-			if (error != 0)
-				log_errx(1, "failed to enable CTL iSCSI port; exiting");
-		} else {
-			error = kernel_port_off();
-			if (error != 0)
-				log_warnx("failed to disable CTL iSCSI port");
-		}
-	}
-
 	/*
 	 * XXX: If target or lun removal fails, we should somehow "move"
 	 * 	the old lun or target into newconf, so that subsequent
@@ -1253,6 +1239,7 @@ conf_apply(struct conf *oldconf, struct conf *newconf)
 				}
 				lun_delete(oldlun);
 			}
+			kernel_port_remove(oldtarg);
 			target_delete(oldtarg);
 			continue;
 		}
@@ -1387,6 +1374,8 @@ conf_apply(struct conf *oldconf, struct conf *newconf)
 				cumulated_error++;
 			}
 		}
+		if (oldtarg == NULL)
+			kernel_port_add(newtarg);
 	}
 
 	/*
diff --git a/usr.sbin/ctld/ctld.h b/usr.sbin/ctld/ctld.h
index 3399648..5f7aea6 100644
--- a/usr.sbin/ctld/ctld.h
+++ b/usr.sbin/ctld/ctld.h
@@ -272,8 +272,8 @@ int			kernel_lun_add(struct lun *lun);
 int			kernel_lun_resize(struct lun *lun);
 int			kernel_lun_remove(struct lun *lun);
 void			kernel_handoff(struct connection *conn);
-int			kernel_port_on(void);
-int			kernel_port_off(void);
+int			kernel_port_add(struct target *targ);
+int			kernel_port_remove(struct target *targ);
 void			kernel_capsicate(void);
 
 #ifdef ICL_KERNEL_PROXY
diff --git a/usr.sbin/ctld/kernel.c b/usr.sbin/ctld/kernel.c
index a8799f8..0225691 100644
--- a/usr.sbin/ctld/kernel.c
+++ b/usr.sbin/ctld/kernel.c
@@ -119,10 +119,21 @@ struct cctl_lun {
 	STAILQ_ENTRY(cctl_lun) links;
 };
 
+struct cctl_port {
+	uint32_t port_id;
+	char *cfiscsi_target;
+	uint16_t cfiscsi_portal_group_tag;
+	STAILQ_HEAD(,cctl_lun_nv) attr_list;
+	STAILQ_ENTRY(cctl_port) links;
+};
+
 struct cctl_devlist_data {
 	int num_luns;
 	STAILQ_HEAD(,cctl_lun) lun_list;
 	struct cctl_lun *cur_lun;
+	int num_ports;
+	STAILQ_HEAD(,cctl_port) port_list;
+	struct cctl_port *cur_port;
 	int level;
 	struct sbuf *cur_sb[32];
 };
@@ -247,6 +258,109 @@ cctl_end_element(void *user_data, const char *name)
 }
 
 static void
+cctl_start_pelement(void *user_data, const char *name, const char **attr)
+{
+	int i;
+	struct cctl_devlist_data *devlist;
+	struct cctl_port *cur_port;
+
+	devlist = (struct cctl_devlist_data *)user_data;
+	cur_port = devlist->cur_port;
+	devlist->level++;
+	if ((u_int)devlist->level >= (sizeof(devlist->cur_sb) /
+	    sizeof(devlist->cur_sb[0])))
+		log_errx(1, "%s: too many nesting levels, %zd max", __func__,
+		     sizeof(devlist->cur_sb) / sizeof(devlist->cur_sb[0]));
+
+	devlist->cur_sb[devlist->level] = sbuf_new_auto();
+	if (devlist->cur_sb[devlist->level] == NULL)
+		log_err(1, "%s: unable to allocate sbuf", __func__);
+
+	if (strcmp(name, "targ_port") == 0) {
+		if (cur_port != NULL)
+			log_errx(1, "%s: improper port element nesting (%s)",
+			    __func__, name);
+
+		cur_port = calloc(1, sizeof(*cur_port));
+		if (cur_port == NULL)
+			log_err(1, "%s: cannot allocate %zd bytes", __func__,
+			    sizeof(*cur_port));
+
+		devlist->num_ports++;
+		devlist->cur_port = cur_port;
+
+		STAILQ_INIT(&cur_port->attr_list);
+		STAILQ_INSERT_TAIL(&devlist->port_list, cur_port, links);
+
+		for (i = 0; attr[i] != NULL; i += 2) {
+			if (strcmp(attr[i], "id") == 0) {
+				cur_port->port_id = strtoul(attr[i+1], NULL, 0);
+			} else {
+				log_errx(1, "%s: invalid LUN attribute %s = %s",
+				     __func__, attr[i], attr[i+1]);
+			}
+		}
+	}
+}
+
+static void
+cctl_end_pelement(void *user_data, const char *name)
+{
+	struct cctl_devlist_data *devlist;
+	struct cctl_port *cur_port;
+	char *str;
+
+	devlist = (struct cctl_devlist_data *)user_data;
+	cur_port = devlist->cur_port;
+
+	if ((cur_port == NULL)
+	 && (strcmp(name, "ctlportlist") != 0))
+		log_errx(1, "%s: cur_port == NULL! (name = %s)", __func__, name);
+
+	if (devlist->cur_sb[devlist->level] == NULL)
+		log_errx(1, "%s: no valid sbuf at level %d (name %s)", __func__,
+		     devlist->level, name);
+
+	sbuf_finish(devlist->cur_sb[devlist->level]);
+	str = checked_strdup(sbuf_data(devlist->cur_sb[devlist->level]));
+
+	if (strlen(str) == 0) {
+		free(str);
+		str = NULL;
+	}
+
+	sbuf_delete(devlist->cur_sb[devlist->level]);
+	devlist->cur_sb[devlist->level] = NULL;
+	devlist->level--;
+
+	if (strcmp(name, "cfiscsi_target") == 0) {
+		cur_port->cfiscsi_target = str;
+		str = NULL;
+	} else if (strcmp(name, "cfiscsi_portal_group_tag") == 0) {
+		cur_port->cfiscsi_portal_group_tag = strtoul(str, NULL, 0);
+	} else if (strcmp(name, "targ_port") == 0) {
+		devlist->cur_port = NULL;
+	} else if (strcmp(name, "ctlportlist") == 0) {
+		
+	} else {
+		struct cctl_lun_nv *nv;
+
+		nv = calloc(1, sizeof(*nv));
+		if (nv == NULL)
+			log_err(1, "%s: can't allocate %zd bytes for nv pair",
+			    __func__, sizeof(*nv));
+
+		nv->name = checked_strdup(name);
+
+		nv->value = str;
+		str = NULL;
+		STAILQ_INSERT_TAIL(&cur_port->attr_list, nv, links);
+	}
+
+	free(str);
+}
+
+static void
 cctl_char_handler(void *user_data, const XML_Char *str, int len)
 {
 	struct cctl_devlist_data *devlist;
@@ -266,50 +380,51 @@ conf_new_from_kernel(void)
 	struct ctl_lun_list list;
 	struct cctl_devlist_data devlist;
 	struct cctl_lun *lun;
+	struct cctl_port *port;
 	XML_Parser parser;
-	char *lun_str = NULL;
-	int lun_len;
-	int retval;
-
-	lun_len = 4096;
+	char *str;
+	int len, retval;
 
 	bzero(&devlist, sizeof(devlist));
 	STAILQ_INIT(&devlist.lun_list);
+	STAILQ_INIT(&devlist.port_list);
 
 	log_debugx("obtaining previously configured CTL luns from the kernel");
 
+	str = NULL;
+	len = 4096;
 retry:
-	lun_str = realloc(lun_str, lun_len);
-	if (lun_str == NULL)
+	str = realloc(str, len);
+	if (str == NULL)
 		log_err(1, "realloc");
 
 	bzero(&list, sizeof(list));
-	list.alloc_len = lun_len;
+	list.alloc_len = len;
 	list.status = CTL_LUN_LIST_NONE;
-	list.lun_xml = lun_str;
+	list.lun_xml = str;
 
 	if (ioctl(ctl_fd, CTL_LUN_LIST, &list) == -1) {
 		log_warn("error issuing CTL_LUN_LIST ioctl");
-		free(lun_str);
+		free(str);
 		return (NULL);
 	}
 
 	if (list.status == CTL_LUN_LIST_ERROR) {
 		log_warnx("error returned from CTL_LUN_LIST ioctl: %s",
 		    list.error_str);
-		free(lun_str);
+		free(str);
 		return (NULL);
 	}
 
 	if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
-		lun_len = lun_len << 1;
+		len = len << 1;
 		goto retry;
 	}
 
 	parser = XML_ParserCreate(NULL);
 	if (parser == NULL) {
 		log_warnx("unable to create XML parser");
-		free(lun_str);
+		free(str);
 		return (NULL);
 	}
 
@@ -317,9 +432,58 @@ retry:
 	XML_SetElementHandler(parser, cctl_start_element, cctl_end_element);
 	XML_SetCharacterDataHandler(parser, cctl_char_handler);
 
-	retval = XML_Parse(parser, lun_str, strlen(lun_str), 1);
+	retval = XML_Parse(parser, str, strlen(str), 1);
+	XML_ParserFree(parser);
+	free(str);
+	if (retval != 1) {
+		log_warnx("XML_Parse failed");
+		return (NULL);
+	}
+
+	str = NULL;
+	len = 4096;
+retry_port:
+	str = realloc(str, len);
+	if (str == NULL)
+		log_err(1, "realloc");
+
+	bzero(&list, sizeof(list));
+	list.alloc_len = len;
+	list.status = CTL_LUN_LIST_NONE;
+	list.lun_xml = str;
+
+	if (ioctl(ctl_fd, CTL_PORT_LIST, &list) == -1) {
+		log_warn("error issuing CTL_PORT_LIST ioctl");
+		free(str);
+		return (NULL);
+	}
+
+	if (list.status == CTL_PORT_LIST_ERROR) {
+		log_warnx("error returned from CTL_PORT_LIST ioctl: %s",
+		    list.error_str);
+		free(str);
+		return (NULL);
+	}
+
+	if (list.status == CTL_LUN_LIST_NEED_MORE_SPACE) {
+		len = len << 1;
+		goto retry_port;
+	}
+
+	parser = XML_ParserCreate(NULL);
+	if (parser == NULL) {
+		log_warnx("unable to create XML parser");
+		free(str);
+		return (NULL);
+	}
+
+	XML_SetUserData(parser, &devlist);
+	XML_SetElementHandler(parser, cctl_start_pelement, cctl_end_pelement);
+	XML_SetCharacterDataHandler(parser, cctl_char_handler);
+
+	retval = XML_Parse(parser, str, strlen(str), 1);
 	XML_ParserFree(parser);
-	free(lun_str);
+	free(str);
 	if (retval != 1) {
 		log_warnx("XML_Parse failed");
 		return (NULL);
@@ -327,6 +491,28 @@ retry:
 
 	conf = conf_new();
 
+	STAILQ_FOREACH(port, &devlist.port_list, links) {
+
+		if (port->cfiscsi_target == NULL) {
+			log_debugx("CTL port %ju wasn't managed by ctld; "
+			    "ignoring", (uintmax_t)port->port_id);
+			continue;
+		}
+
+		targ = target_find(conf, port->cfiscsi_target);
+		if (targ == NULL) {
+#if 0
+			log_debugx("found new kernel target %s for CTL port %ld",
+			    port->cfiscsi_target, port->port_id);
+#endif
+			targ = target_new(conf, port->cfiscsi_target);
+			if (targ == NULL) {
+				log_warnx("target_new failed");
+				continue;
+			}
+		}
+	}
+
 	STAILQ_FOREACH(lun, &devlist.lun_list, links) {
 		struct cctl_lun_nv *nv;
 
@@ -391,6 +577,17 @@ retry:
 	return (conf);
 }
 
+static void
+str_arg(struct ctl_be_arg *arg, const char *name, const char *value)
+{
+
+	arg->namelen = strlen(name) + 1;
+	arg->name = __DECONST(char *, name);
+	arg->vallen = strlen(value) + 1;
+	arg->value = __DECONST(char *, value);
+	arg->flags = CTL_BEARG_ASCII | CTL_BEARG_RD;
+}
+
 int
 kernel_lun_add(struct lun *lun)
 {
@@ -482,14 +679,7 @@ kernel_lun_add(struct lun *lun)
 
 		i = 0;
 		TAILQ_FOREACH(lo, &lun->l_options, lo_next) {
-			 /*
-			  * +1 for the terminating '\0'
-			  */
-			req.be_args[i].namelen = strlen(lo->lo_name) + 1;
-			req.be_args[i].name = lo->lo_name;
-			req.be_args[i].vallen = strlen(lo->lo_value) + 1;
-			req.be_args[i].value = lo->lo_value;
-			req.be_args[i].flags = CTL_BEARG_ASCII | CTL_BEARG_RD;
+			str_arg(&req.be_args[i], lo->lo_name, lo->lo_value);
 			i++;
 		}
 		assert(i == num_options);
@@ -634,15 +824,50 @@ kernel_handoff(struct connection *conn)
 }
 
 int
-kernel_port_on(void)
+kernel_port_add(struct target *targ)
 {
 	struct ctl_port_entry entry;
+	struct ctl_req req;
+	char tagstr[16];
 	int error;
+	uint32_t port_id = -1;
 
-	bzero(&entry, sizeof(entry));
+	bzero(&req, sizeof(req));
+	strlcpy(req.driver, "iscsi", sizeof(req.driver));
+	req.reqtype = CTL_REQ_CREATE;
+	req.num_args = 4;
+	req.args = malloc(req.num_args * sizeof(*req.args));
+	req.args[0].namelen = sizeof("port_id");
+	req.args[0].name = __DECONST(char *, "port_id");
+	req.args[0].vallen = sizeof(port_id);
+	req.args[0].value = &port_id;
+	req.args[0].flags = CTL_BEARG_WR;
+	str_arg(&req.args[1], "cfiscsi_target", targ->t_name);
+	str_arg(&req.args[2], "cfiscsi_target_alias", targ->t_alias);
+	snprintf(tagstr, sizeof(tagstr), "%d", targ->t_portal_group->pg_tag);
+	str_arg(&req.args[3], "cfiscsi_portal_group_tag", tagstr);
+
+	error = ioctl(ctl_fd, CTL_PORT_REQ, &req);
+	free(req.args);
+	if (error != 0) {
+		log_warn("error issuing CTL_PORT_REQ ioctl");
+		return (1);
+	}
+
+	if (req.status == CTL_LUN_ERROR) {
+		log_warnx("error returned from port creation request: %s",
+		    req.error_str);
+		return (1);
+	}
 
-	entry.port_type = CTL_PORT_ISCSI;
-	entry.targ_port = -1;
+	if (req.status != CTL_LUN_OK) {
+		log_warnx("unknown port creation request status %d",
+		    req.status);
+		return (1);
+	}
+
+	bzero(&entry, sizeof(entry));
+	entry.targ_port = port_id;
 
 	error = ioctl(ctl_fd, CTL_ENABLE_PORT, &entry);
 	if (error != 0) {
@@ -654,20 +879,42 @@ kernel_port_on(void)
 }
 
 int
-kernel_port_off(void)
+kernel_port_remove(struct target *targ)
 {
-	struct ctl_port_entry entry;
+	struct ctl_req req;
+	char tagstr[16];
 	int error;
 
-	bzero(&entry, sizeof(entry));
+	bzero(&req, sizeof(req));
+	strlcpy(req.driver, "iscsi", sizeof(req.driver));
+	req.reqtype = CTL_REQ_REMOVE;
+	req.num_args = 2;
+	req.args = malloc(req.num_args * sizeof(*req.args));
+	str_arg(&req.args[0], "cfiscsi_target", targ->t_name);
+	if (targ->t_portal_group) {
+		snprintf(tagstr, sizeof(tagstr), "%d",
+		    targ->t_portal_group->pg_tag);
+		str_arg(&req.args[1], "cfiscsi_portal_group_tag", tagstr);
+	} else
+		req.num_args--;
+
+	error = ioctl(ctl_fd, CTL_PORT_REQ, &req);
+	free(req.args);
+	if (error != 0) {
+		log_warn("error issuing CTL_PORT_REQ ioctl");
+		return (1);
+	}
 
-	entry.port_type = CTL_PORT_ISCSI;
-	entry.targ_port = -1;
+	if (req.status == CTL_LUN_ERROR) {
+		log_warnx("error returned from port removal request: %s",
+		    req.error_str);
+		return (1);
+	}
 
-	error = ioctl(ctl_fd, CTL_DISABLE_PORT, &entry);
-	if (error != 0) {
-		log_warn("CTL_DISABLE_PORT ioctl failed");
-		return (-1);
+	if (req.status != CTL_LUN_OK) {
+		log_warnx("unknown port removal request status %d",
+		    req.status);
+		return (1);
 	}
 
 	return (0);