- Eliminate the need for hio_nv.

- Introduce hio_clear() function for clearing hio before returning it
  onto free queue.

MFC after:	3 days

1 files changed, 55 insertions, 25 deletions

diff --git a/sbin/hastd/secondary.c b/sbin/hastd/secondary.c
index 7a6d533..75529f3 100644
--- a/sbin/hastd/secondary.c
+++ b/sbin/hastd/secondary.c
@@ -67,7 +67,6 @@ __FBSDID("$FreeBSD$");
 struct hio {
 	uint64_t	 hio_seq;
 	int		 hio_error;
-	struct nv	*hio_nv;
 	void		*hio_data;
 	uint8_t		 hio_cmd;
 	uint64_t	 hio_offset;
@@ -126,6 +125,17 @@ static void *send_thread(void *arg);
 	TAILQ_REMOVE(&hio_##name##_list, (hio), hio_next);		\
 	mtx_unlock(&hio_##name##_list_lock);				\
 } while (0)
+ 
+static void
+hio_clear(struct hio *hio)
+{
+
+	hio->hio_seq = 0;
+	hio->hio_error = 0;
+	hio->hio_cmd = HIO_UNDEF;
+	hio->hio_offset = 0;
+	hio->hio_length = 0;
+}
 
 static void
 init_environment(void)
@@ -156,13 +166,13 @@ init_environment(void)
 			    "Unable to allocate memory (%zu bytes) for hio request.",
 			    sizeof(*hio));
 		}
-		hio->hio_error = 0;
 		hio->hio_data = malloc(MAXPHYS);
 		if (hio->hio_data == NULL) {
 			pjdlog_exitx(EX_TEMPFAIL,
 			    "Unable to allocate memory (%zu bytes) for gctl_data.",
 			    (size_t)MAXPHYS);
 		}
+		hio_clear(hio);
 		TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_next);
 	}
 }
@@ -268,6 +278,7 @@ init_remote(struct hast_resource *res, struct nv *nvin)
 	} else if (res->hr_resuid != resuid) {
 		char errmsg[256];
 
+		free(map);
 		(void)snprintf(errmsg, sizeof(errmsg),
 		    "Resource unique ID mismatch (primary=%ju, secondary=%ju).",
 		    (uintmax_t)resuid, (uintmax_t)res->hr_resuid);
@@ -315,11 +326,17 @@ init_remote(struct hast_resource *res, struct nv *nvin)
 		/*
 		 * Not good, we have split-brain condition.
 		 */
+		free(map);
 		pjdlog_error("Split-brain detected, exiting.");
 		nv_add_string(nvout, "Split-brain condition!", "errmsg");
-		free(map);
-		map = NULL;
-		mapsize = 0;
+		if (hast_proto_send(res, res->hr_remotein, nvout, NULL, 0) < 0) {
+			pjdlog_exit(EX_TEMPFAIL, "Unable to send response to %s",
+			    res->hr_remoteaddr);
+		}
+		nv_free(nvout);
+		/* Exit on split-brain. */
+		event_send(res, EVENT_SPLITBRAIN);
+		exit(EX_CONFIG);
 	} else /* if (res->hr_secondary_localcnt < res->hr_primary_remotecnt ||
 	    res->hr_primary_localcnt < res->hr_secondary_remotecnt) */ {
 		/*
@@ -358,12 +375,6 @@ init_remote(struct hast_resource *res, struct nv *nvin)
 	if (proto_recv(res->hr_remotein, NULL, 0) == -1)
 		pjdlog_errno(LOG_WARNING, "Unable to set connection direction");
 #endif
-	if (res->hr_secondary_localcnt > res->hr_primary_remotecnt &&
-	     res->hr_primary_localcnt > res->hr_secondary_remotecnt) {
-		/* Exit on split-brain. */
-		event_send(res, EVENT_SPLITBRAIN);
-		exit(EX_CONFIG);
-	}
 }
 
 void
@@ -508,15 +519,23 @@ reqlog(int loglevel, int debuglevel, int error, struct hio *hio, const char *fmt
 }
 
 static int
-requnpack(struct hast_resource *res, struct hio *hio)
+requnpack(struct hast_resource *res, struct hio *hio, struct nv *nv)
 {
 
-	hio->hio_cmd = nv_get_uint8(hio->hio_nv, "cmd");
+	hio->hio_cmd = nv_get_uint8(nv, "cmd");
 	if (hio->hio_cmd == 0) {
 		pjdlog_error("Header contains no 'cmd' field.");
 		hio->hio_error = EINVAL;
 		goto end;
 	}
+	if (hio->hio_cmd != HIO_KEEPALIVE) {
+		hio->hio_seq = nv_get_uint64(nv, "seq");
+		if (hio->hio_seq == 0) {
+			pjdlog_error("Header contains no 'seq' field.");
+			hio->hio_error = EINVAL;
+			goto end;
+		}
+	}
 	switch (hio->hio_cmd) {
 	case HIO_FLUSH:
 	case HIO_KEEPALIVE:
@@ -524,14 +543,14 @@ requnpack(struct hast_resource *res, struct hio *hio)
 	case HIO_READ:
 	case HIO_WRITE:
 	case HIO_DELETE:
-		hio->hio_offset = nv_get_uint64(hio->hio_nv, "offset");
-		if (nv_error(hio->hio_nv) != 0) {
+		hio->hio_offset = nv_get_uint64(nv, "offset");
+		if (nv_error(nv) != 0) {
 			pjdlog_error("Header is missing 'offset' field.");
 			hio->hio_error = EINVAL;
 			goto end;
 		}
-		hio->hio_length = nv_get_uint64(hio->hio_nv, "length");
-		if (nv_error(hio->hio_nv) != 0) {
+		hio->hio_length = nv_get_uint64(nv, "length");
+		if (nv_error(nv) != 0) {
 			pjdlog_error("Header is missing 'length' field.");
 			hio->hio_error = EINVAL;
 			goto end;
@@ -600,16 +619,18 @@ recv_thread(void *arg)
 {
 	struct hast_resource *res = arg;
 	struct hio *hio;
+	struct nv *nv;
 
 	for (;;) {
 		pjdlog_debug(2, "recv: Taking free request.");
 		QUEUE_TAKE(free, hio);
 		pjdlog_debug(2, "recv: (%p) Got request.", hio);
-		if (hast_proto_recv_hdr(res->hr_remotein, &hio->hio_nv) < 0) {
+		if (hast_proto_recv_hdr(res->hr_remotein, &nv) < 0) {
 			secondary_exit(EX_TEMPFAIL,
 			    "Unable to receive request header");
 		}
-		if (requnpack(res, hio) != 0) {
+		if (requnpack(res, hio, nv) != 0) {
+			nv_free(nv);
 			pjdlog_debug(2,
 			    "recv: (%p) Moving request to the send queue.",
 			    hio);
@@ -629,23 +650,30 @@ recv_thread(void *arg)
 		case HIO_FLUSH:
 			res->hr_stat_flush++;
 			break;
+		case HIO_KEEPALIVE:
+			break;
+		default:
+			PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
+			    hio->hio_cmd);
 		}
 		reqlog(LOG_DEBUG, 2, -1, hio,
 		    "recv: (%p) Got request header: ", hio);
 		if (hio->hio_cmd == HIO_KEEPALIVE) {
+			nv_free(nv);
 			pjdlog_debug(2,
 			    "recv: (%p) Moving request to the free queue.",
 			    hio);
-			nv_free(hio->hio_nv);
+			hio_clear(hio);
 			QUEUE_INSERT(free, hio);
 			continue;
 		} else if (hio->hio_cmd == HIO_WRITE) {
-			if (hast_proto_recv_data(res, res->hr_remotein,
-			    hio->hio_nv, hio->hio_data, MAXPHYS) < 0) {
+			if (hast_proto_recv_data(res, res->hr_remotein, nv,
+			    hio->hio_data, MAXPHYS) < 0) {
 				secondary_exit(EX_TEMPFAIL,
 				    "Unable to receive request data");
 			}
 		}
+		nv_free(nv);
 		pjdlog_debug(2, "recv: (%p) Moving request to the disk queue.",
 		    hio);
 		QUEUE_INSERT(disk, hio);
@@ -752,6 +780,9 @@ disk_thread(void *arg)
 				hio->hio_error = 0;
 			}
 			break;
+		default:
+			PJDLOG_ABORT("Unexpected command (cmd=%hhu).",
+			    hio->hio_cmd);
 		}
 		if (logerror && hio->hio_error != 0) {
 			reqlog(LOG_ERR, 0, hio->hio_error, hio,
@@ -783,7 +814,7 @@ send_thread(void *arg)
 		reqlog(LOG_DEBUG, 2, -1, hio, "send: (%p) Got request: ", hio);
 		nvout = nv_alloc();
 		/* Copy sequence number. */
-		nv_add_uint64(nvout, nv_get_uint64(hio->hio_nv, "seq"), "seq");
+		nv_add_uint64(nvout, hio->hio_seq, "seq");
 		switch (hio->hio_cmd) {
 		case HIO_READ:
 			if (hio->hio_error == 0) {
@@ -814,8 +845,7 @@ send_thread(void *arg)
 		nv_free(nvout);
 		pjdlog_debug(2, "send: (%p) Moving request to the free queue.",
 		    hio);
-		nv_free(hio->hio_nv);
-		hio->hio_error = 0;
+		hio_clear(hio);
 		QUEUE_INSERT(free, hio);
 	}
 	/* NOTREACHED */