From cfbb7640a74a9b74f45fb2fcd3bc27b11f1476e4 Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Mon, 5 Oct 2015 10:40:15 +0000
Subject: MFC r287957: Kill HA link and shutdown the threads on shutdown.

---
 sys/cam/ctl/ctl.c    | 11 +++++------
 sys/cam/ctl/ctl_ha.c | 45 +++++++++++++++++++++++++++++++++++++++++----
 sys/cam/ctl/ctl_ha.h |  3 ++-
 3 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c
index 47401d6..b2036e4 100644
--- a/sys/cam/ctl/ctl.c
+++ b/sys/cam/ctl/ctl.c
@@ -1583,13 +1583,12 @@ ctl_shutdown(void)
 	softc = (struct ctl_softc *)control_softc;
 
 	if (softc->is_single == 0) {
+		ctl_ha_msg_shutdown(softc);
 		if (ctl_ha_msg_deregister(CTL_HA_CHAN_CTL)
-		    != CTL_HA_STATUS_SUCCESS) {
-			printf("ctl_shutdown: ctl_ha_msg_deregister failed.\n");
-		}
-		if (ctl_ha_msg_shutdown(softc) != CTL_HA_STATUS_SUCCESS) {
-			printf("ctl_shutdown: ctl_ha_msg_shutdown failed.\n");
-		}
+		    != CTL_HA_STATUS_SUCCESS)
+			printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
+		if (ctl_ha_msg_destroy(softc) != CTL_HA_STATUS_SUCCESS)
+			printf("%s: ctl_ha_msg_destroy failed.\n", __func__);
 		ctl_frontend_deregister(&ha_frontend);
 	}
 
diff --git a/sys/cam/ctl/ctl_ha.c b/sys/cam/ctl/ctl_ha.c
index 952e92a..13cb2e1 100644
--- a/sys/cam/ctl/ctl_ha.c
+++ b/sys/cam/ctl/ctl_ha.c
@@ -155,6 +155,8 @@ struct ha_softc {
 	int		 ha_receiving;
 	int		 ha_wakeup;
 	int		 ha_disconnect;
+	int		 ha_shutdown;
+	eventhandler_tag ha_shutdown_eh;
 	TAILQ_HEAD(, ctl_ha_dt_req) ha_dts;
 } ha_softc;
 
@@ -568,10 +570,12 @@ ctl_ha_conn_thread(void *arg)
 	int error;
 
 	while (1) {
-		if (softc->ha_disconnect) {
+		if (softc->ha_disconnect || softc->ha_shutdown) {
 			ctl_ha_close(softc);
 			ctl_ha_lclose(softc);
 			softc->ha_disconnect = 0;
+			if (softc->ha_shutdown)
+				break;
 		} else if (softc->ha_so != NULL &&
 		    (softc->ha_so->so_error ||
 		     softc->ha_so->so_rcv.sb_state & SBS_CANTRCVMORE))
@@ -614,6 +618,11 @@ ctl_ha_conn_thread(void *arg)
 		softc->ha_wakeup = 0;
 		mtx_unlock(&softc->ha_lock);
 	}
+	mtx_lock(&softc->ha_lock);
+	softc->ha_shutdown = 2;
+	wakeup(&softc->ha_wakeup);
+	mtx_unlock(&softc->ha_lock);
+	kthread_exit();
 }
 
 static int
@@ -936,6 +945,8 @@ ctl_ha_msg_init(struct ctl_softc *ctl_softc)
 		mtx_destroy(&softc->ha_lock);
 		return (CTL_HA_STATUS_ERROR);
 	}
+	softc->ha_shutdown_eh = EVENTHANDLER_REGISTER(shutdown_pre_sync,
+	    ctl_ha_msg_shutdown, ctl_softc, SHUTDOWN_PRI_FIRST);
 	SYSCTL_ADD_PROC(&ctl_softc->sysctl_ctx,
 	    SYSCTL_CHILDREN(ctl_softc->sysctl_tree),
 	    OID_AUTO, "ha_peer", CTLTYPE_STRING | CTLFLAG_RWTUN,
@@ -949,14 +960,40 @@ ctl_ha_msg_init(struct ctl_softc *ctl_softc)
 	return (CTL_HA_STATUS_SUCCESS);
 };
 
-ctl_ha_status
+void
 ctl_ha_msg_shutdown(struct ctl_softc *ctl_softc)
 {
 	struct ha_softc *softc = &ha_softc;
 
-	if (ctl_ha_msg_deregister(CTL_HA_CHAN_DATA) != CTL_HA_STATUS_SUCCESS) {
-		printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
+	/* Disconnect and shutdown threads. */
+	mtx_lock(&softc->ha_lock);
+	if (softc->ha_shutdown < 2) {
+		softc->ha_shutdown = 1;
+		softc->ha_wakeup = 1;
+		wakeup(&softc->ha_wakeup);
+		while (softc->ha_shutdown < 2) {
+			msleep(&softc->ha_wakeup, &softc->ha_lock, 0,
+			    "shutdown", hz);
+		}
 	}
+	mtx_unlock(&softc->ha_lock);
+};
+
+ctl_ha_status
+ctl_ha_msg_destroy(struct ctl_softc *ctl_softc)
+{
+	struct ha_softc *softc = &ha_softc;
+
+	if (softc->ha_shutdown_eh != NULL) {
+		EVENTHANDLER_DEREGISTER(shutdown_pre_sync,
+		    softc->ha_shutdown_eh);
+		softc->ha_shutdown_eh = NULL;
+	}
+
+	ctl_ha_msg_shutdown(ctl_softc);	/* Just in case. */
+
+	if (ctl_ha_msg_deregister(CTL_HA_CHAN_DATA) != CTL_HA_STATUS_SUCCESS)
+		printf("%s: ctl_ha_msg_deregister failed.\n", __func__);
 
 	mtx_destroy(&softc->ha_lock);
 	return (CTL_HA_STATUS_SUCCESS);
diff --git a/sys/cam/ctl/ctl_ha.h b/sys/cam/ctl/ctl_ha.h
index 072492a..0d2c011 100644
--- a/sys/cam/ctl/ctl_ha.h
+++ b/sys/cam/ctl/ctl_ha.h
@@ -109,7 +109,8 @@ struct ctl_ha_dt_req {
 
 struct ctl_softc;
 ctl_ha_status ctl_ha_msg_init(struct ctl_softc *softc);
-ctl_ha_status ctl_ha_msg_shutdown(struct ctl_softc *softc);
+void ctl_ha_msg_shutdown(struct ctl_softc *softc);
+ctl_ha_status ctl_ha_msg_destroy(struct ctl_softc *softc);
 
 typedef void (*ctl_evt_handler)(ctl_ha_channel channel, ctl_ha_event event,
 				int param);
-- 
cgit v1.1