In hast.conf we define the other node's address in 'remote' variable.

This way we know how to connect to secondary node when we are primary.
The same variable is used by the secondary node - it only accepts
connections from the address stored in 'remote' variable.
In cluster configurations it is common that each node has its individual
IP address and there is one addtional shared IP address which is assigned
to primary node. It seems it is possible that if the shared IP address is
from the same network as the individual IP address it might be choosen by
the kernel as a source address for connection with the secondary node.
Such connection will be rejected by secondary, as it doesn't come from
primary node individual IP.

Add 'source' variable that allows to specify source IP address we want to
bind to before connecting to the secondary node.

MFC after:	1 week

14 files changed, 124 insertions, 36 deletions

diff --git a/sbin/hastd/control.c b/sbin/hastd/control.c
index 8fcd5ef..b0f38d6 100644
--- a/sbin/hastd/control.c
+++ b/sbin/hastd/control.c
@@ -234,6 +234,8 @@ control_status(struct hastd_config *cfg, struct nv *nvout,
 	nv_add_string(nvout, res->hr_provname, "provname%u", no);
 	nv_add_string(nvout, res->hr_localpath, "localpath%u", no);
 	nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr%u", no);
+	if (res->hr_sourceaddr[0] != '\0')
+		nv_add_string(nvout, res->hr_sourceaddr, "sourceaddr%u", no);
 	switch (res->hr_replication) {
 	case HAST_REPLICATION_FULLSYNC:
 		nv_add_string(nvout, "fullsync", "replication%u", no);
diff --git a/sbin/hastd/hast.conf.5 b/sbin/hastd/hast.conf.5
index f4d0754..9bf6c4e 100644
--- a/sbin/hastd/hast.conf.5
+++ b/sbin/hastd/hast.conf.5
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 30, 2010
+.Dd March 20, 2011
 .Dt HAST.CONF 5
 .Os
 .Sh NAME
@@ -93,6 +93,7 @@ resource <name> {
 		local <path>
 		# Required
 		remote <addr>
+		source <addr>
 	}
 	on <node> {
 		# Resource-node section
@@ -101,6 +102,7 @@ resource <name> {
 		local <path>
 		# Required
 		remote <addr>
+		source <addr>
 	}
 }
 .Ed
@@ -337,6 +339,14 @@ A special value of
 .Va none
 can be used when the remote address is not yet known (eg. the other node is not
 set up yet).
+.It Ic source Aq addr
+.Pp
+Local address to bind to before connecting to the remote
+.Nm hastd
+daemon.
+Format is the same as for the
+.Ic listen
+statement.
 .El
 .Sh FILES
 .Bl -tag -width ".Pa /var/run/hastctl" -compact
@@ -367,10 +377,12 @@ resource shared {
 resource tank {
 	on hasta {
 		local /dev/mirror/tanka
+		source tcp4://10.0.0.1
 		remote tcp4://10.0.0.2
 	}
 	on hastb {
 		local /dev/mirror/tankb
+		source tcp4://10.0.0.2
 		remote tcp4://10.0.0.1
 	}
 }
diff --git a/sbin/hastd/hast.h b/sbin/hastd/hast.h
index 15bcdb5..ce3dcaa 100644
--- a/sbin/hastd/hast.h
+++ b/sbin/hastd/hast.h
@@ -169,6 +169,8 @@ struct hast_resource {
 
 	/* Address of the remote component. */
 	char	hr_remoteaddr[HAST_ADDRSIZE];
+	/* Local address to bind to for outgoing connections. */
+	char	hr_sourceaddr[HAST_ADDRSIZE];
 	/* Connection for incoming data. */
 	struct proto_conn *hr_remotein;
 	/* Connection for outgoing data. */
diff --git a/sbin/hastd/hastd.c b/sbin/hastd/hastd.c
index d1b8e9f..c09c92e 100644
--- a/sbin/hastd/hastd.c
+++ b/sbin/hastd/hastd.c
@@ -360,6 +360,8 @@ resource_needs_restart(const struct hast_resource *res0,
 	    res0->hr_role == HAST_ROLE_SECONDARY) {
 		if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0)
 			return (true);
+		if (strcmp(res0->hr_sourceaddr, res1->hr_sourceaddr) != 0)
+			return (true);
 		if (res0->hr_replication != res1->hr_replication)
 			return (true);
 		if (res0->hr_checksum != res1->hr_checksum)
@@ -388,6 +390,8 @@ resource_needs_reload(const struct hast_resource *res0,
 
 	if (strcmp(res0->hr_remoteaddr, res1->hr_remoteaddr) != 0)
 		return (true);
+	if (strcmp(res0->hr_sourceaddr, res1->hr_sourceaddr) != 0)
+		return (true);
 	if (res0->hr_replication != res1->hr_replication)
 		return (true);
 	if (res0->hr_checksum != res1->hr_checksum)
@@ -412,6 +416,7 @@ resource_reload(const struct hast_resource *res)
 	nvout = nv_alloc();
 	nv_add_uint8(nvout, HASTCTL_RELOAD, "cmd");
 	nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr");
+	nv_add_string(nvout, res->hr_sourceaddr, "sourceaddr");
 	nv_add_int32(nvout, (int32_t)res->hr_replication, "replication");
 	nv_add_int32(nvout, (int32_t)res->hr_checksum, "checksum");
 	nv_add_int32(nvout, (int32_t)res->hr_compression, "compression");
@@ -572,6 +577,8 @@ hastd_reload(void)
 			    cres->hr_name);
 			strlcpy(cres->hr_remoteaddr, nres->hr_remoteaddr,
 			    sizeof(cres->hr_remoteaddr));
+			strlcpy(cres->hr_sourceaddr, nres->hr_sourceaddr,
+			    sizeof(cres->hr_sourceaddr));
 			cres->hr_replication = nres->hr_replication;
 			cres->hr_checksum = nres->hr_checksum;
 			cres->hr_compression = nres->hr_compression;
@@ -849,7 +856,8 @@ connection_migrate(struct hast_resource *res)
 		    "Unable to receive connection command");
 		return;
 	}
-	if (proto_client(res->hr_remoteaddr, &conn) < 0) {
+	if (proto_client(res->hr_sourceaddr[0] != '\0' ? res->hr_sourceaddr : NULL,
+	    res->hr_remoteaddr, &conn) < 0) {
 		val = errno;
 		pjdlog_errno(LOG_WARNING,
 		    "Unable to create outgoing connection to %s",
diff --git a/sbin/hastd/parse.y b/sbin/hastd/parse.y
index be2cc1e..3500fdf 100644
--- a/sbin/hastd/parse.y
+++ b/sbin/hastd/parse.y
@@ -276,7 +276,7 @@ yy_config_free(struct hastd_config *config)
 %}
 
 %token CONTROL LISTEN PORT REPLICATION CHECKSUM COMPRESSION
-%token TIMEOUT EXEC EXTENTSIZE RESOURCE NAME LOCAL REMOTE ON
+%token TIMEOUT EXEC EXTENTSIZE RESOURCE NAME LOCAL REMOTE SOURCE ON
 %token FULLSYNC MEMSYNC ASYNC NONE CRC32 SHA256 HOLE LZF
 %token NUM STR OB CB
 
@@ -652,6 +652,7 @@ resource_start:	STR
 		curres->hr_localpath[0] = '\0';
 		curres->hr_localfd = -1;
 		curres->hr_remoteaddr[0] = '\0';
+		curres->hr_sourceaddr[0] = '\0';
 		curres->hr_ggateunit = -1;
 	}
 	;
@@ -778,6 +779,8 @@ resource_node_entry:
 	local_statement
 	|
 	remote_statement
+	|
+	source_statement
 	;
 
 remote_statement:	REMOTE STR
@@ -796,3 +799,20 @@ remote_statement:	REMOTE STR
 		free($2);
 	}
 	;
+
+source_statement:	SOURCE STR
+	{
+		assert(depth == 2);
+		if (mynode) {
+			assert(curres != NULL);
+			if (strlcpy(curres->hr_sourceaddr, $2,
+			    sizeof(curres->hr_sourceaddr)) >=
+			    sizeof(curres->hr_sourceaddr)) {
+				pjdlog_error("source argument is too long.");
+				free($2);
+				return (1);
+			}
+		}
+		free($2);
+	}
+	;
diff --git a/sbin/hastd/primary.c b/sbin/hastd/primary.c
index c9266e5..8beba48 100644
--- a/sbin/hastd/primary.c
+++ b/sbin/hastd/primary.c
@@ -86,7 +86,7 @@ struct hio {
 	 */
 	int			*hio_errors;
 	/*
-	 * Structure used to comunicate with GEOM Gate class.
+	 * Structure used to communicate with GEOM Gate class.
 	 */
 	struct g_gate_ctl_io	 hio_ggio;
 	TAILQ_ENTRY(hio)	*hio_next;
@@ -808,7 +808,7 @@ hastd_primary(struct hast_resource *res)
 	 * Create communication channel for sending control commands from
 	 * parent to child.
 	 */
-	if (proto_client("socketpair://", &res->hr_ctrl) < 0) {
+	if (proto_client(NULL, "socketpair://", &res->hr_ctrl) < 0) {
 		/* TODO: There's no need for this to be fatal error. */
 		KEEP_ERRNO((void)pidfile_remove(pfh));
 		pjdlog_exit(EX_OSERR,
@@ -817,7 +817,7 @@ hastd_primary(struct hast_resource *res)
 	/*
 	 * Create communication channel for sending events from child to parent.
 	 */
-	if (proto_client("socketpair://", &res->hr_event) < 0) {
+	if (proto_client(NULL, "socketpair://", &res->hr_event) < 0) {
 		/* TODO: There's no need for this to be fatal error. */
 		KEEP_ERRNO((void)pidfile_remove(pfh));
 		pjdlog_exit(EX_OSERR,
@@ -827,7 +827,7 @@ hastd_primary(struct hast_resource *res)
 	 * Create communication channel for sending connection requests from
 	 * child to parent.
 	 */
-	if (proto_client("socketpair://", &res->hr_conn) < 0) {
+	if (proto_client(NULL, "socketpair://", &res->hr_conn) < 0) {
 		/* TODO: There's no need for this to be fatal error. */
 		KEEP_ERRNO((void)pidfile_remove(pfh));
 		pjdlog_exit(EX_OSERR,
@@ -1918,6 +1918,7 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 	PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY);
 	PJDLOG_ASSERT(gres == res);
 	nv_assert(nv, "remoteaddr");
+	nv_assert(nv, "sourceaddr");
 	nv_assert(nv, "replication");
 	nv_assert(nv, "checksum");
 	nv_assert(nv, "compression");
@@ -1927,11 +1928,12 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 	ncomps = HAST_NCOMPONENTS;
 
 #define MODIFIED_REMOTEADDR	0x01
-#define MODIFIED_REPLICATION	0x02
-#define MODIFIED_CHECKSUM	0x04
-#define MODIFIED_COMPRESSION	0x08
-#define MODIFIED_TIMEOUT	0x10
-#define MODIFIED_EXEC		0x20
+#define MODIFIED_SOURCEADDR	0x02
+#define MODIFIED_REPLICATION	0x04
+#define MODIFIED_CHECKSUM	0x08
+#define MODIFIED_COMPRESSION	0x10
+#define MODIFIED_TIMEOUT	0x20
+#define MODIFIED_EXEC		0x40
 	modified = 0;
 
 	vstr = nv_get_string(nv, "remoteaddr");
@@ -1943,6 +1945,11 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 		 */
 		modified |= MODIFIED_REMOTEADDR;
 	}
+	vstr = nv_get_string(nv, "sourceaddr");
+	if (strcmp(gres->hr_sourceaddr, vstr) != 0) {
+		strlcpy(gres->hr_sourceaddr, vstr, sizeof(gres->hr_sourceaddr));
+		modified |= MODIFIED_SOURCEADDR;
+	}
 	vint = nv_get_int32(nv, "replication");
 	if (gres->hr_replication != vint) {
 		gres->hr_replication = vint;
@@ -1974,7 +1981,8 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 	 * Don't bother if we need to reconnect.
 	 */
 	if ((modified & MODIFIED_TIMEOUT) != 0 &&
-	    (modified & (MODIFIED_REMOTEADDR | MODIFIED_REPLICATION)) == 0) {
+	    (modified & (MODIFIED_REMOTEADDR | MODIFIED_SOURCEADDR |
+	    MODIFIED_REPLICATION)) == 0) {
 		for (ii = 0; ii < ncomps; ii++) {
 			if (!ISREMOTE(ii))
 				continue;
@@ -1996,7 +2004,8 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 			}
 		}
 	}
-	if ((modified & (MODIFIED_REMOTEADDR | MODIFIED_REPLICATION)) != 0) {
+	if ((modified & (MODIFIED_REMOTEADDR | MODIFIED_SOURCEADDR |
+	    MODIFIED_REPLICATION)) != 0) {
 		for (ii = 0; ii < ncomps; ii++) {
 			if (!ISREMOTE(ii))
 				continue;
@@ -2009,6 +2018,7 @@ primary_config_reload(struct hast_resource *res, struct nv *nv)
 		}
 	}
 #undef	MODIFIED_REMOTEADDR
+#undef	MODIFIED_SOURCEADDR
 #undef	MODIFIED_REPLICATION
 #undef	MODIFIED_CHECKSUM
 #undef	MODIFIED_COMPRESSION
diff --git a/sbin/hastd/proto.c b/sbin/hastd/proto.c
index 8b1c686..ae576a8 100644
--- a/sbin/hastd/proto.c
+++ b/sbin/hastd/proto.c
@@ -105,7 +105,8 @@ proto_free(struct proto_conn *conn)
 }
 
 static int
-proto_common_setup(const char *addr, struct proto_conn **connp, int side)
+proto_common_setup(const char *srcaddr, const char *dstaddr,
+    struct proto_conn **connp, int side)
 {
 	struct hast_proto *proto;
 	struct proto_conn *conn;
@@ -120,16 +121,16 @@ proto_common_setup(const char *addr, struct proto_conn **connp, int side)
 			if (proto->hp_client == NULL)
 				ret = -1;
 			else
-				ret = proto->hp_client(addr, &ctx);
+				ret = proto->hp_client(srcaddr, dstaddr, &ctx);
 		} else /* if (side == PROTO_SIDE_SERVER_LISTEN) */ {
 			if (proto->hp_server == NULL)
 				ret = -1;
 			else
-				ret = proto->hp_server(addr, &ctx);
+				ret = proto->hp_server(dstaddr, &ctx);
 		}
 		/*
 		 * ret == 0  - success
-		 * ret == -1 - addr is not for this protocol
+		 * ret == -1 - dstaddr is not for this protocol
 		 * ret > 0   - right protocol, but an error occured
 		 */
 		if (ret >= 0)
@@ -159,10 +160,11 @@ proto_common_setup(const char *addr, struct proto_conn **connp, int side)
 }
 
 int
-proto_client(const char *addr, struct proto_conn **connp)
+proto_client(const char *srcaddr, const char *dstaddr,
+    struct proto_conn **connp)
 {
 
-	return (proto_common_setup(addr, connp, PROTO_SIDE_CLIENT));
+	return (proto_common_setup(srcaddr, dstaddr, connp, PROTO_SIDE_CLIENT));
 }
 
 int
@@ -211,7 +213,7 @@ int
 proto_server(const char *addr, struct proto_conn **connp)
 {
 
-	return (proto_common_setup(addr, connp, PROTO_SIDE_SERVER_LISTEN));
+	return (proto_common_setup(NULL, addr, connp, PROTO_SIDE_SERVER_LISTEN));
 }
 
 int
diff --git a/sbin/hastd/proto.h b/sbin/hastd/proto.h
index 278fe78..1a60e5b 100644
--- a/sbin/hastd/proto.h
+++ b/sbin/hastd/proto.h
@@ -37,7 +37,8 @@
 
 struct proto_conn;
 
-int proto_client(const char *addr, struct proto_conn **connp);
+int proto_client(const char *srcaddr, const char *dstaddr,
+    struct proto_conn **connp);
 int proto_connect(struct proto_conn *conn, int timeout);
 int proto_connect_wait(struct proto_conn *conn, int timeout);
 int proto_server(const char *addr, struct proto_conn **connp);
diff --git a/sbin/hastd/proto_impl.h b/sbin/hastd/proto_impl.h
index 9dc5a74..37df4f4 100644
--- a/sbin/hastd/proto_impl.h
+++ b/sbin/hastd/proto_impl.h
@@ -39,7 +39,7 @@
 
 #define	__constructor	__attribute__((constructor))
 
-typedef int hp_client_t(const char *, void **);
+typedef int hp_client_t(const char *, const char *, void **);
 typedef int hp_connect_t(void *, int);
 typedef int hp_connect_wait_t(void *, int);
 typedef int hp_server_t(const char *, void **);
diff --git a/sbin/hastd/proto_socketpair.c b/sbin/hastd/proto_socketpair.c
index d681493..74034d7 100644
--- a/sbin/hastd/proto_socketpair.c
+++ b/sbin/hastd/proto_socketpair.c
@@ -57,14 +57,16 @@ struct sp_ctx {
 static void sp_close(void *ctx);
 
 static int
-sp_client(const char *addr, void **ctxp)
+sp_client(const char *srcaddr, const char *dstaddr, void **ctxp)
 {
 	struct sp_ctx *spctx;
 	int ret;
 
-	if (strcmp(addr, "socketpair://") != 0)
+	if (strcmp(dstaddr, "socketpair://") != 0)
 		return (-1);
 
+	PJDLOG_ASSERT(srcaddr == NULL);
+
 	spctx = malloc(sizeof(*spctx));
 	if (spctx == NULL)
 		return (errno);
diff --git a/sbin/hastd/proto_tcp4.c b/sbin/hastd/proto_tcp4.c
index 1fc9140..11b19a5 100644
--- a/sbin/hastd/proto_tcp4.c
+++ b/sbin/hastd/proto_tcp4.c
@@ -112,7 +112,7 @@ invalid:
 }
 
 static int
-tcp4_addr(const char *addr, struct sockaddr_in *sinp)
+tcp4_addr(const char *addr, int defport, struct sockaddr_in *sinp)
 {
 	char iporhost[MAXHOSTNAMELEN];
 	const char *pp;
@@ -139,7 +139,7 @@ tcp4_addr(const char *addr, struct sockaddr_in *sinp)
 	pp = strrchr(addr, ':');
 	if (pp == NULL) {
 		/* Port not given, use the default. */
-		sinp->sin_port = htons(HASTD_PORT);
+		sinp->sin_port = htons(defport);
 	} else {
 		intmax_t port;
 
@@ -183,7 +183,7 @@ tcp4_setup_new(const char *addr, int side, void **ctxp)
 		return (errno);
 
 	/* Parse given address. */
-	if ((ret = tcp4_addr(addr, &tctx->tc_sin)) != 0) {
+	if ((ret = tcp4_addr(addr, HASTD_PORT, &tctx->tc_sin)) != 0) {
 		free(tctx);
 		return (ret);
 	}
@@ -197,6 +197,8 @@ tcp4_setup_new(const char *addr, int side, void **ctxp)
 		return (ret);
 	}
 
+	PJDLOG_ASSERT(tctx->tc_sin.sin_family != AF_UNSPEC);
+
 	/* Socket settings. */
 	nodelay = 1;
 	if (setsockopt(tctx->tc_fd, IPPROTO_TCP, TCP_NODELAY, &nodelay,
@@ -235,10 +237,29 @@ tcp4_setup_wrap(int fd, int side, void **ctxp)
 }
 
 static int
-tcp4_client(const char *addr, void **ctxp)
+tcp4_client(const char *srcaddr, const char *dstaddr, void **ctxp)
 {
+	struct tcp4_ctx *tctx;
+	struct sockaddr_in sin;
+	int ret;
 
-	return (tcp4_setup_new(addr, TCP4_SIDE_CLIENT, ctxp));
+	ret = tcp4_setup_new(dstaddr, TCP4_SIDE_CLIENT, ctxp);
+	if (ret != 0)
+		return (ret);
+	tctx = *ctxp;
+	if (srcaddr == NULL)
+		return (0);
+	ret = tcp4_addr(srcaddr, 0, &sin);
+	if (ret != 0) {
+		tcp4_close(tctx);
+		return (ret);
+	}
+	if (bind(tctx->tc_fd, (struct sockaddr *)&sin, sizeof(sin)) < 0) {
+		ret = errno;
+		tcp4_close(tctx);
+		return (ret);
+	}
+	return (0);
 }
 
 static int
@@ -486,7 +507,7 @@ tcp4_address_match(const void *ctx, const char *addr)
 	PJDLOG_ASSERT(tctx != NULL);
 	PJDLOG_ASSERT(tctx->tc_magic == TCP4_CTX_MAGIC);
 
-	if (tcp4_addr(addr, &sin) != 0)
+	if (tcp4_addr(addr, HASTD_PORT, &sin) != 0)
 		return (false);
 	ip1 = sin.sin_addr.s_addr;
 
diff --git a/sbin/hastd/proto_uds.c b/sbin/hastd/proto_uds.c
index 8429d85..eddce0d 100644
--- a/sbin/hastd/proto_uds.c
+++ b/sbin/hastd/proto_uds.c
@@ -119,10 +119,17 @@ uds_common_setup(const char *addr, void **ctxp, int side)
 }
 
 static int
-uds_client(const char *addr, void **ctxp)
+uds_client(const char *srcaddr, const char *dstaddr, void **ctxp)
 {
+	int ret;
+
+	ret = uds_common_setup(dstaddr, ctxp, UDS_SIDE_CLIENT);
+	if (ret != 0)
+		return (ret);
+
+	PJDLOG_ASSERT(srcaddr == NULL);
 
-	return (uds_common_setup(addr, ctxp, UDS_SIDE_CLIENT));
+	return (0);
 }
 
 static int
diff --git a/sbin/hastd/secondary.c b/sbin/hastd/secondary.c
index 218c802..e718b2c 100644
--- a/sbin/hastd/secondary.c
+++ b/sbin/hastd/secondary.c
@@ -351,7 +351,7 @@ hastd_secondary(struct hast_resource *res, struct nv *nvin)
 	/*
 	 * Create communication channel between parent and child.
 	 */
-	if (proto_client("socketpair://", &res->hr_ctrl) < 0) {
+	if (proto_client(NULL, "socketpair://", &res->hr_ctrl) < 0) {
 		KEEP_ERRNO((void)pidfile_remove(pfh));
 		pjdlog_exit(EX_OSERR,
 		    "Unable to create control sockets between parent and child");
@@ -359,7 +359,7 @@ hastd_secondary(struct hast_resource *res, struct nv *nvin)
 	/*
 	 * Create communication channel between child and parent.
 	 */
-	if (proto_client("socketpair://", &res->hr_event) < 0) {
+	if (proto_client(NULL, "socketpair://", &res->hr_event) < 0) {
 		KEEP_ERRNO((void)pidfile_remove(pfh));
 		pjdlog_exit(EX_OSERR,
 		    "Unable to create event sockets between child and parent");
@@ -368,7 +368,7 @@ hastd_secondary(struct hast_resource *res, struct nv *nvin)
 	 * Create communication channel for sending connection requests from
 	 * parent to child.
 	 */
-	if (proto_client("socketpair://", &res->hr_conn) < 0) {
+	if (proto_client(NULL, "socketpair://", &res->hr_conn) < 0) {
 		/* TODO: There's no need for this to be fatal error. */
 		KEEP_ERRNO((void)pidfile_remove(pfh));
 		pjdlog_exit(EX_OSERR,
diff --git a/sbin/hastd/token.l b/sbin/hastd/token.l
index 1992b77..389dfed 100644
--- a/sbin/hastd/token.l
+++ b/sbin/hastd/token.l
@@ -57,6 +57,7 @@ resource		{ DP; return RESOURCE; }
 name			{ DP; return NAME; }
 local			{ DP; return LOCAL; }
 remote			{ DP; return REMOTE; }
+source			{ DP; return SOURCE; }
 on			{ DP; return ON; }
 fullsync		{ DP; return FULLSYNC; }
 memsync			{ DP; return MEMSYNC; }