Fix the nfs related daemons so that they don't intermittently

fail with "bind: address already in use". This problem was reported
to the freebsd-stable@ mailing list on Feb. 19 under the subject
heading "statd/lockd startup failure" by george+freebsd at m5p dot com.
The problem is that the first combination of {udp,tcp X ipv4,ipv6}
would select a port# dynamically, but one of the other three combinations
would have that port# already in use. The patch is somewhat involved
because it was requested by dougb@ that the four combinations use the
same port# wherever possible. The patch splits the create_service()
function into two functions. The first goes as far as bind(2) in a
loop for up to GETPORT_MAXTRY - 1 times, attempting to use the same port#
for all four cases. If these attempts fail, the last attempt allows
the 4 cases to use different port #s. After this function has succeeded,
the second function, called complete_service(), does the rest of what
create_service() did.
The three daemons mountd, rpc.lockd and rpc.statd all have a
create_service() function that is patched in a similar way. However,
create_service() has non-trivial differences for the three daemons
that made it impractical to share the same functions between them.

Reviewed by:	jhb
MFC after:	2 weeks

1 files changed, 221 insertions, 28 deletions

diff --git a/usr.sbin/rpc.lockd/lockd.c b/usr.sbin/rpc.lockd/lockd.c
index fb9e536..b3402ff 100644
--- a/usr.sbin/rpc.lockd/lockd.c
+++ b/usr.sbin/rpc.lockd/lockd.c
@@ -74,6 +74,8 @@ __RCSID("$NetBSD: lockd.c,v 1.7 2000/08/12 18:08:44 thorpej Exp $");
 #include "lockd.h"
 #include <rpcsvc/nlm_prot.h>
 
+#define	GETPORT_MAXTRY	20	/* Max tries to get a port # */
+
 int		debug_level = 0;	/* 0 = no debugging syslog() calls */
 int		_rpcsvcdirty = 0;
 
@@ -84,13 +86,19 @@ int kernel_lockd_client;
 pid_t client_pid;
 struct mon mon_host;
 char **hosts, *svcport_str = NULL;
+static int	mallocd_svcport = 0;
+static int	*sock_fd;
+static int	sock_fdcnt;
+static int	sock_fdpos;
 int nhosts = 0;
 int xcreated = 0;
 char **addrs;			/* actually (netid, uaddr) pairs */
 int naddrs;			/* count of how many (netid, uaddr) pairs */
 char localhost[] = "localhost";
 
-void 	create_service(struct netconfig *nconf);
+static int	create_service(struct netconfig *nconf);
+static void	complete_service(struct netconfig *nconf, char *port_str);
+static void	clearout_service(void);
 void 	lookup_addresses(struct netconfig *nconf);
 void	init_nsm(void);
 void	nlm_prog_0(struct svc_req *, SVCXPRT *);
@@ -119,6 +127,8 @@ main(int argc, char **argv)
 	int have_v6 = 1;
 	int maxrec = RPC_MAXDATASIZE;
 	in_port_t svcport = 0;
+	int attempt_cnt, port_len, port_pos, ret;
+	char **port_list;
 
 	while ((ch = getopt(argc, argv, "d:g:h:p:")) != (-1)) {
 		switch (ch) {
@@ -309,6 +319,11 @@ main(int argc, char **argv)
 		}
 		endnetconfig(nc_handle);
 	} else {
+		attempt_cnt = 1;
+		sock_fdcnt = 0;
+		sock_fd = NULL;
+		port_list = NULL;
+		port_len = 0;
 		nc_handle = setnetconfig();
 		while ((nconf = getnetconfig(nc_handle))) {
 			/* We want to listen only on udp6, tcp6, udp, tcp transports */
@@ -317,11 +332,96 @@ main(int argc, char **argv)
 				if (have_v6 == 0 && strcmp(nconf->nc_protofmly, "inet6") == 0) {
 					/* DO NOTHING */
 				} else {
-					create_service(nconf);
+					ret = create_service(nconf);
+					if (ret == 1)
+						/* Ignore this call */
+						continue;
+					if (ret < 0) {
+						/*
+						 * Failed to bind port, so close
+						 * off all sockets created and
+						 * try again if the port# was
+						 * dynamically assigned via
+						 * bind(2).
+						 */
+						clearout_service();
+						if (mallocd_svcport != 0 &&
+						    attempt_cnt <
+						    GETPORT_MAXTRY) {
+							free(svcport_str);
+							svcport_str = NULL;
+							mallocd_svcport = 0;
+						} else {
+							errno = EADDRINUSE;
+							syslog(LOG_ERR,
+							 "bindresvport_sa: %m");
+							exit(1);
+						}
+	
+						/*
+						 * Start over at the first
+						 * service.
+						 */
+						free(sock_fd);
+						sock_fdcnt = 0;
+						sock_fd = NULL;
+						nc_handle = setnetconfig();
+						attempt_cnt++;
+					} else if (mallocd_svcport != 0 &&
+					    attempt_cnt == GETPORT_MAXTRY) {
+						/*
+						 * For the last attempt, allow
+						 * different port #s for each
+						 * nconf by saving the
+						 * svcport_str and setting it
+						 * back to NULL.
+						 */
+						port_list = realloc(port_list,
+						    (port_len + 1) *
+						    sizeof(char *));
+						if (port_list == NULL)
+							out_of_mem();
+						port_list[port_len++] =
+						    svcport_str;
+						svcport_str = NULL;
+						mallocd_svcport = 0;
+					}
 				}
 			}
 		}
+
+		/*
+		 * Successfully bound the ports, so call complete_service() to
+		 * do the rest of the setup on the service(s).
+		 */
+		sock_fdpos = 0;
+		port_pos = 0;
+		nc_handle = setnetconfig();
+		while ((nconf = getnetconfig(nc_handle))) {
+			/* We want to listen only on udp6, tcp6, udp, tcp transports */
+			if (nconf->nc_flag & NC_VISIBLE) {
+				/* Skip if there's no IPv6 support */
+				if (have_v6 == 0 && strcmp(nconf->nc_protofmly, "inet6") == 0) {
+					/* DO NOTHING */
+				} else if (port_list != NULL) {
+					if (port_pos >= port_len) {
+						syslog(LOG_ERR,
+						    "too many port#s");
+						exit(1);
+					}
+					complete_service(nconf,
+					    port_list[port_pos++]);
+				} else
+					complete_service(nconf, svcport_str);
+			}
+		}
 		endnetconfig(nc_handle);
+		free(sock_fd);
+		if (port_list != NULL) {
+			for (port_pos = 0; port_pos < port_len; port_pos++)
+				free(port_list[port_pos]);
+			free(port_list);
+		}
 	}
 
 	/*
@@ -386,29 +486,30 @@ main(int argc, char **argv)
 
 /*
  * This routine creates and binds sockets on the appropriate
- * addresses. It gets called one time for each transport and
- * registrates the service with rpcbind on that trasport.
+ * addresses. It gets called one time for each transport.
+ * It returns 0 upon success, 1 for ingore the call and -1 to indicate
+ * bind failed with EADDRINUSE.
+ * Any file descriptors that have been created are stored in sock_fd and
+ * the total count of them is maintained in sock_fdcnt.
  */
-void
+static int
 create_service(struct netconfig *nconf)
 {
 	struct addrinfo hints, *res = NULL;
 	struct sockaddr_in *sin;
 	struct sockaddr_in6 *sin6;
 	struct __rpc_sockinfo si;
-	struct netbuf servaddr;
-	SVCXPRT	*transp = NULL;
 	int aicode;
 	int fd;
 	int nhostsbak;
 	int r;
-	int registered = 0;
 	u_int32_t host_addr[4];  /* IPv4 or IPv6 */
+	int mallocd_res;
 
 	if ((nconf->nc_semantics != NC_TPI_CLTS) &&
 	    (nconf->nc_semantics != NC_TPI_COTS) &&
 	    (nconf->nc_semantics != NC_TPI_COTS_ORD))
-		return;	/* not my type */
+		return (1);	/* not my type */
 
 	/*
 	 * XXX - using RPC library internal functions.
@@ -416,7 +517,7 @@ create_service(struct netconfig *nconf)
 	if (!__rpc_nconf2sockinfo(nconf, &si)) {
 		syslog(LOG_ERR, "cannot get information for %s",
 		    nconf->nc_netid);
-		return;
+		return (1);
 	}
 
 	/* Get rpc.statd's address on this transport */
@@ -432,6 +533,11 @@ create_service(struct netconfig *nconf)
 	nhostsbak = nhosts;
 	while (nhostsbak > 0) {
 		--nhostsbak;
+		sock_fd = realloc(sock_fd, (sock_fdcnt + 1) * sizeof(int));
+		if (sock_fd == NULL)
+			out_of_mem();
+		sock_fd[sock_fdcnt++] = -1;	/* Set invalid for now. */
+		mallocd_res = 0;
 
 		/*	
 		 * XXX - using RPC library internal functions.
@@ -446,7 +552,7 @@ create_service(struct netconfig *nconf)
 			case AF_INET:
 				if (inet_pton(AF_INET, hosts[nhostsbak],
 				    host_addr) == 1) {
-					hints.ai_flags &= AI_NUMERICHOST;
+					hints.ai_flags |= AI_NUMERICHOST;
 				} else {
 					/*
 					 * Skip if we have an AF_INET6 address.
@@ -461,7 +567,7 @@ create_service(struct netconfig *nconf)
 			case AF_INET6:
 				if (inet_pton(AF_INET6, hosts[nhostsbak],
 				    host_addr) == 1) {
-					hints.ai_flags &= AI_NUMERICHOST;
+					hints.ai_flags |= AI_NUMERICHOST;
 				} else {
 					/*
 					 * Skip if we have an AF_INET address.
@@ -485,6 +591,7 @@ create_service(struct netconfig *nconf)
 				res = malloc(sizeof(struct addrinfo));
 				if (res == NULL) 
 					out_of_mem();
+				mallocd_res = 1;
 				res->ai_flags = hints.ai_flags;
 				res->ai_family = hints.ai_family;
 				res->ai_protocol = hints.ai_protocol;
@@ -498,7 +605,7 @@ create_service(struct netconfig *nconf)
 						sin->sin_addr.s_addr = htonl(INADDR_ANY);
 						res->ai_addr = (struct sockaddr*) sin;
 						res->ai_addrlen = (socklen_t)
-						    sizeof(res->ai_addr);
+						    sizeof(struct sockaddr_in);
 						break;
 					case AF_INET6:
 						sin6 = malloc(sizeof(struct sockaddr_in6));
@@ -508,10 +615,14 @@ create_service(struct netconfig *nconf)
 						sin6->sin6_port = htons(0);
 						sin6->sin6_addr = in6addr_any;
 						res->ai_addr = (struct sockaddr*) sin6;
-						res->ai_addrlen = (socklen_t) sizeof(res->ai_addr);
+						res->ai_addrlen = (socklen_t)
+						    sizeof(struct sockaddr_in6);
 						break;
 					default:
-						break;
+						syslog(LOG_ERR,
+						    "bad addr fam %d",
+						    res->ai_family);
+						exit(1);
 				}
 			} else { 
 				if ((aicode = getaddrinfo(NULL, svcport_str,
@@ -520,6 +631,7 @@ create_service(struct netconfig *nconf)
 					    "cannot get local address for %s: %s",
 					    nconf->nc_netid,
 					    gai_strerror(aicode));
+					close(fd);
 					continue;
 				}
 			}
@@ -529,16 +641,92 @@ create_service(struct netconfig *nconf)
 				syslog(LOG_ERR,
 				    "cannot get local address for %s: %s",
 				    nconf->nc_netid, gai_strerror(aicode));
+				close(fd);
 				continue;
 			}
 		}
 
+
+		/* Store the fd. */
+		sock_fd[sock_fdcnt - 1] = fd;
+
+		/* Now, attempt the bind. */
 		r = bindresvport_sa(fd, res->ai_addr);
 		if (r != 0) {
+			if (errno == EADDRINUSE && mallocd_svcport != 0) {
+				if (mallocd_res != 0) {
+					free(res->ai_addr);
+					free(res);
+				} else
+					freeaddrinfo(res);
+				return (-1);
+			}
 			syslog(LOG_ERR, "bindresvport_sa: %m");
 			exit(1);
 		}
 
+		if (svcport_str == NULL) {
+			svcport_str = malloc(NI_MAXSERV * sizeof(char));
+			if (svcport_str == NULL)
+				out_of_mem();
+			mallocd_svcport = 1;
+
+			if (getnameinfo(res->ai_addr,
+			    res->ai_addr->sa_len, NULL, NI_MAXHOST,
+			    svcport_str, NI_MAXSERV * sizeof(char),
+			    NI_NUMERICHOST | NI_NUMERICSERV))
+				errx(1, "Cannot get port number");
+		}
+		if (mallocd_res != 0) {
+			free(res->ai_addr);
+			free(res);
+		} else
+			freeaddrinfo(res);
+		res = NULL;
+	}
+	return (0);
+}
+
+/*
+ * Called after all the create_service() calls have succeeded, to complete
+ * the setup and registration.
+ */
+static void
+complete_service(struct netconfig *nconf, char *port_str)
+{
+	struct addrinfo hints, *res = NULL;
+	struct __rpc_sockinfo si;
+	struct netbuf servaddr;
+	SVCXPRT	*transp = NULL;
+	int aicode, fd, nhostsbak;
+	int registered = 0;
+
+	if ((nconf->nc_semantics != NC_TPI_CLTS) &&
+	    (nconf->nc_semantics != NC_TPI_COTS) &&
+	    (nconf->nc_semantics != NC_TPI_COTS_ORD))
+		return;	/* not my type */
+
+	/*
+	 * XXX - using RPC library internal functions.
+	 */
+	if (!__rpc_nconf2sockinfo(nconf, &si)) {
+		syslog(LOG_ERR, "cannot get information for %s",
+		    nconf->nc_netid);
+		return;
+	}
+
+	nhostsbak = nhosts;
+	while (nhostsbak > 0) {
+		--nhostsbak;
+		if (sock_fdpos >= sock_fdcnt) {
+			/* Should never happen. */
+			syslog(LOG_ERR, "Ran out of socket fd's");
+			return;
+		}
+		fd = sock_fd[sock_fdpos++];
+		if (fd < 0)
+			continue;
+
 		if (nconf->nc_semantics != NC_TPI_CLTS)
 		    listen(fd, SOMAXCONN);
 
@@ -582,19 +770,7 @@ create_service(struct netconfig *nconf)
 			hints.ai_socktype = si.si_socktype;
 			hints.ai_protocol = si.si_proto;
 
-			if (svcport_str == NULL) {
-				svcport_str = malloc(NI_MAXSERV * sizeof(char));
-				if (svcport_str == NULL)
-					out_of_mem();
-
-				if (getnameinfo(res->ai_addr,
-				    res->ai_addr->sa_len, NULL, NI_MAXHOST,
-				    svcport_str, NI_MAXSERV * sizeof(char),
-				    NI_NUMERICHOST | NI_NUMERICSERV))
-					errx(1, "Cannot get port number");
-			}
-
-			if((aicode = getaddrinfo(NULL, svcport_str, &hints,
+			if ((aicode = getaddrinfo(NULL, port_str, &hints,
 			    &res)) != 0) {
 				syslog(LOG_ERR, "cannot get local address: %s",
 				    gai_strerror(aicode));
@@ -617,6 +793,23 @@ create_service(struct netconfig *nconf)
 }
 
 /*
+ * Clear out sockets after a failure to bind one of them, so that the
+ * cycle of socket creation/binding can start anew.
+ */
+static void
+clearout_service(void)
+{
+	int i;
+
+	for (i = 0; i < sock_fdcnt; i++) {
+		if (sock_fd[i] >= 0) {
+			shutdown(sock_fd[i], SHUT_RDWR);
+			close(sock_fd[i]);
+		}
+	}
+}
+
+/*
  * Look up addresses for the kernel to create transports for.
  */
 void