Re-implement the client side of rpc.lockd in the kernel. This implementation

provides the correct semantics for flock(2) style locks which are used by the
lockf(1) command line tool and the pidfile(3) library. It also implements
recovery from server restarts and ensures that dirty cache blocks are written
to the server before obtaining locks (allowing multiple clients to use file
locking to safely share data).

Sponsored by:	Isilon Systems
PR:		94256
MFC after:	2 weeks

1 files changed, 90 insertions, 13 deletions

diff --git a/sys/rpc/clnt_rc.c b/sys/rpc/clnt_rc.c
index 8b5fc26..a6b2dfd 100644
--- a/sys/rpc/clnt_rc.c
+++ b/sys/rpc/clnt_rc.c
@@ -30,6 +30,7 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/limits.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
@@ -44,8 +45,8 @@ __FBSDID("$FreeBSD$");
 #include <rpc/rpc.h>
 #include <rpc/rpc_com.h>
 
-static enum clnt_stat clnt_reconnect_call(CLIENT *, rpcproc_t,
-    xdrproc_t, void *, xdrproc_t, void *, struct timeval);
+static enum clnt_stat clnt_reconnect_call(CLIENT *, struct rpc_callextra *,
+    rpcproc_t, xdrproc_t, void *, xdrproc_t, void *, struct timeval);
 static void clnt_reconnect_geterr(CLIENT *, struct rpc_err *);
 static bool_t clnt_reconnect_freeres(CLIENT *, xdrproc_t, void *);
 static void clnt_reconnect_abort(CLIENT *);
@@ -62,6 +63,7 @@ static struct clnt_ops clnt_reconnect_ops = {
 };
 
 struct rc_data {
+	struct mtx		rc_lock;
 	struct sockaddr_storage	rc_addr; /* server address */
 	struct netconfig*	rc_nconf; /* network type */
 	rpcprog_t		rc_prog;  /* program number */
@@ -70,8 +72,10 @@ struct rc_data {
 	size_t			rc_recvsz;
 	struct timeval		rc_timeout;
 	struct timeval		rc_retry;
+	int			rc_retries;
 	const char		*rc_waitchan;
 	int			rc_intr;
+	int			rc_connecting;
 	CLIENT*			rc_client; /* underlying RPC client */
 };
 
@@ -94,6 +98,7 @@ clnt_reconnect_create(
 
 	cl = mem_alloc(sizeof (CLIENT));
 	rc = mem_alloc(sizeof (*rc));
+	mtx_init(&rc->rc_lock, "rc->rc_lock", NULL, MTX_DEF);
 	(void) memcpy(&rc->rc_addr, svcaddr, (size_t)svcaddr->sa_len);
 	rc->rc_nconf = nconf;
 	rc->rc_prog = program;
@@ -102,12 +107,15 @@ clnt_reconnect_create(
 	rc->rc_recvsz = recvsz;
 	rc->rc_timeout.tv_sec = -1;
 	rc->rc_timeout.tv_usec = -1;
-	rc->rc_retry.tv_sec = 15;
+	rc->rc_retry.tv_sec = 3;
 	rc->rc_retry.tv_usec = 0;
+	rc->rc_retries = INT_MAX;
 	rc->rc_waitchan = "rpcrecv";
 	rc->rc_intr = 0;
+	rc->rc_connecting = FALSE;
 	rc->rc_client = NULL;
 
+	cl->cl_refs = 1;
 	cl->cl_ops = &clnt_reconnect_ops;
 	cl->cl_private = (caddr_t)(void *)rc;
 	cl->cl_auth = authnone_create();
@@ -121,13 +129,39 @@ clnt_reconnect_connect(CLIENT *cl)
 {
 	struct rc_data *rc = (struct rc_data *)cl->cl_private;
 	struct socket *so;
+	enum clnt_stat stat;
+	int error;
 	int one = 1;
 
+	mtx_lock(&rc->rc_lock);
+again:
+	if (rc->rc_connecting) {
+		while (!rc->rc_client) {
+			error = msleep(rc, &rc->rc_lock,
+			    rc->rc_intr ? PCATCH : 0, "rpcrecon", 0);
+			if (error) {
+				mtx_unlock(&rc->rc_lock);
+				return (RPC_INTR);
+			}
+		}
+		/*
+		 * If the other guy failed to connect, we might as
+		 * well have another go.
+		 */
+		if (!rc->rc_client && !rc->rc_connecting)
+			goto again;
+		mtx_unlock(&rc->rc_lock);
+		return (RPC_SUCCESS);
+	} else {
+		rc->rc_connecting = TRUE;
+	}
+	mtx_unlock(&rc->rc_lock);
+
 	so = __rpc_nconf2socket(rc->rc_nconf);
 	if (!so) {
-		rpc_createerr.cf_stat = RPC_TLIERROR;
+		stat = rpc_createerr.cf_stat = RPC_TLIERROR;
 		rpc_createerr.cf_error.re_errno = 0;
-		return (RPC_TLIERROR);
+		goto out;
 	}
 
 	if (rc->rc_nconf->nc_semantics == NC_TPI_CLTS)
@@ -139,8 +173,10 @@ clnt_reconnect_connect(CLIENT *cl)
 		    (struct sockaddr *) &rc->rc_addr, rc->rc_prog, rc->rc_vers,
 		    rc->rc_sendsz, rc->rc_recvsz);
 
-	if (!rc->rc_client)
-		return (rpc_createerr.cf_stat);
+	if (!rc->rc_client) {
+		stat = rpc_createerr.cf_stat;
+		goto out;
+	}
 
 	CLNT_CONTROL(rc->rc_client, CLSET_FD_CLOSE, 0);
 	CLNT_CONTROL(rc->rc_client, CLSET_CONNECT, &one);
@@ -148,13 +184,21 @@ clnt_reconnect_connect(CLIENT *cl)
 	CLNT_CONTROL(rc->rc_client, CLSET_RETRY_TIMEOUT, &rc->rc_retry);
 	CLNT_CONTROL(rc->rc_client, CLSET_WAITCHAN, &rc->rc_waitchan);
 	CLNT_CONTROL(rc->rc_client, CLSET_INTERRUPTIBLE, &rc->rc_intr);
+	stat = RPC_SUCCESS;
+
+out:
+	mtx_lock(&rc->rc_lock);
+	rc->rc_connecting = FALSE;
+	wakeup(rc);
+	mtx_unlock(&rc->rc_lock);
 
-	return (RPC_SUCCESS);
+	return (stat);
 }
 
 static enum clnt_stat
 clnt_reconnect_call(
-	CLIENT	*cl,			/* client handle */
+	CLIENT		*cl,		/* client handle */
+	struct rpc_callextra *ext,	/* call metadata */
 	rpcproc_t	proc,		/* procedure number */
 	xdrproc_t	xargs,		/* xdr routine for args */
 	void		*argsp,		/* pointer to args */
@@ -163,8 +207,11 @@ clnt_reconnect_call(
 	struct timeval	utimeout)	/* seconds to wait before giving up */
 {
 	struct rc_data *rc = (struct rc_data *)cl->cl_private;
+	CLIENT *client;
 	enum clnt_stat stat;
+	int tries;
 
+	tries = 0;
 	do {
 		if (!rc->rc_client) {
 			stat = clnt_reconnect_connect(cl);
@@ -172,9 +219,14 @@ clnt_reconnect_call(
 				return (stat);
 		}
 
-		stat = CLNT_CALL(rc->rc_client, proc, xargs, argsp,
+		mtx_lock(&rc->rc_lock);
+		CLNT_ACQUIRE(rc->rc_client);
+		client = rc->rc_client;
+		mtx_unlock(&rc->rc_lock);
+		stat = CLNT_CALL_EXT(client, ext, proc, xargs, argsp,
 		    xresults, resultsp, utimeout);
 
+		CLNT_RELEASE(client);
 		if (stat == RPC_TIMEDOUT) {
 			/*
 			 * Check for async send misfeature for NLM
@@ -184,16 +236,33 @@ clnt_reconnect_call(
 				&& rc->rc_timeout.tv_usec == 0)
 			    || (rc->rc_timeout.tv_sec == -1
 				&& utimeout.tv_sec == 0
-				&& utimeout.tv_usec == 0))
+				&& utimeout.tv_usec == 0)) {
 				break;
+			}
 		}
 
 		if (stat == RPC_INTR)
 			break;
 
 		if (stat != RPC_SUCCESS) {
-			CLNT_DESTROY(rc->rc_client);
-			rc->rc_client = NULL;
+			tries++;
+			if (tries >= rc->rc_retries)
+				break;
+
+			if (ext && ext->rc_feedback)
+				ext->rc_feedback(FEEDBACK_RECONNECT, proc,
+				    ext->rc_feedback_arg);
+
+			mtx_lock(&rc->rc_lock);
+			/*
+			 * Make sure that someone else hasn't already
+			 * reconnected.
+			 */
+			if (rc->rc_client == client) {
+				CLNT_RELEASE(rc->rc_client);
+				rc->rc_client = NULL;
+			}
+			mtx_unlock(&rc->rc_lock);
 		}
 	} while (stat != RPC_SUCCESS);
 
@@ -294,6 +363,14 @@ clnt_reconnect_control(CLIENT *cl, u_int request, void *info)
 		*(int *) info = rc->rc_intr;
 		break;
 
+	case CLSET_RETRIES:
+		rc->rc_retries = *(int *) info;
+		break;
+
+	case CLGET_RETRIES:
+		*(int *) info = rc->rc_retries;
+		break;
+
 	default:
 		return (FALSE);
 	}