Do not synchronously start the nfsiod threads at all. The r212506

fixed the issues with file descriptor locks, but the same problems are
present for vnode lock/user map lock.

If the nfs_asyncio() cannot find the free nfsiod, schedule task to
create new nfsiod and return error. This causes fall back to the
synchronous i/o for nfs_strategy(), or does not start read at all in
the case of readahead. The caller that holds vnode and potentially
user map lock does not wait for kproc_create() to finish, preventing
the LORs.

The change effectively reverts r203072, because we never hand off the
request to newly created nfsiod thread anymore.

Reviewed by:	jhb
Tested by:	jhb, pluknet
MFC after:	3 weeks

1 files changed, 37 insertions, 58 deletions

diff --git a/sys/nfsclient/nfs_nfsiod.c b/sys/nfsclient/nfs_nfsiod.c
index 5e2fbcc..fcb987e 100644
--- a/sys/nfsclient/nfs_nfsiod.c
+++ b/sys/nfsclient/nfs_nfsiod.c
@@ -76,16 +76,6 @@ static MALLOC_DEFINE(M_NFSSVC, "nfsclient_srvsock", "Nfs server structure");
 
 static void	nfssvc_iod(void *);
 
-struct nfsiod_str {
-	STAILQ_ENTRY(nfsiod_str) ni_links;
-	int *ni_inst;
-	int ni_iod;
-	int ni_error;
-	int ni_done;
-};
-static STAILQ_HEAD(, nfsiod_str) nfsiodhead =
-    STAILQ_HEAD_INITIALIZER(nfsiodhead);
-
 static int nfs_asyncdaemon[NFS_MAXASYNCDAEMON];
 
 SYSCTL_DECL(_vfs_nfs);
@@ -101,6 +91,8 @@ unsigned int nfs_iodmax = 20;
 /* Minimum number of nfsiod kthreads to keep as spares */
 static unsigned int nfs_iodmin = 0;
 
+static int nfs_nfsiodnew_sync(void);
+
 static int
 sysctl_iodmin(SYSCTL_HANDLER_ARGS)
 {
@@ -124,7 +116,7 @@ sysctl_iodmin(SYSCTL_HANDLER_ARGS)
 	 * than the new minimum, create some more.
 	 */
 	for (i = nfs_iodmin - nfs_numasync; i > 0; i--)
-		nfs_nfsiodnew(0);
+		nfs_nfsiodnew_sync();
 out:
 	mtx_unlock(&nfs_iod_mtx);	
 	return (0);
@@ -170,68 +162,55 @@ SYSCTL_PROC(_vfs_nfs, OID_AUTO, iodmax, CTLTYPE_UINT | CTLFLAG_RW, 0,
     sizeof (nfs_iodmax), sysctl_iodmax, "IU",
     "Max number of nfsiod kthreads");
 
+static int
+nfs_nfsiodnew_sync(void)
+{
+	int error, i;
+
+	mtx_assert(&nfs_iod_mtx, MA_OWNED);
+	for (i = 0; i < nfs_iodmax; i++) {
+		if (nfs_asyncdaemon[i] == 0) {
+			nfs_asyncdaemon[i] = 1;
+			break;
+		}
+	}
+	if (i == nfs_iodmax)
+		return (0);
+	mtx_unlock(&nfs_iod_mtx);
+	error = kproc_create(nfssvc_iod, nfs_asyncdaemon + i, NULL,
+	    RFHIGHPID, 0, "nfsiod %d", i);
+	mtx_lock(&nfs_iod_mtx);
+	if (error == 0) {
+		nfs_numasync++;
+		nfs_iodwant[i] = NFSIOD_AVAILABLE;
+	} else
+		nfs_asyncdaemon[i] = 0;
+	return (error);
+}
+
 void
 nfs_nfsiodnew_tq(__unused void *arg, int pending)
 {
-	struct nfsiod_str *nip;
 
 	mtx_lock(&nfs_iod_mtx);
-	while ((nip = STAILQ_FIRST(&nfsiodhead)) != NULL) {
-		STAILQ_REMOVE_HEAD(&nfsiodhead, ni_links);
-		mtx_unlock(&nfs_iod_mtx);
-		nip->ni_error = kproc_create(nfssvc_iod, nip->ni_inst, NULL,
-		    RFHIGHPID, 0, "nfsiod %d", nip->ni_iod);
-		nip->ni_done = 1;
-		mtx_lock(&nfs_iod_mtx);
-		wakeup(nip);
+	while (pending > 0) {
+		pending--;
+		nfs_nfsiodnew_sync();
 	}
 	mtx_unlock(&nfs_iod_mtx);
 }
 
-int
-nfs_nfsiodnew(int set_iodwant)
+void
+nfs_nfsiodnew(void)
 {
-	int error, i;
-	int newiod;
-	struct nfsiod_str *nip;
 
-	if (nfs_numasync >= nfs_iodmax)
-		return (-1);
-	newiod = -1;
-	for (i = 0; i < nfs_iodmax; i++)
-		if (nfs_asyncdaemon[i] == 0) {
-			nfs_asyncdaemon[i]++;
-			newiod = i;
-			break;
-		}
-	if (newiod == -1)
-		return (-1);
-	if (set_iodwant > 0)
-		nfs_iodwant[i] = NFSIOD_CREATED_FOR_NFS_ASYNCIO;
-	mtx_unlock(&nfs_iod_mtx);
-	nip = malloc(sizeof(*nip), M_TEMP, M_WAITOK | M_ZERO);
-	nip->ni_inst = nfs_asyncdaemon + i;
-	nip->ni_iod = newiod;
-	mtx_lock(&nfs_iod_mtx);
-	STAILQ_INSERT_TAIL(&nfsiodhead, nip, ni_links);
+	mtx_assert(&nfs_iod_mtx, MA_OWNED);
 	taskqueue_enqueue(taskqueue_thread, &nfs_nfsiodnew_task);
-	while (!nip->ni_done)
-		mtx_sleep(nip, &nfs_iod_mtx, 0, "niwt", 0);
-	error = nip->ni_error;
-	free(nip, M_TEMP);
-	if (error) {
-		if (set_iodwant > 0)
-			nfs_iodwant[i] = NFSIOD_NOT_AVAILABLE;
-		return (-1);
-	}
-	nfs_numasync++;
-	return (newiod);
 }
 
 static void
 nfsiod_setup(void *dummy)
 {
-	int i;
 	int error;
 
 	TUNABLE_INT_FETCH("vfs.nfs.iodmin", &nfs_iodmin);
@@ -240,8 +219,8 @@ nfsiod_setup(void *dummy)
 	if (nfs_iodmin > NFS_MAXASYNCDAEMON)
 		nfs_iodmin = NFS_MAXASYNCDAEMON;
 
-	for (i = 0; i < nfs_iodmin; i++) {
-		error = nfs_nfsiodnew(0);
+	while (nfs_numasync < nfs_iodmin) {
+		error = nfs_nfsiodnew_sync();
 		if (error == -1)
 			panic("nfsiod_setup: nfs_nfsiodnew failed");
 	}