Autoconfigure nfsd threads based on ncpu.

Rick Macklem and I discussed the default number of nfsd threads and
concluded that it is too low to perform adiquitely on today's hardware.

We decided to auto tune the number of nfsds based on the number of cpus
in the system.

While I'm here I've also added:

1) ability to set the minthreads/maxthreads from userland.
2) ability to run nfsd in debug mode via the cli.

Reviewed by: rmacklem
MFC after: 2 weeks

1 files changed, 97 insertions, 20 deletions

diff --git a/usr.sbin/nfsd/nfsd.c b/usr.sbin/nfsd/nfsd.c
index c269e47..f5cb5e2 100644
--- a/usr.sbin/nfsd/nfsd.c
+++ b/usr.sbin/nfsd/nfsd.c
@@ -53,6 +53,7 @@ static const char rcsid[] =
 #include <sys/module.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <sys/sysctl.h>
 #include <sys/ucred.h>
 
 #include <rpc/rpc.h>
@@ -71,14 +72,11 @@ static const char rcsid[] =
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <sysexits.h>
+
+#include <getopt.h>
 
-/* Global defs */
-#ifdef DEBUG
-#define	syslog(e, s...)	fprintf(stderr,s)
-static int	debug = 1;
-#else
 static int	debug = 0;
-#endif
 
 #define	NFSD_STABLERESTART	"/var/db/nfs-stablerestart"
 #define	NFSD_STABLEBACKUP	"/var/db/nfs-stablerestart.bak"
@@ -86,11 +84,26 @@ static int	debug = 0;
 #define	DEFNFSDCNT	 4
 static pid_t children[MAXNFSDCNT]; /* PIDs of children */
 static int nfsdcnt;		/* number of children */
+static int nfsdcnt_set;
+static int minthreads;
+static int maxthreads;
 static int new_syscall;
 static int run_v4server = 1;	/* Force running of nfsv4 server */
 static int nfssvc_nfsd;		/* Set to correct NFSSVC_xxx flag */
 static int stablefd = -1;	/* Fd for the stable restart file */
 static int backupfd;		/* Fd for the backup stable restart file */
+static const char *getopt_shortopts;
+static const char *getopt_usage;
+
+static int minthreads_set;
+static int maxthreads_set;
+
+static struct option longopts[] = {
+	{ "debug", no_argument, &debug, 1 },
+	{ "minthreads", required_argument, &minthreads_set, 1 },
+	{ "maxthreads", required_argument, &maxthreads_set, 1 },
+	{ NULL, 0, NULL, 0}
+};
 
 void	cleanup(int);
 void	child_cleanup(int);
@@ -145,26 +158,28 @@ main(int argc, char **argv)
 	int udpflag, ecode, error, s, srvcnt;
 	int bindhostc, bindanyflag, rpcbreg, rpcbregcnt;
 	int nfssvc_addsock;
+	int longindex = 0;
+	const char *lopt;
 	char **bindhost = NULL;
 	pid_t pid;
 
 	nfsdcnt = DEFNFSDCNT;
 	unregister = reregister = tcpflag = maxsock = 0;
 	bindanyflag = udpflag = connect_type_cnt = bindhostc = 0;
-#define	GETOPT	"ah:n:rdtueo"
-#define	USAGE	"[-ardtueo] [-n num_servers] [-h bindip]"
-	while ((ch = getopt(argc, argv, GETOPT)) != -1)
+	getopt_shortopts = "ah:n:rdtueo";
+	getopt_usage =
+	    "usage:\n"
+	    "  nfsd [-ardtueo] [-h bindip]\n"
+	    "       [-n numservers] [--minthreads #] [--maxthreads #]\n";
+	while ((ch = getopt_long(argc, argv, getopt_shortopts, longopts,
+		    &longindex)) != -1)
 		switch (ch) {
 		case 'a':
 			bindanyflag = 1;
 			break;
 		case 'n':
+			nfsdcnt_set = 1;
 			nfsdcnt = atoi(optarg);
-			if (nfsdcnt < 1 || nfsdcnt > MAXNFSDCNT) {
-				warnx("nfsd count %d; reset to %d", nfsdcnt,
-				    DEFNFSDCNT);
-				nfsdcnt = DEFNFSDCNT;
-			}
 			break;
 		case 'h':
 			bindhostc++;
@@ -193,6 +208,14 @@ main(int argc, char **argv)
 		case 'o':
 			run_v4server = 0;
 			break;
+		case 0:
+			lopt = longopts[longindex].name;
+			if (!strcmp(lopt, "minthreads")) {
+				minthreads = atoi(optarg);
+			} else if (!strcmp(lopt, "maxthreads")) {
+				maxthreads = atoi(optarg);
+			}
+			break;
 		default:
 		case '?':
 			usage();
@@ -209,6 +232,7 @@ main(int argc, char **argv)
 	if (argc > 1)
 		usage();
 	if (argc == 1) {
+		nfsdcnt_set = 1;
 		nfsdcnt = atoi(argv[0]);
 		if (nfsdcnt < 1 || nfsdcnt > MAXNFSDCNT) {
 			warnx("nfsd count %d; reset to %d", nfsdcnt,
@@ -356,7 +380,7 @@ main(int argc, char **argv)
 	(void)signal(SIGCHLD, reapchild);
 	(void)signal(SIGUSR2, backup_stable);
 
-	openlog("nfsd", LOG_PID, LOG_DAEMON);
+	openlog("nfsd", LOG_PID | (debug ? LOG_PERROR : 0), LOG_DAEMON);
 
 	/*
 	 * For V4, we open the stablerestart file and call nfssvc()
@@ -374,13 +398,13 @@ main(int argc, char **argv)
 	if (run_v4server > 0) {
 		open_stable(&stablefd, &backupfd);
 		if (stablefd < 0) {
-			syslog(LOG_ERR, "Can't open %s\n", NFSD_STABLERESTART);
+			syslog(LOG_ERR, "Can't open %s: %m\n", NFSD_STABLERESTART);
 			exit(1);
 		}
 		/* This system call will fail for old kernels, but that's ok. */
 		nfssvc(NFSSVC_BACKUPSTABLE, NULL);
 		if (nfssvc(NFSSVC_STABLERESTART, (caddr_t)&stablefd) < 0) {
-			syslog(LOG_ERR, "Can't read stable storage file\n");
+			syslog(LOG_ERR, "Can't read stable storage file: %m\n");
 			exit(1);
 		}
 		nfssvc_addsock = NFSSVC_NFSDADDSOCK;
@@ -401,6 +425,16 @@ main(int argc, char **argv)
 	}
 
 	if (!new_syscall) {
+		if (nfsdcnt < 1) {
+			warnx("nfsd count too low %d; reset to %d", nfsdcnt,
+			    DEFNFSDCNT);
+			nfsdcnt = DEFNFSDCNT;
+		}
+		if (nfsdcnt > MAXNFSDCNT) {
+			warnx("nfsd counta too high %d; reset to %d", nfsdcnt,
+			    DEFNFSDCNT);
+			nfsdcnt = MAXNFSDCNT;
+		}
 		/* If we use UDP only, we start the last server below. */
 		srvcnt = tcpflag ? nfsdcnt : nfsdcnt - 1;
 		for (i = 0; i < srvcnt; i++) {
@@ -855,7 +889,7 @@ setbindhost(struct addrinfo **ai, const char *bindhost, struct addrinfo hints)
 void
 usage(void)
 {
-	(void)fprintf(stderr, "usage: nfsd %s\n", USAGE);
+	(void)fprintf(stderr, "%s", getopt_usage);
 	exit(1);
 }
 
@@ -923,6 +957,29 @@ nfsd_exit(int status)
 	exit(status);
 }
 
+static int
+get_tuned_nfsdcount(void)
+{
+	int ncpu, error, tuned_nfsdcnt;
+	size_t ncpu_size;
+
+	ncpu_size = sizeof(ncpu);
+	error = sysctlbyname("hw.ncpu", &ncpu, &ncpu_size, NULL, 0);
+	if (error) {
+		warnx("sysctlbyname(hw.ncpu) failed defaulting to %d nfs servers",
+		    DEFNFSDCNT);
+		tuned_nfsdcnt = DEFNFSDCNT;
+	} else {
+		tuned_nfsdcnt = ncpu * 8;
+	}
+	if (!new_syscall && tuned_nfsdcnt > MAXNFSDCNT) {
+		warnx("nfsd count %d; truncated to %d", tuned_nfsdcnt,
+		    MAXNFSDCNT);
+		tuned_nfsdcnt = MAXNFSDCNT;
+	}
+	return tuned_nfsdcnt;
+}
+
 void
 start_server(int master)
 {
@@ -952,8 +1009,28 @@ start_server(int master)
 			}
 		}
 		nfsdargs.principal = principal;
-		nfsdargs.minthreads = nfsdcnt;
-		nfsdargs.maxthreads = nfsdcnt;
+
+		if (minthreads_set) {
+			nfsdargs.minthreads = minthreads;
+			if (!maxthreads_set)
+				nfsdargs.maxthreads = minthreads;
+		}
+		if (maxthreads_set) {
+			nfsdargs.maxthreads = maxthreads;
+			if (!minthreads_set)
+				nfsdargs.minthreads = maxthreads;
+		}
+		if (nfsdcnt_set) {
+			nfsdargs.minthreads = nfsdcnt;
+			nfsdargs.maxthreads = nfsdcnt;
+		}
+		if (!minthreads_set && !maxthreads_set && !nfsdcnt_set) {
+			int tuned_nfsdcnt;
+
+			tuned_nfsdcnt = get_tuned_nfsdcount();
+			nfsdargs.minthreads = tuned_nfsdcnt;
+			nfsdargs.maxthreads = tuned_nfsdcnt;
+		}
 		error = nfssvc(nfssvc_nfsd, &nfsdargs);
 		if (error < 0 && errno == EAUTH) {
 			/*