Performance enhancements (I hope) and new stuff:

yp_dblookup.c:

- Implement database handle caching. What this means is that instead
  of opening and closing map databases for each request, we open a
  database and save the handle (and, if requested, the key index)
  in an array. This saves a bit of overhead on things like repeated
  YPPROC_NEXT calls, such as you'd get from getpwent(). Normally,
  each YPPROC_NEXT would require open()ing the database, seeking
  to the location supplied by the caller (which is time consuming with
  hash databases as the R_CURSOR flag doesn't work), reading the
  data, close()ing the database and then shipping the data off to
  the caller. The system call overhead is prohibitive, especially
  with very large maps. By caching the handle to an open database,
  we elimitate at least the open()/close() system calls, as well
  as the associated DB setup and tear-down operations, for a large
  percentage of the time. This improves performance substantially at
  the cost of consuming a little more memory than before.

  Note that all the caching support is surrounded by #ifdef DB_CACHE
  so that this same source module can still be used by other programs
  that don't need it.

- Make yp_open_db() call yp_validdomain(). Doing it here saves cycles
  when caching is enabled since a hit on the map cache list by
  definition means that the domain being referenced is valid.

- Also make yp_open_db() check for exhaustion of file descriptors,
  just in case.

yp_server.c:

- Reorganize things a little to take advantage of the database
  handle caching. Add a call to yp_flush_all() in ypproc_clear_2_svc().

- Remove calls to yp_validdomain() from some of the service procedures.
  yp_validdomain() is called inside yp_open_db() now, so procedures that
  call into the database package don't need to use yp_validdomain()
  themselves.

- Fix a bogosity in ypproc_maplist_2_svc(): don't summarily initiallize
  the result.maps pointer to NULL. This causes yp_maplist_free()
  to fail and leaks memory.

- Make ypproc_master_2_svc() copy the string it gets from the database
  package into a private static buffer before trying to NUL terminate it.
  This is necessary with the DB handle caching: stuffing a NUL into the
  data returned by DB package will goof it up internally.

yp_main.c:

- Stuff for DB handle caching: call yp_init_dbs() to clear the
  handle array and add call to yp_flush_all() to the SIGHUP
  signal handler.

Makefile.yp:

- Reorganize to deal with database caching. yp_mkdb(8) can now be used
  to send a YPPROC_CLEAR signal to ypserv(8). Call it after each map
  is created to refresh ypserv's cache.

- Add support for mail.alias map.
  Contributed by Mike Murphy (mrm@sceard.com).

- Make default location for the netgroups source file be /var/yp/netgroup
  instead of /etc/netgroup.

mkaliases:

- New file: script to generate mail.alias map.
  Contributed by Mike Murphy (mrm@sceard.com).

Makefile:

- Install Makefile.yp as /var/yp/Makefile.dist and link it to
  /var/yp/Makefile only if /var/yp/Makefile doesn't already exist.
  Suggested by Peter Wemm.

- Install new mkaliases script in /usr/libexec along with mknetid.

- Use somewhat saner approach to generating rpcgen-dependent files
  as suggested by Garrett Wollman.

1 files changed, 292 insertions, 31 deletions

diff --git a/usr.sbin/ypserv/yp_dblookup.c b/usr.sbin/ypserv/yp_dblookup.c
index 3f2415b..0cffdb3 100644
--- a/usr.sbin/ypserv/yp_dblookup.c
+++ b/usr.sbin/ypserv/yp_dblookup.c
@@ -29,7 +29,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: yp_dblookup.c,v 1.3 1996/02/04 05:39:35 wpaul Exp $
+ *	$Id: yp_dblookup.c,v 1.7 1996/04/27 17:50:18 wpaul Exp wpaul $
  *
  */
 #include <stdio.h>
@@ -40,9 +40,10 @@
 #include <unistd.h>
 #include <db.h>
 #include <sys/stat.h>
+#include <sys/param.h>
 #include <errno.h>
 #include <paths.h>
-#include "yp.h"
+#include <rpcsvc/yp.h>
 #include "yp_extern.h"
 
 int ypdb_debug = 0;
@@ -52,22 +53,212 @@ int yp_errno = YP_TRUE;
 HASHINFO openinfo = {
 	4096,		/* bsize */
 	32,		/* ffactor */
-	256,		/* nelem */
-	2048 * 1024, 	/* cachesize */
+	512,		/* nelem */
+	2048 * 512, 	/* cachesize */
 	NULL,		/* hash */
 	0,		/* lorder */
 };
 
+#ifdef DB_CACHE
+#define MAXDBS 20
+#define LASTDB (MAXDBS - 1)
+
+struct dbent {
+	DB *dbp;
+	char *name;
+	char *key;
+	int size;
+};
+
+static struct dbent *dbs[MAXDBS];
+static int numdbs = 0;
+
 /*
- * Open a DB database
+ * Make sure all the DB entries are NULL to start with.
  */
-DB *yp_open_db(domain, map)
+void yp_init_dbs()
+{
+	register int i;
+
+	for (i = 0; i < MAXDBS; i++);
+		dbs[i] = NULL;
+	return;
+}
+
+static inline void yp_flush(i)
+	register int i;
+{
+	(void)(dbs[i]->dbp->close)(dbs[i]->dbp);
+	dbs[i]->dbp = NULL;
+	free(dbs[i]->name);
+	dbs[i]->name = NULL;
+	dbs[i]->key = NULL;
+	dbs[i]->size = 0;
+	free(dbs[i]);
+	dbs[i] = NULL;
+	numdbs--;
+}
+
+/*
+ * Close all databases and erase all database names.
+ * Don't free the memory allocated for each DB entry though: we
+ * can just reuse it later.
+ */
+void yp_flush_all()
+{
+	register int i;
+
+	for (i = 0; i < MAXDBS; i++) {
+		if (dbs[i] != NULL && dbs[i]->dbp != NULL) {
+			yp_flush(i);
+		}
+	}
+}
+
+
+/*
+ * Add a DB handle and database name to the cache. We only maintain
+ * fixed number of entries in the cache, so if we're asked to store
+ * a new entry when all our slots are already filled, we have to kick
+ * out the entry in the last slot to make room.
+ */
+static inline void yp_add_db(dbp, name, size)
+	DB *dbp;
+	char *name;
+	int size;
+{
+	register int i;
+	register struct dbent *tmp;
+	static int count = 0;
+
+
+	tmp = dbs[LASTDB];
+
+	/* Rotate */
+	for (i = LASTDB; i > 0; i--)
+		dbs[i] = dbs[i - 1];
+
+	dbs[0] = tmp;
+
+	if (dbs[0]) {
+		if (ypdb_debug)
+			yp_error("table overflow -- releasing last slot");
+		yp_flush(0);
+	}
+
+	/*
+	 * Add the new entry. We allocate memory for the dbent
+	 * structure if we need it. We shoudly only end up calling
+	 * malloc(2) MAXDB times. Once all the slots are filled, we
+	 * hold onto the memory and recycle it.
+	 */
+	if (dbs[0] == NULL) {
+		count++;
+		if (ypdb_debug)
+			yp_error("allocating new DB member (%d)", count);
+		dbs[0] = (struct dbent *)malloc(sizeof(struct dbent));
+		bzero((char *)dbs[0], sizeof(struct dbent));
+	}
+
+	numdbs++;
+	dbs[0]->dbp = dbp;
+	dbs[0]->name = strdup(name);
+	dbs[0]->size = size;
+	return;
+}
+
+/*
+ * Search the list for a database matching 'name.' If we find it,
+ * move it to the head of the list and return its DB handle. If
+ * not, just fail: yp_open_db_cache() will subsequently try to open
+ * the database itself and call yp_add_db() to add it to the
+ * list.
+ *
+ * The search works like this:
+ *
+ * - The caller specifies the name of a database to locate. We try to
+ *   find an entry in our list with a matching name.
+ *
+ * - If the caller doesn't specify a key or size, we assume that the
+ *   first entry that we encounter with a matching name is returned.
+ *   This will result in matches regardless of the pointer index.
+ *
+ * - If the caller also specifies a key and length, we check name
+ *   matches to see if their saved key indexes and lengths also match.
+ *   This lets us return a DB handle that's already positioned at the
+ *   correct location within a database.
+ *
+ * - Once we have a match, it gets migrated to the top of the list
+ *   array so that it will be easier to find if another request for
+ *   the same database comes in later.
+ */
+static inline DB *yp_find_db(name, key, size)
+	char *name;
+	char *key;
+{
+	register int i, j;
+	register struct dbent *tmp;
+
+	for (i = 0; i < numdbs; i++) {
+		if (dbs[i]->name != NULL && !strcmp(dbs[i]->name, name)) {
+			if (size) {
+				if (size != dbs[i]->size ||
+					strncmp(dbs[i]->key, key, size))
+					continue;
+			} else {
+				if (dbs[i]->size) {
+					continue;
+				}
+			}
+			if (i > 0) {
+				tmp = dbs[i];
+				for (j = i; j > 0; j--)
+					dbs[j] = dbs[j - 1];
+				dbs[0] = tmp;
+			}
+			return(dbs[0]->dbp);
+		}
+	}
+	return(NULL);
+}
+
+/*
+ * Open a DB database and cache the handle for later use. We first
+ * check the cache to see if the required database is already open.
+ * If so, we fetch the handle from the cache. If not, we try to open
+ * the database and save the handle in the cache for later use.
+ */
+DB *yp_open_db_cache(domain, map, key, size)
 	const char *domain;
 	const char *map;
+	const char *key;
+	const int size;
 {
-	DB *dbp;
-	char buf[1025];
+	DB *dbp = NULL;
+	char buf[MAXPATHLEN + 2];
 
+	snprintf(buf, sizeof(buf), "%s/%s", domain, map);
+
+	if ((dbp = yp_find_db((char *)&buf, key, size)) != NULL) {
+		return(dbp);
+	} else {
+		if ((dbp = yp_open_db(domain, map)) != NULL) 
+			yp_add_db(dbp, (char *)&buf, size);
+	}
+
+	return (dbp);
+}
+#endif
+
+/*
+ * Open a DB database.
+ */
+DB *yp_open_db(domain, map)
+	const char *domain;
+	const char *map;
+{
+	DB *dbp = NULL;
+	char buf[MAXPATHLEN + 2];
 
 	yp_errno = YP_TRUE;
 
@@ -76,12 +267,32 @@ DB *yp_open_db(domain, map)
 		return (NULL);
 	}
 
+#ifdef DB_CACHE
+	if (yp_validdomain(domain)) {
+		yp_errno = YP_NODOM;
+		return(NULL);
+	}
+#endif
 	snprintf(buf, sizeof(buf), "%s/%s/%s", yp_dir, domain, map);
 
-	dbp = dbopen(buf,O_RDONLY|O_EXCL, PERM_SECURE, DB_HASH, NULL);
+#ifdef DB_CACHE
+again:
+#endif
+	dbp = dbopen(buf,O_RDONLY, PERM_SECURE, DB_HASH, NULL);
 
 	if (dbp == NULL) {
 		switch(errno) {
+#ifdef DB_CACHE
+		case ENFILE:
+			/*
+			 * We ran out of file descriptors. Nuke an
+			 * open one and try again.
+			 */
+			yp_error("ran out of file descriptors");
+			yp_flush(numdbs - 1);
+			goto again;
+			break;
+#endif
 		case ENOENT:
 			yp_errno = YP_NOMAP;
 			break;
@@ -117,7 +328,7 @@ int yp_get_record(domain,map,key,data,allow)
 	int allow;
 {
 	DB *dbp;
-	int rval;
+	int rval = 0;
 
 	if (ypdb_debug)
 		yp_error("Looking up key [%.*s] in map [%s]",
@@ -131,31 +342,47 @@ int yp_get_record(domain,map,key,data,allow)
 	if (!allow && !strncmp(key->data, "YP_", 3))
 		return(YP_NOKEY);
 
+#ifdef DB_CACHE
+	if ((dbp = yp_open_db_cache(domain, map, NULL, 0)) == NULL) {
+#else
 	if ((dbp = yp_open_db(domain, map)) == NULL) {
+#endif
 		return(yp_errno);
 	}
 
-	if ((rval = (dbp->get)(dbp,key,data,0)) != 0) {
+	if ((rval = (dbp->get)(dbp, key, data, 0)) != 0) {
+#ifdef DB_CACHE
+		dbs[0]->size = 0;
+#else
 		(void)(dbp->close)(dbp);
+#endif
 		if (rval == 1)
 			return(YP_NOKEY);
 		else
 			return(YP_BADDB);
 	}
 
-	(void)(dbp->close)(dbp);
-
 	if (ypdb_debug)
 		yp_error("Result of lookup: key: [%.*s] data: [%.*s]",
 			 key->size, key->data, data->size, data->data);
 
+#ifdef DB_CACHE
+	if (dbs[0]->size) {
+		dbs[0]->key = key->data;
+		dbs[0]->size = key->size;
+	}
+#else
+	(void)(dbp->close)(dbp);
+#endif
+
 	return(YP_TRUE);
 }
 
-int yp_first_record(dbp,key,data)
+int yp_first_record(dbp,key,data,allow)
 	const DB *dbp;
 	DBT *key;
 	DBT *data;
+	int allow;
 {
 	int rval;
 
@@ -163,6 +390,9 @@ int yp_first_record(dbp,key,data)
 		yp_error("Retrieving first key in map.");
 
 	if ((rval = (dbp->seq)(dbp,key,data,R_FIRST)) != 0) {
+#ifdef DB_CACHE
+		dbs[0]->size = 0;
+#endif
 		if (rval == 1)
 			return(YP_NOKEY);
 		else 
@@ -170,8 +400,11 @@ int yp_first_record(dbp,key,data)
 	}
 
 	/* Avoid passing back magic "YP_*" records. */
-	while (!strncmp(key->data, "YP_", 3)) {
+	while (!strncmp(key->data, "YP_", 3) && !allow) {
 		if ((rval = (dbp->seq)(dbp,key,data,R_NEXT)) != 0) {
+#ifdef DB_CACHE
+			dbs[0]->size = 0;
+#endif
 			if (rval == 1)
 				return(YP_NOKEY);
 			else
@@ -183,20 +416,29 @@ int yp_first_record(dbp,key,data)
 		yp_error("Result of lookup: key: [%.*s] data: [%.*s]",
 			 key->size, key->data, data->size, data->data);
 
+#ifdef DB_CACHE
+	if (dbs[0]->size) {
+		dbs[0]->key = key->data;
+		dbs[0]->size = key->size;
+	}
+#endif
+
 	return(YP_TRUE);
 }
 
-int yp_next_record(dbp,key,data,all)
+int yp_next_record(dbp,key,data,all,allow)
 	const DB *dbp;
 	DBT *key;
 	DBT *data;
 	int all;
+	int allow;
 {
-	DBT lkey, ldata;
+	static DBT lkey = { NULL, 0 };
+	static DBT ldata = { NULL, 0 };
 	int rval;
 
 	if (key == NULL || key->data == NULL) {
-		rval = yp_first_record(dbp,key,data);
+		rval = yp_first_record(dbp,key,data,allow);
 		if (rval == YP_NOKEY)
 			return(YP_NOMORE);
 		else
@@ -208,29 +450,48 @@ int yp_next_record(dbp,key,data,all)
 			  key->size, key->data);
 
 	if (!all) {
-		(dbp->seq)(dbp,&lkey,&ldata,R_FIRST);
-		while(strncmp((char *)key->data,lkey.data,(int)key->size) ||
-		      key->size != lkey.size)
-			(dbp->seq)(dbp,&lkey,&ldata,R_NEXT);
+#ifndef DB_CACHE
+		if (key->size != lkey.size ||
+			strncmp(key->data, lkey.data, key->size)) {
+#else
+		if (!dbs[0]->size) {
+#endif
+			(dbp->seq)(dbp,&lkey,&ldata,R_FIRST);
+			while(strncmp((char *)key->data,lkey.data,
+				(int)key->size) || key->size != lkey.size)
+				(dbp->seq)(dbp,&lkey,&ldata,R_NEXT);
+		}
 	}
 
-	if ((dbp->seq)(dbp,&lkey,&ldata,R_NEXT))
+	if ((dbp->seq)(dbp,key,data,R_NEXT)) {
+#ifdef DB_CACHE
+		dbs[0]->size = 0;
+#endif
 		return(YP_NOMORE);
+	}
 
 	/* Avoid passing back magic "YP_*" records. */
-	while (!strncmp(lkey.data, "YP_", 3))
-		if ((dbp->seq)(dbp,&lkey,&ldata,R_NEXT))
+	while (!strncmp(key->data, "YP_", 3) && !allow)
+		if ((dbp->seq)(dbp,key,data,R_NEXT)) {
+#ifdef DB_CACHE
+			dbs[0]->size = 0;
+#endif
 			return(YP_NOMORE);
-
-	if ((dbp->get)(dbp,&lkey,&ldata,0))
-		return(YP_FALSE);
-
-	*key = lkey;
-	*data = ldata;
+		}
 
 	if (ypdb_debug)
 		yp_error("Result of lookup: key: [%.*s] data: [%.*s]",
 			 key->size, key->data, data->size, data->data);
 
+#ifdef DB_CACHE
+	if (dbs[0]->size) {
+		dbs[0]->key = key->data;
+		dbs[0]->size = key->size;
+	}
+#else
+	lkey.data = key->data;
+	lkey.size = key->size;
+#endif
+
 	return(YP_TRUE);
 }