Rename the GLOB_MAXPATH flag of glob(3) to GLOB_LIMIT to be compatible

with NetBSD and OpenBSD. glob(3) will now return GLOB_NOSPACE with
errno set to 0 instead of GLOB_LIMIT when we match more than `gl_matchc'
patterns. GLOB_MAXPATH has been left as an alias of GLOB_LIMIT to
maintain backwards compatibility.

Reviewed by:	sheldonh, assar
Obtained from:	NetBSD/OpenBSD

2 files changed, 24 insertions, 21 deletions

diff --git a/lib/libc/gen/glob.3 b/lib/libc/gen/glob.3
index 5c2da21..fcbb90f 100644
--- a/lib/libc/gen/glob.3
+++ b/lib/libc/gen/glob.3
@@ -260,14 +260,15 @@ character, avoiding any special interpretation of the character.
 Expand patterns that start with
 .Ql ~
 to user name home directories.
-.It Dv GLOB_MAXPATH
+.It Dv GLOB_LIMIT
 Limit the total number of returned pathnames to the value provided in
-.Fa gl_matchc .
-If
-.Fn glob
-would match more pathnames,
-.Dv GLOB_LIMIT
-will be returned.
+.Fa gl_matchc
+(default ARG_MAX).
+This option should be set for programs
+that can be coerced into a denial of service attack
+via patterns that expand to a very large number of matches,
+such as a long string of
+.Ql */../*/.. .
 .El
 .Pp
 If, during the search, a directory is encountered that cannot be opened
@@ -377,21 +378,19 @@ file
 .Aq Pa glob.h :
 .Bl -tag -width GLOB_NOCHECK
 .It Dv GLOB_NOSPACE
-An attempt to allocate memory failed.
+An attempt to allocate memory failed, or if
+.Fa errno
+was 0
+.Dv GLOB_LIMIT
+was specified in the flags and
+.Fa pglob\->gl_matchc
+or more patterns were patched.
 .It Dv GLOB_ABEND
 The scan was stopped because an error was encountered and either
 .Dv GLOB_ERR
 was set or
 .Fa \*(lp*errfunc\*(rp\*(lp\*(rp
 returned non-zero.
-.It Dv GLOB_LIMIT
-The flag
-.Dv GLOB_MAXPATH
-was provided, and the specified limit passed to
-.Fn glob
-in
-.Fa pglob\->gl_matchc
-was reached.
 .El
 .Pp
 The arguments
@@ -427,8 +426,8 @@ compatible with the exception
 that the flags
 .Dv GLOB_ALTDIRFUNC ,
 .Dv GLOB_BRACE ,
+.Dv GLOB_LIMIT ,
 .Dv GLOB_MAGCHAR ,
-.Dv GLOB_MAXPATH ,
 .Dv GLOB_NOMAGIC ,
 .Dv GLOB_QUOTE ,
 and
diff --git a/lib/libc/gen/glob.c b/lib/libc/gen/glob.c
index 41a2136..58f8b97 100644
--- a/lib/libc/gen/glob.c
+++ b/lib/libc/gen/glob.c
@@ -170,9 +170,11 @@ glob(pattern, flags, errfunc, pglob)
 		if (!(flags & GLOB_DOOFFS))
 			pglob->gl_offs = 0;
 	}
-	if (flags & GLOB_MAXPATH)
+	if (flags & GLOB_LIMIT) {
 		limit = pglob->gl_matchc;
-	else
+		if (limit == 0)
+			limit = ARG_MAX;
+	} else
 		limit = 0;
 	pglob->gl_flags = flags & ~GLOB_MAGCHAR;
 	pglob->gl_errfunc = errfunc;
@@ -687,8 +689,10 @@ globextend(path, pglob, limit)
 	char *copy;
 	const Char *p;
 
-	if (*limit && pglob->gl_pathc > *limit)
-		return (GLOB_LIMIT);
+	if (*limit && pglob->gl_pathc > *limit) {
+		errno = 0;
+		return (GLOB_NOSPACE);
+	}
 
 	newsize = sizeof(*pathv) * (2 + pglob->gl_pathc + pglob->gl_offs);
 	pathv = pglob->gl_pathv ?