MFC r279186:

setmode(3): Make sure that setmode sets errno on failure.

Our man page already documented this partially but now
we provide more consistent behavior.

PR:		136669
Obtained from:	NetBSD (CVS rev. 1.31, 1.33)
Relnotes:	yes

2 files changed, 47 insertions, 20 deletions

diff --git a/lib/libc/gen/setmode.3 b/lib/libc/gen/setmode.3
index 5cab44a..684c2e1 100644
--- a/lib/libc/gen/setmode.3
+++ b/lib/libc/gen/setmode.3
@@ -28,7 +28,7 @@
 .\"     @(#)setmode.3	8.2 (Berkeley) 4/28/95
 .\" $FreeBSD$
 .\"
-.Dd April 28, 1995
+.Dd February 22, 2015
 .Dt SETMODE 3
 .Os
 .Sh NAME
@@ -99,7 +99,20 @@ The
 function
 may fail and set errno for any of the errors specified for the library
 routine
-.Xr malloc 3 .
+.Xr malloc 3
+or
+.Xr strtol 3 .
+In addition,
+.Fn setmode
+will fail and set
+.Va errno
+to:
+.Bl -tag -width Er
+.It Bq Er EINVAL
+The
+.Fa mode
+argument does not represent a valid mode.
+.El
 .Sh SEE ALSO
 .Xr chmod 1 ,
 .Xr stat 2 ,
diff --git a/lib/libc/gen/setmode.c b/lib/libc/gen/setmode.c
index 3966fd0..9dc5d9fb 100644
--- a/lib/libc/gen/setmode.c
+++ b/lib/libc/gen/setmode.c
@@ -41,6 +41,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 
 #include <ctype.h>
+#include <errno.h>
+#include <limits.h>
 #include <signal.h>
 #include <stddef.h>
 #include <stdlib.h>
@@ -66,7 +68,7 @@ typedef struct bitcmd {
 #define	CMD2_OBITS	0x08
 #define	CMD2_UBITS	0x10
 
-static BITCMD	*addcmd(BITCMD *, int, int, int, u_int);
+static BITCMD	*addcmd(BITCMD *, mode_t, mode_t, mode_t, mode_t);
 static void	 compress_mode(BITCMD *);
 #ifdef SETMODE_DEBUG
 static void	 dumpmode(BITCMD *);
@@ -151,33 +153,32 @@ common:			if (set->cmd2 & CMD2_CLR) {
 		BITCMD *newset;						\
 		setlen += SET_LEN_INCR;					\
 		newset = realloc(saveset, sizeof(BITCMD) * setlen);	\
-		if (!newset) {						\
-			if (saveset)					\
-				free(saveset);				\
-			saveset = NULL;					\
-			return (NULL);					\
-		}							\
+		if (newset == NULL)					\
+			goto out;					\
 		set = newset + (set - saveset);				\
 		saveset = newset;					\
 		endset = newset + (setlen - 2);				\
 	}								\
-	set = addcmd(set, (a), (b), (c), (d))
+	set = addcmd(set, (mode_t)(a), (mode_t)(b), (mode_t)(c), (d))
 
 #define	STANDARD_BITS	(S_ISUID|S_ISGID|S_IRWXU|S_IRWXG|S_IRWXO)
 
 void *
 setmode(const char *p)
 {
-	int perm, who;
+	int serrno;
 	char op, *ep;
 	BITCMD *set, *saveset, *endset;
 	sigset_t sigset, sigoset;
-	mode_t mask;
-	int equalopdone=0, permXbits, setlen;
+	mode_t mask, perm, permXbits, who;
 	long perml;
+	int equalopdone;
+	int setlen;
 
-	if (!*p)
+	if (!*p) {
+		errno = EINVAL;
 		return (NULL);
+	}
 
 	/*
 	 * Get a copy of the mask for the permissions that are mask relative.
@@ -203,10 +204,17 @@ setmode(const char *p)
 	 * or illegal bits.
 	 */
 	if (isdigit((unsigned char)*p)) {
+		errno = 0;
 		perml = strtol(p, &ep, 8);
-		if (*ep || perml < 0 || perml & ~(STANDARD_BITS|S_ISTXT)) {
-			free(saveset);
-			return (NULL);
+		if (*ep) {
+			errno = EINVAL;
+			goto out;
+		}
+		if (errno == ERANGE && (perml == LONG_MAX || perml == LONG_MIN))
+			goto out;
+		if (perml & ~(STANDARD_BITS|S_ISTXT)) {
+			errno = EINVAL;
+			goto out;
 		}
 		perm = (mode_t)perml;
 		ADDCMD('=', (STANDARD_BITS|S_ISTXT), perm, mask);
@@ -218,6 +226,7 @@ setmode(const char *p)
 	 * Build list of structures to set/clear/copy bits as described by
 	 * each clause of the symbolic mode.
 	 */
+	equalopdone = 0;
 	for (;;) {
 		/* First, find out which bits might be modified. */
 		for (who = 0;; ++p) {
@@ -240,8 +249,8 @@ setmode(const char *p)
 		}
 
 getop:		if ((op = *p++) != '+' && op != '-' && op != '=') {
-			free(saveset);
-			return (NULL);
+			errno = EINVAL;
+			goto out;
 		}
 		if (op == '=')
 			equalopdone = 0;
@@ -330,10 +339,15 @@ apply:		if (!*p)
 	dumpmode(saveset);
 #endif
 	return (saveset);
+out:
+	serrno = errno;
+	free(saveset);
+	errno = serrno;
+	return NULL;
 }
 
 static BITCMD *
-addcmd(BITCMD *set, int op, int who, int oparg, u_int mask)
+addcmd(BITCMD *set, mode_t op, mode_t who, mode_t oparg, mode_t mask)
 {
 	switch (op) {
 	case '=':