MFC r278803, r278905:

ulimit(3): Fix broken check.

The existing implementation had a broken comparison that could
overflow and return confusing values.  Replace this with a check
that avoids the overflow before it happens.

Consistently return a maximum value also on the case of negative
arguments since negative is considered an overflow and means
infinity for our current setrlimit().

New revamped version is credited to Bruce Evans.

CID:		1199295

1 files changed, 7 insertions, 5 deletions

diff --git a/lib/libc/gen/ulimit.c b/lib/libc/gen/ulimit.c
index e1bc020..2c090c0 100644
--- a/lib/libc/gen/ulimit.c
+++ b/lib/libc/gen/ulimit.c
@@ -40,7 +40,7 @@ ulimit(int cmd, ...)
 {
 	struct rlimit limit;
 	va_list ap;
-	long arg;
+	rlim_t arg;
 
 	if (cmd == UL_GETFSIZE) {
 		if (getrlimit(RLIMIT_FSIZE, &limit) == -1)
@@ -53,14 +53,16 @@ ulimit(int cmd, ...)
 		va_start(ap, cmd);
 		arg = va_arg(ap, long);
 		va_end(ap);
-		limit.rlim_max = limit.rlim_cur = (rlim_t)arg * 512;
+		if (arg < 0)
+			arg = LONG_MAX;
+		if (arg > RLIM_INFINITY / 512)
+			arg = RLIM_INFINITY / 512;
+		limit.rlim_max = limit.rlim_cur = arg * 512;
 
 		/* The setrlimit() function sets errno to EPERM if needed. */
 		if (setrlimit(RLIMIT_FSIZE, &limit) == -1)
 			return (-1);
-		if (arg * 512 > LONG_MAX)
-			return (LONG_MAX);
-		return (arg);
+		return ((long)arg);
 	} else {
 		errno = EINVAL;
 		return (-1);