Signal an error instead of giving the caller less memory than they asked

for when num * size would cause integer overflow.

MFC after:	1 week

1 files changed, 7 insertions, 0 deletions

diff --git a/lib/libc/stdlib/calloc.c b/lib/libc/stdlib/calloc.c
index ced9273..863e546 100644
--- a/lib/libc/stdlib/calloc.c
+++ b/lib/libc/stdlib/calloc.c
@@ -37,6 +37,8 @@ static char sccsid[] = "@(#)calloc.c	8.1 (Berkeley) 6/4/93";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <errno.h>
+#include <limits.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -47,6 +49,11 @@ calloc(num, size)
 {
 	void *p;
 
+	if (size != 0 && SIZE_MAX / size < num) {
+		errno = ENOMEM;
+		return (NULL);
+	}
+
 	size *= num;
 	if ( (p = malloc(size)) )
 		bzero(p, size);