- When there is no room for returning the result, nss backend

  have to return ERANGE and terminate with NS_RETURN.
- When gethostbyname_r(3) and the friends end with an error,
  set errno to the value nss backend returns, and return errno
  value.

PR:		kern/131623
MFC after:	2 weeks

11 files changed, 109 insertions, 49 deletions

diff --git a/lib/libc/net/gethostbydns.c b/lib/libc/net/gethostbydns.c
index ff93972..dc10479 100644
--- a/lib/libc/net/gethostbydns.c
+++ b/lib/libc/net/gethostbydns.c
@@ -536,9 +536,12 @@ _dns_gethostbyname(void *rval, void *cb_data, va_list ap)
 		return (NS_NOTFOUND);
 	}
 	if (__copy_hostent(&he, hptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
+	RES_SET_H_ERRNO(statp, NETDB_SUCCESS);
 	*((struct hostent **)rval) = hptr;
 	return (NS_SUCCESS);
 }
@@ -683,11 +686,13 @@ _dns_gethostbyaddr(void *rval, void *cb_data, va_list ap)
 		he.h_addrtype = AF_INET6;
 		he.h_length = NS_IN6ADDRSZ;
 	}
-	RES_SET_H_ERRNO(statp, NETDB_SUCCESS);
 	if (__copy_hostent(&he, hptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
+	RES_SET_H_ERRNO(statp, NETDB_SUCCESS);
 	*((struct hostent **)rval) = hptr;
 	return (NS_SUCCESS);
 }
diff --git a/lib/libc/net/gethostbyht.c b/lib/libc/net/gethostbyht.c
index 8c41d9a..4253d49 100644
--- a/lib/libc/net/gethostbyht.c
+++ b/lib/libc/net/gethostbyht.c
@@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$");
 #include <netdb.h>
 #include <stdio.h>
 #include <ctype.h>
+#include <errno.h>
 #include <string.h>
 #include <stdarg.h>
 #include <nsswitch.h>
@@ -192,8 +193,11 @@ gethostent_r(struct hostent *hptr, char *buffer, size_t buflen,
 	}
 	if (gethostent_p(&he, hed, statp->options & RES_USE_INET6, statp) != 0)
 		return (-1);
-	if (__copy_hostent(&he, hptr, buffer, buflen) != 0)
-		return (-1);
+	if (__copy_hostent(&he, hptr, buffer, buflen) != 0) {
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
+		*h_errnop = statp->res_h_errno;
+		return ((errno != 0) ? errno : -1);
+	}
 	*result = hptr;
 	return (0);
 }
@@ -268,8 +272,10 @@ found:
 		return (NS_NOTFOUND);
 	}
 	if (__copy_hostent(&he, hptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 	*((struct hostent **)rval) = hptr;
 	return (NS_SUCCESS);
@@ -323,8 +329,10 @@ _ht_gethostbyaddr(void *rval, void *cb_data, va_list ap)
 	if (error != 0)
 		return (NS_NOTFOUND);
 	if (__copy_hostent(&he, hptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 	*((struct hostent **)rval) = hptr;
 	return (NS_SUCCESS);
diff --git a/lib/libc/net/gethostbynis.c b/lib/libc/net/gethostbynis.c
index 7921ced..11de8ae 100644
--- a/lib/libc/net/gethostbynis.c
+++ b/lib/libc/net/gethostbynis.c
@@ -288,8 +288,10 @@ _nis_gethostbyname(void *rval, void *cb_data, va_list ap)
 		return (NS_NOTFOUND);
 	}
 	if (__copy_hostent(&he, hptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 	*((struct hostent **)rval) = hptr;
 	return (NS_SUCCESS);
@@ -336,8 +338,10 @@ _nis_gethostbyaddr(void *rval, void *cb_data, va_list ap)
 		return (NS_NOTFOUND);
 	}
 	if (__copy_hostent(&he, hptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 	*((struct hostent **)rval) = hptr;
 	return (NS_SUCCESS);
diff --git a/lib/libc/net/gethostnamadr.c b/lib/libc/net/gethostnamadr.c
index ffad6f2..2dc9d02 100644
--- a/lib/libc/net/gethostnamadr.c
+++ b/lib/libc/net/gethostnamadr.c
@@ -480,8 +480,12 @@ fakeaddr(const char *name, int af, struct hostent *hp, char *buf,
 	hed->h_addr_ptrs[0] = (char *)hed->host_addr;
 	hed->h_addr_ptrs[1] = NULL;
 	he.h_addr_list = hed->h_addr_ptrs;
+	if (__copy_hostent(&he, hp, buf, buflen) != 0) {
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
+		return (-1);
+	}
 	RES_SET_H_ERRNO(statp, NETDB_SUCCESS);
-	return (__copy_hostent(&he, hp, buf, buflen));
+	return (0);
 }
 
 int
@@ -528,7 +532,7 @@ gethostbyname_internal(const char *name, int af, struct hostent *hp, char *buf,
     size_t buflen, struct hostent **result, int *h_errnop, res_state statp)
 {
 	const char *cp;
-	int rval, ret_errno;
+	int rval, ret_errno = 0;
 	char abuf[MAXDNAME];
 
 #ifdef NS_CACHING
@@ -576,7 +580,11 @@ gethostbyname_internal(const char *name, int af, struct hostent *hp, char *buf,
 	    "gethostbyname2_r", default_src, name, af, hp, buf, buflen,
 	    &ret_errno, h_errnop);
 
-	return ((rval == NS_SUCCESS) ? 0 : -1);
+	if (rval != NS_SUCCESS) {
+		errno = ret_errno;
+		return ((ret_errno != 0) ? ret_errno : -1);
+	}
+	return (0);
 }
 
 int
@@ -586,7 +594,7 @@ gethostbyaddr_r(const void *addr, socklen_t len, int af, struct hostent *hp,
 	const u_char *uaddr = (const u_char *)addr;
 	const struct in6_addr *addr6;
 	socklen_t size;
-	int rval, ret_errno;
+	int rval, ret_errno = 0;
 	res_state statp;
 
 #ifdef NS_CACHING
@@ -651,7 +659,11 @@ gethostbyaddr_r(const void *addr, socklen_t len, int af, struct hostent *hp,
 	    "gethostbyaddr_r", default_src, uaddr, len, af, hp, buf, buflen,
 	    &ret_errno, h_errnop);
 
-	return ((rval == NS_SUCCESS) ? 0 : -1);
+	if (rval != NS_SUCCESS) {
+		errno = ret_errno;
+		return ((ret_errno != 0) ? ret_errno : -1);
+	}
+	return (0);
 }
 
 struct hostent *
diff --git a/lib/libc/net/getnetbydns.c b/lib/libc/net/getnetbydns.c
index 6fd7d2f..79e40a5 100644
--- a/lib/libc/net/getnetbydns.c
+++ b/lib/libc/net/getnetbydns.c
@@ -355,8 +355,10 @@ _dns_getnetbyaddr(void *rval, void *cb_data, va_list ap)
 			net >>= 8;
 		ne.n_net = net;
 		if (__copy_netent(&ne, nptr, buffer, buflen) != 0) {
+			*errnop = errno;
+			RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 			*h_errnop = statp->res_h_errno;
-			return (NS_NOTFOUND);
+			return (NS_RETURN);
 		}
 		*((struct netent **)rval) = nptr;
 		return (NS_SUCCESS);
@@ -431,8 +433,10 @@ _dns_getnetbyname(void *rval, void *cb_data, va_list ap)
 		return (NS_NOTFOUND);
 	}
 	if (__copy_netent(&ne, nptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 	*((struct netent **)rval) = nptr;
 	return (NS_SUCCESS);
diff --git a/lib/libc/net/getnetbyht.c b/lib/libc/net/getnetbyht.c
index 2bbba06..d2df570 100644
--- a/lib/libc/net/getnetbyht.c
+++ b/lib/libc/net/getnetbyht.c
@@ -49,6 +49,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include <arpa/nameser.h>
+#include <errno.h>
 #include <netdb.h>
 #include <resolv.h>
 #include <stdio.h>
@@ -162,8 +163,11 @@ getnetent_r(struct netent *nptr, char *buffer, size_t buflen,
 	}
 	if (getnetent_p(&ne, ned) != 0)
 		return (-1);
-	if (__copy_netent(&ne, nptr, buffer, buflen) != 0)
-		return (-1);
+	if (__copy_netent(&ne, nptr, buffer, buflen) != 0) {
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
+		*h_errnop = statp->res_h_errno;
+		return ((errno != 0) ? errno : -1);
+	}
 	*result = nptr;
 	return (0);
 }
@@ -226,8 +230,10 @@ found:
 		return (NS_NOTFOUND);
 	}
 	if (__copy_netent(&ne, nptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 	*((struct netent **)rval) = nptr;
 	return (NS_SUCCESS);
@@ -272,8 +278,10 @@ _ht_getnetbyaddr(void *rval, void *cb_data, va_list ap)
 		return (NS_NOTFOUND);
 	}
 	if (__copy_netent(&ne, nptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 	*((struct netent **)rval) = nptr;
 	return (NS_SUCCESS);
diff --git a/lib/libc/net/getnetbynis.c b/lib/libc/net/getnetbynis.c
index d00c7f2..811e431 100644
--- a/lib/libc/net/getnetbynis.c
+++ b/lib/libc/net/getnetbynis.c
@@ -160,8 +160,10 @@ _nis_getnetbyname(void *rval, void *cb_data, va_list ap)
 		return (NS_NOTFOUND);
 	}
 	if (__copy_netent(&ne, nptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 	*((struct netent **)rval) = nptr;
 	return (NS_SUCCESS);
@@ -244,8 +246,10 @@ _nis_getnetbyaddr(void *rval, void *cb_data, va_list ap)
 		return (NS_NOTFOUND);
 	}
 	if (__copy_netent(&ne, nptr, buffer, buflen) != 0) {
+		*errnop = errno;
+		RES_SET_H_ERRNO(statp, NETDB_INTERNAL);
 		*h_errnop = statp->res_h_errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 	*((struct netent **)rval) = nptr;
 	return (NS_SUCCESS);
diff --git a/lib/libc/net/getnetnamadr.c b/lib/libc/net/getnetnamadr.c
index ec7e94b..9aa4d51 100644
--- a/lib/libc/net/getnetnamadr.c
+++ b/lib/libc/net/getnetnamadr.c
@@ -360,13 +360,17 @@ getnetbyname_r(const char *name, struct netent *ne, char *buffer,
 #endif
 		{ 0 }
 	};
-	int rval, ret_errno;
+	int rval, ret_errno = 0;
 
 	rval = _nsdispatch((void *)result, dtab, NSDB_NETWORKS,
 	    "getnetbyname_r", default_src, name, ne, buffer, buflen,
 	    &ret_errno, h_errorp);
 
-	return ((rval == NS_SUCCESS) ? 0 : -1);
+	if (rval != NS_SUCCESS) {
+		errno = ret_errno;
+		return ((ret_errno != 0) ? ret_errno : -1);
+	}
+	return (0);
 }
 
 int
@@ -388,13 +392,17 @@ getnetbyaddr_r(uint32_t addr, int af, struct netent *ne, char *buffer,
 #endif
 		{ 0 }
 	};
-	int rval, ret_errno;
+	int rval, ret_errno = 0;
 
 	rval = _nsdispatch((void *)result, dtab, NSDB_NETWORKS,
 	    "getnetbyaddr_r", default_src, addr, af, ne, buffer, buflen,
 	    &ret_errno, h_errorp);
 
-	return ((rval == NS_SUCCESS) ? 0 : -1);
+	if (rval != NS_SUCCESS) {
+		errno = ret_errno;
+		return ((ret_errno != 0) ? ret_errno : -1);
+	}
+	return (0);
 }
 
 struct netent *
diff --git a/lib/libc/net/getproto.c b/lib/libc/net/getproto.c
index 85b40e1..cb3a3ed 100644
--- a/lib/libc/net/getproto.c
+++ b/lib/libc/net/getproto.c
@@ -33,6 +33,7 @@ static char sccsid[] = "@(#)getproto.c	8.1 (Berkeley) 6/4/93";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <errno.h>
 #include <netdb.h>
 #include <nsswitch.h>
 #include "netdb_private.h"
@@ -72,7 +73,7 @@ files_getprotobynumber(void *retval, void *mdata, va_list ap)
 	errnop = va_arg(ap, int *);
 
 	if ((ped = __protoent_data_init()) == NULL) {
-		*errnop = -1;
+		*errnop = errno;
 		return (NS_NOTFOUND);
 	}
 
@@ -83,12 +84,12 @@ files_getprotobynumber(void *retval, void *mdata, va_list ap)
 	if (!ped->stayopen)
 		__endprotoent_p(ped);
 	if (error != 0) {
-		*errnop = -1;
+		*errnop = errno;
 		return (NS_NOTFOUND);
 	}
 	if (__copy_protoent(&pe, pptr, buffer, buflen) != 0) {
-		*errnop = -1;
-		return (NS_NOTFOUND);
+		*errnop = errno;
+		return (NS_RETURN);
 	}
 
 	*((struct protoent **)retval) = pptr;
@@ -120,10 +121,11 @@ getprotobynumber_r(int proto, struct protoent *pptr, char *buffer,
 	rv = nsdispatch(result, dtab, NSDB_PROTOCOLS, "getprotobynumber_r",
 		defaultsrc, proto, pptr, buffer, buflen, &ret_errno);
 
-	if (rv == NS_SUCCESS)
-		return (0);
-	else
-		return (ret_errno);
+	if (rv != NS_SUCCESS) {
+		errno = ret_errno;
+		return ((ret_errno != 0) ? ret_errno : -1);
+	}
+	return (0);
 }
 
 struct protoent *
diff --git a/lib/libc/net/getprotoent.c b/lib/libc/net/getprotoent.c
index dda5d18..ccd5e12 100644
--- a/lib/libc/net/getprotoent.c
+++ b/lib/libc/net/getprotoent.c
@@ -424,8 +424,10 @@ files_getprotoent_r(void *retval, void *mdata, va_list ap)
 	buflen = va_arg(ap, size_t);
 	errnop = va_arg(ap, int *);
 
-	if ((ped = __protoent_data_init()) == NULL)
-		return (-1);
+	if ((ped = __protoent_data_init()) == NULL) {
+		*errnop = errno;
+		return (NS_NOTFOUND);
+	}
 
 	if (__getprotoent_p(&pe, ped) != 0) {
 		*errnop = errno;
@@ -434,7 +436,7 @@ files_getprotoent_r(void *retval, void *mdata, va_list ap)
 
 	if (__copy_protoent(&pe, pptr, buffer, buflen) != 0) {
 		*errnop = errno;
-		return (NS_NOTFOUND);
+		return (NS_RETURN);
 	}
 
 	*((struct protoent **)retval) = pptr;
@@ -490,10 +492,11 @@ getprotoent_r(struct protoent *pptr, char *buffer, size_t buflen,
 	rv = nsdispatch(result, dtab, NSDB_PROTOCOLS, "getprotoent_r",
 	    defaultsrc, pptr, buffer, buflen, &ret_errno);
 
-	if (rv == NS_SUCCESS)
-		return (0);
-	else
-		return (ret_errno);
+	if (rv != NS_SUCCESS) {
+		errno = ret_errno;
+		return ((ret_errno != 0) ? ret_errno : -1);
+	}
+	return (0);
 }
 
 void
diff --git a/lib/libc/net/getprotoname.c b/lib/libc/net/getprotoname.c
index bead63a..08822c1 100644
--- a/lib/libc/net/getprotoname.c
+++ b/lib/libc/net/getprotoname.c
@@ -33,6 +33,7 @@ static char sccsid[] = "@(#)getprotoname.c	8.1 (Berkeley) 6/4/93";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <errno.h>
 #include <netdb.h>
 #include <nsswitch.h>
 #include <string.h>
@@ -75,7 +76,7 @@ files_getprotobyname(void *retval, void *mdata, va_list ap)
 
 
 	if ((ped = __protoent_data_init()) == NULL) {
-		*errnop = -1;
+		*errnop = errno;
 		return (NS_NOTFOUND);
 	}
 
@@ -91,12 +92,12 @@ found:
 	if (!ped->stayopen)
 		__endprotoent_p(ped);
 	if (error != 0) {
-		*errnop = -1;
+		*errnop = errno;
 		return (NS_NOTFOUND);
 	}
 	if (__copy_protoent(&pe, pptr, buffer, buflen) != 0) {
-		*errnop = -1;
-		return (NS_NOTFOUND);
+		*errnop = errno;
+		return (NS_RETURN);
 	}
 
 	*((struct protoent **)retval) = pptr;
@@ -128,10 +129,11 @@ getprotobyname_r(const char *name, struct protoent *pptr, char *buffer,
 	rv = nsdispatch(result, dtab, NSDB_PROTOCOLS, "getprotobyname_r",
 	    defaultsrc, name, pptr, buffer, buflen, &ret_errno);
 
-	if (rv == NS_SUCCESS)
-		return (0);
-	else
-		return (ret_errno);
+	if (rv != NS_SUCCESS) {
+		errno = ret_errno;
+		return ((ret_errno != 0) ? ret_errno : -1);
+	}
+	return (0);
 }
 
 struct protoent *