Fix problem on big endian systems introduced in r271579 - when we were

returning from handling a nested nvlist we were resetting big-endian flag.

Reported by:	Kuleshov Aleksey @ yandex.ru
Tested by:	Kuleshov Aleksey @ yandex.ru

4 files changed, 40 insertions, 40 deletions

diff --git a/lib/libnv/nvlist.c b/lib/libnv/nvlist.c
index c7c3cb9..b495441 100644
--- a/lib/libnv/nvlist.c
+++ b/lib/libnv/nvlist.c
@@ -698,7 +698,7 @@ nvlist_check_header(struct nvlist_header *nvlhdrp)
 
 const unsigned char *
 nvlist_unpack_header(nvlist_t *nvl, const unsigned char *ptr, size_t nfds,
-    int *flagsp, size_t *leftp)
+    bool *isbep, size_t *leftp)
 {
 	struct nvlist_header nvlhdr;
 
@@ -725,7 +725,8 @@ nvlist_unpack_header(nvlist_t *nvl, const unsigned char *ptr, size_t nfds,
 	nvl->nvl_flags = (nvlhdr.nvlh_flags & NV_FLAG_PUBLIC_MASK);
 
 	ptr += sizeof(nvlhdr);
-	*flagsp = (int)nvlhdr.nvlh_flags;
+	if (isbep != NULL)
+		*isbep = (((int)nvlhdr.nvlh_flags & NV_FLAG_BIG_ENDIAN) != 0);
 	*leftp -= sizeof(nvlhdr);
 
 	return (ptr);
@@ -741,7 +742,7 @@ nvlist_xunpack(const void *buf, size_t size, const int *fds, size_t nfds)
 	nvlist_t *nvl, *retnvl, *tmpnvl;
 	nvpair_t *nvp;
 	size_t left;
-	int flags;
+	bool isbe;
 
 	left = size;
 	ptr = buf;
@@ -751,44 +752,43 @@ nvlist_xunpack(const void *buf, size_t size, const int *fds, size_t nfds)
 	if (nvl == NULL)
 		goto failed;
 
-	ptr = nvlist_unpack_header(nvl, ptr, nfds, &flags, &left);
+	ptr = nvlist_unpack_header(nvl, ptr, nfds, &isbe, &left);
 	if (ptr == NULL)
 		goto failed;
 
 	while (left > 0) {
-		ptr = nvpair_unpack(flags, ptr, &left, &nvp);
+		ptr = nvpair_unpack(isbe, ptr, &left, &nvp);
 		if (ptr == NULL)
 			goto failed;
 		switch (nvpair_type(nvp)) {
 		case NV_TYPE_NULL:
-			ptr = nvpair_unpack_null(flags, nvp, ptr, &left);
+			ptr = nvpair_unpack_null(isbe, nvp, ptr, &left);
 			break;
 		case NV_TYPE_BOOL:
-			ptr = nvpair_unpack_bool(flags, nvp, ptr, &left);
+			ptr = nvpair_unpack_bool(isbe, nvp, ptr, &left);
 			break;
 		case NV_TYPE_NUMBER:
-			ptr = nvpair_unpack_number(flags, nvp, ptr, &left);
+			ptr = nvpair_unpack_number(isbe, nvp, ptr, &left);
 			break;
 		case NV_TYPE_STRING:
-			ptr = nvpair_unpack_string(flags, nvp, ptr, &left);
+			ptr = nvpair_unpack_string(isbe, nvp, ptr, &left);
 			break;
 		case NV_TYPE_NVLIST:
-			ptr = nvpair_unpack_nvlist(&flags, nvp, ptr, &left,
-			    nfds, &tmpnvl);
+			ptr = nvpair_unpack_nvlist(isbe, nvp, ptr, &left, nfds,
+			    &tmpnvl);
 			nvlist_set_parent(tmpnvl, nvp);
 			break;
 		case NV_TYPE_DESCRIPTOR:
-			ptr = nvpair_unpack_descriptor(flags, nvp, ptr, &left,
+			ptr = nvpair_unpack_descriptor(isbe, nvp, ptr, &left,
 			    fds, nfds);
 			break;
 		case NV_TYPE_BINARY:
-			ptr = nvpair_unpack_binary(flags, nvp, ptr, &left);
+			ptr = nvpair_unpack_binary(isbe, nvp, ptr, &left);
 			break;
 		case NV_TYPE_NVLIST_UP:
 			if (nvl->nvl_parent == NULL)
 				goto failed;
 			nvl = nvpair_nvlist(nvl->nvl_parent);
-			flags = nvl->nvl_flags;
 			continue;
 		default:
 			PJDLOG_ABORT("Invalid type (%d).", nvpair_type(nvp));
diff --git a/lib/libnv/nvlist_impl.h b/lib/libnv/nvlist_impl.h
index 28894c0..ef32dea 100644
--- a/lib/libnv/nvlist_impl.h
+++ b/lib/libnv/nvlist_impl.h
@@ -42,6 +42,6 @@ nvlist_t *nvlist_xunpack(const void *buf, size_t size, const int *fds,
 
 nvpair_t *nvlist_get_nvpair_parent(const nvlist_t *nvl);
 const unsigned char *nvlist_unpack_header(nvlist_t *nvl,
-    const unsigned char *ptr, size_t nfds, int *flagsp, size_t *leftp);
+    const unsigned char *ptr, size_t nfds, bool *isbep, size_t *leftp);
 
 #endif	/* !_NVLIST_IMPL_H_ */
diff --git a/lib/libnv/nvpair.c b/lib/libnv/nvpair.c
index 8ff8538..4f0bd72 100644
--- a/lib/libnv/nvpair.c
+++ b/lib/libnv/nvpair.c
@@ -389,7 +389,7 @@ nvpair_init_datasize(nvpair_t *nvp)
 }
 
 const unsigned char *
-nvpair_unpack_header(int flags, nvpair_t *nvp, const unsigned char *ptr,
+nvpair_unpack_header(bool isbe, nvpair_t *nvp, const unsigned char *ptr,
     size_t *leftp)
 {
 	struct nvpair_header nvphdr;
@@ -411,12 +411,12 @@ nvpair_unpack_header(int flags, nvpair_t *nvp, const unsigned char *ptr,
 	}
 
 #if BYTE_ORDER == BIG_ENDIAN
-	if ((flags & NV_FLAG_BIG_ENDIAN) == 0) {
+	if (!isbe) {
 		nvphdr.nvph_namesize = le16toh(nvphdr.nvph_namesize);
 		nvphdr.nvph_datasize = le64toh(nvphdr.nvph_datasize);
 	}
 #else
-	if ((flags & NV_FLAG_BIG_ENDIAN) != 0) {
+	if (isbe) {
 		nvphdr.nvph_namesize = be16toh(nvphdr.nvph_namesize);
 		nvphdr.nvph_datasize = be64toh(nvphdr.nvph_datasize);
 	}
@@ -451,7 +451,7 @@ failed:
 }
 
 const unsigned char *
-nvpair_unpack_null(int flags __unused, nvpair_t *nvp, const unsigned char *ptr,
+nvpair_unpack_null(bool isbe __unused, nvpair_t *nvp, const unsigned char *ptr,
     size_t *leftp __unused)
 {
 
@@ -466,7 +466,7 @@ nvpair_unpack_null(int flags __unused, nvpair_t *nvp, const unsigned char *ptr,
 }
 
 const unsigned char *
-nvpair_unpack_bool(int flags __unused, nvpair_t *nvp, const unsigned char *ptr,
+nvpair_unpack_bool(bool isbe __unused, nvpair_t *nvp, const unsigned char *ptr,
     size_t *leftp)
 {
 	uint8_t value;
@@ -497,7 +497,7 @@ nvpair_unpack_bool(int flags __unused, nvpair_t *nvp, const unsigned char *ptr,
 }
 
 const unsigned char *
-nvpair_unpack_number(int flags, nvpair_t *nvp, const unsigned char *ptr,
+nvpair_unpack_number(bool isbe, nvpair_t *nvp, const unsigned char *ptr,
      size_t *leftp)
 {
 
@@ -512,7 +512,7 @@ nvpair_unpack_number(int flags, nvpair_t *nvp, const unsigned char *ptr,
 		return (NULL);
 	}
 
-	if ((flags & NV_FLAG_BIG_ENDIAN) != 0)
+	if (isbe)
 		nvp->nvp_data = be64dec(ptr);
 	else
 		nvp->nvp_data = le64dec(ptr);
@@ -523,7 +523,7 @@ nvpair_unpack_number(int flags, nvpair_t *nvp, const unsigned char *ptr,
 }
 
 const unsigned char *
-nvpair_unpack_string(int flags __unused, nvpair_t *nvp,
+nvpair_unpack_string(bool isbe __unused, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp)
 {
 
@@ -551,8 +551,8 @@ nvpair_unpack_string(int flags __unused, nvpair_t *nvp,
 }
 
 const unsigned char *
-nvpair_unpack_nvlist(int *flagsp, nvpair_t *nvp, const unsigned char *ptr,
-    size_t *leftp, size_t nfds, nvlist_t **child)
+nvpair_unpack_nvlist(bool isbe __unused, nvpair_t *nvp,
+    const unsigned char *ptr, size_t *leftp, size_t nfds, nvlist_t **child)
 {
 	nvlist_t *value;
 
@@ -567,7 +567,7 @@ nvpair_unpack_nvlist(int *flagsp, nvpair_t *nvp, const unsigned char *ptr,
 	if (value == NULL)
 		return (NULL);
 
-	ptr = nvlist_unpack_header(value, ptr, nfds, flagsp, leftp);
+	ptr = nvlist_unpack_header(value, ptr, nfds, NULL, leftp);
 	if (ptr == NULL)
 		return (NULL);
 
@@ -578,7 +578,7 @@ nvpair_unpack_nvlist(int *flagsp, nvpair_t *nvp, const unsigned char *ptr,
 }
 
 const unsigned char *
-nvpair_unpack_descriptor(int flags, nvpair_t *nvp, const unsigned char *ptr,
+nvpair_unpack_descriptor(bool isbe, nvpair_t *nvp, const unsigned char *ptr,
     size_t *leftp, const int *fds, size_t nfds)
 {
 	int64_t idx;
@@ -594,7 +594,7 @@ nvpair_unpack_descriptor(int flags, nvpair_t *nvp, const unsigned char *ptr,
 		return (NULL);
 	}
 
-	if ((flags & NV_FLAG_BIG_ENDIAN) != 0)
+	if (isbe)
 		idx = be64dec(ptr);
 	else
 		idx = le64dec(ptr);
@@ -618,7 +618,7 @@ nvpair_unpack_descriptor(int flags, nvpair_t *nvp, const unsigned char *ptr,
 }
 
 const unsigned char *
-nvpair_unpack_binary(int flags __unused, nvpair_t *nvp,
+nvpair_unpack_binary(bool isbe __unused, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp)
 {
 	void *value;
@@ -644,7 +644,7 @@ nvpair_unpack_binary(int flags __unused, nvpair_t *nvp,
 }
 
 const unsigned char *
-nvpair_unpack(int flags, const unsigned char *ptr, size_t *leftp,
+nvpair_unpack(bool isbe, const unsigned char *ptr, size_t *leftp,
     nvpair_t **nvpp)
 {
 	nvpair_t *nvp, *tmp;
@@ -654,7 +654,7 @@ nvpair_unpack(int flags, const unsigned char *ptr, size_t *leftp,
 		return (NULL);
 	nvp->nvp_name = (char *)(nvp + 1);
 
-	ptr = nvpair_unpack_header(flags, nvp, ptr, leftp);
+	ptr = nvpair_unpack_header(isbe, nvp, ptr, leftp);
 	if (ptr == NULL)
 		goto failed;
 	tmp = realloc(nvp, sizeof(*nvp) + strlen(nvp->nvp_name) + 1);
diff --git a/lib/libnv/nvpair_impl.h b/lib/libnv/nvpair_impl.h
index 70dbbc0..a72000b 100644
--- a/lib/libnv/nvpair_impl.h
+++ b/lib/libnv/nvpair_impl.h
@@ -48,7 +48,7 @@ void nvpair_insert(struct nvl_head *head, nvpair_t *nvp, nvlist_t *nvl);
 void nvpair_remove(struct nvl_head *head, nvpair_t *nvp, const nvlist_t *nvl);
 size_t nvpair_header_size(void);
 size_t nvpair_size(const nvpair_t *nvp);
-const unsigned char *nvpair_unpack(int flags, const unsigned char *ptr,
+const unsigned char *nvpair_unpack(bool isbe, const unsigned char *ptr,
     size_t *leftp, nvpair_t **nvpp);
 void nvpair_free_structure(nvpair_t *nvp);
 void nvpair_init_datasize(nvpair_t *nvp);
@@ -72,21 +72,21 @@ unsigned char *nvpair_pack_binary(const nvpair_t *nvp, unsigned char *ptr,
 unsigned char *nvpair_pack_nvlist_up(unsigned char *ptr, size_t *leftp);
 
 /* Unpack data functions. */
-const unsigned char *nvpair_unpack_header(int flags, nvpair_t *nvp,
+const unsigned char *nvpair_unpack_header(bool isbe, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp);
-const unsigned char *nvpair_unpack_null(int flags, nvpair_t *nvp,
+const unsigned char *nvpair_unpack_null(bool isbe, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp);
-const unsigned char *nvpair_unpack_bool(int flags, nvpair_t *nvp,
+const unsigned char *nvpair_unpack_bool(bool isbe, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp);
-const unsigned char *nvpair_unpack_number(int flags, nvpair_t *nvp,
+const unsigned char *nvpair_unpack_number(bool isbe, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp);
-const unsigned char *nvpair_unpack_string(int flags, nvpair_t *nvp,
+const unsigned char *nvpair_unpack_string(bool isbe, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp);
-const unsigned char *nvpair_unpack_nvlist(int *flagsp, nvpair_t *nvp,
+const unsigned char *nvpair_unpack_nvlist(bool isbe, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp, size_t nvlist, nvlist_t **child);
-const unsigned char *nvpair_unpack_descriptor(int flags, nvpair_t *nvp,
+const unsigned char *nvpair_unpack_descriptor(bool isbe, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp, const int *fds, size_t nfds);
-const unsigned char *nvpair_unpack_binary(int flags, nvpair_t *nvp,
+const unsigned char *nvpair_unpack_binary(bool isbe, nvpair_t *nvp,
     const unsigned char *ptr, size_t *leftp);
 
 #endif	/* !_NVPAIR_IMPL_H_ */