MFp4: libarchive 2.5.4b. (Still 'b' until I get a bit more

feedback, but the 2.5 branch is shaping up nicely.)

In addition to many small bug fixes and code improvements:
 * Another iteration of versioning; I think I've got it right now.
 * Portability:  A lot of progress on Windows support (though I'm
   not committing all of the Windows support files to FreeBSD CVS)
 * Explicit tracking of MBS, WCS, and UTF-8 versions of strings
   in archive_entry; the archive_entry routines now correctly return
   NULL only when something is unset, setting NULL properly clears
   string values.  Most charset conversions have been pushed down to
   archive_string.
 * Better handling of charset conversion failure when writing or
   reading UTF-8 headers in pax archives
 * archive_entry_linkify() provides multiple strategies for
   hardlink matching to suit different format expectations
 * More accurate bzip2 format detection
 * Joerg Sonnenberger's extensive improvements to mtree support
 * Rough support for self-extracting ZIP archives.  Not an ideal
   approach, but it works for the archives I've tried.
 * New "sparsify" option in archive_write_disk converts blocks of nulls
   into seeks.
 * Better default behavior for the test harness; it now reports
   all failures by default instead of coredumping at the first one.

1 files changed, 256 insertions, 7 deletions

diff --git a/lib/libarchive/archive_string.c b/lib/libarchive/archive_string.c
index 8e1d11c..ba7605a 100644
--- a/lib/libarchive/archive_string.c
+++ b/lib/libarchive/archive_string.c
@@ -37,6 +37,17 @@ __FBSDID("$FreeBSD$");
 #ifdef HAVE_STRING_H
 #include <string.h>
 #endif
+#ifdef HAVE_WCHAR_H
+#include <wchar.h>
+#endif
+
+#ifdef __sgi
+/*
+ * The following prototype is missing on IRXI,
+ * even though the function is implemented in libc.
+ */
+size_t wcrtomb(char *, wchar_t, mbstate_t *);
+#endif
 
 #include "archive_private.h"
 #include "archive_string.h"
@@ -55,11 +66,15 @@ __archive_string_append(struct archive_string *as, const char *p, size_t s)
 void
 __archive_string_copy(struct archive_string *dest, struct archive_string *src)
 {
-	if (__archive_string_ensure(dest, src->length + 1) == NULL)
-		__archive_errx(1, "Out of memory");
-	memcpy(dest->s, src->s, src->length);
-	dest->length = src->length;
-	dest->s[dest->length] = 0;
+	if (src->length == 0)
+		dest->length = 0;
+	else {
+		if (__archive_string_ensure(dest, src->length + 1) == NULL)
+			__archive_errx(1, "Out of memory");
+		memcpy(dest->s, src->s, src->length);
+		dest->length = src->length;
+		dest->s[dest->length] = 0;
+	}
 }
 
 void
@@ -67,21 +82,52 @@ __archive_string_free(struct archive_string *as)
 {
 	as->length = 0;
 	as->buffer_length = 0;
-	if (as->s != NULL)
+	if (as->s != NULL) {
 		free(as->s);
+		as->s = NULL;
+	}
 }
 
 /* Returns NULL on any allocation failure. */
 struct archive_string *
 __archive_string_ensure(struct archive_string *as, size_t s)
 {
+	/* If buffer is already big enough, don't reallocate. */
 	if (as->s && (s <= as->buffer_length))
 		return (as);
 
+	/*
+	 * Growing the buffer at least exponentially ensures that
+	 * append operations are always linear in the number of
+	 * characters appended.  Using a smaller growth rate for
+	 * larger buffers reduces memory waste somewhat at the cost of
+	 * a larger constant factor.
+	 */
 	if (as->buffer_length < 32)
+		/* Start with a minimum 32-character buffer. */
 		as->buffer_length = 32;
-	while (as->buffer_length < s)
+	else if (as->buffer_length < 8192)
+		/* Buffers under 8k are doubled for speed. */
 		as->buffer_length *= 2;
+	else {
+		/* Buffers 8k and over grow by at least 25% each time. */
+		size_t old_length = as->buffer_length;
+		as->buffer_length = (as->buffer_length * 5) / 4;
+		/* Be safe: If size wraps, release buffer and return NULL. */
+		if (as->buffer_length < old_length) {
+			free(as->s);
+			as->s = NULL;
+			return (NULL);
+		}
+	}
+	/*
+	 * The computation above is a lower limit to how much we'll
+	 * grow the buffer.  In any case, we have to grow it enough to
+	 * hold the request.
+	 */
+	if (as->buffer_length < s)
+		as->buffer_length = s;
+	/* Now we can reallocate the buffer. */
 	as->s = (char *)realloc(as->s, as->buffer_length);
 	if (as->s == NULL)
 		return (NULL);
@@ -124,3 +170,206 @@ __archive_strappend_int(struct archive_string *as, int d, int base)
 	__archive_strappend_char(as, digits[d % base]);
 	return (as);
 }
+
+/*
+ * Home-grown wcrtomb for UTF-8.
+ */
+static size_t
+my_wcrtomb_utf8(char *p, wchar_t wc, mbstate_t *s)
+{
+	(void)s; /* UNUSED */
+
+	if (p == NULL)
+		return (0);
+	if (wc <= 0x7f) {
+		p[0] = (char)wc;
+		return (1);
+	}
+	if (wc <= 0x7ff) {
+		p[0] = 0xc0 | ((wc >> 6) & 0x1f);
+		p[1] = 0x80 | (wc & 0x3f);
+		return (2);
+	}
+	if (wc <= 0xffff) {
+		p[0] = 0xe0 | ((wc >> 12) & 0x0f);
+		p[1] = 0x80 | ((wc >> 6) & 0x3f);
+		p[2] = 0x80 | (wc & 0x3f);
+		return (3);
+	}
+	if (wc <= 0x1fffff) {
+		p[0] = 0xf0 | ((wc >> 18) & 0x07);
+		p[1] = 0x80 | ((wc >> 12) & 0x3f);
+		p[2] = 0x80 | ((wc >> 6) & 0x3f);
+		p[3] = 0x80 | (wc & 0x3f);
+		return (4);
+	}
+	/* Unicode has no codes larger than 0x1fffff. */
+	/*
+	 * Awkward point:  UTF-8 <-> wchar_t conversions
+	 * can actually fail.
+	 */
+	return ((size_t)-1);
+}
+
+static int
+my_wcstombs(struct archive_string *as, const wchar_t *w,
+    size_t (*func)(char *, wchar_t, mbstate_t *))
+{
+	size_t n;
+	char *p;
+	mbstate_t shift_state;
+	char buff[256];
+
+	/*
+	 * Convert one wide char at a time into 'buff', whenever that
+	 * fills, append it to the string.
+	 */
+	p = buff;
+	wcrtomb(NULL, L'\0', &shift_state);
+	while (*w != L'\0') {
+		/* Flush the buffer when we have <=16 bytes free. */
+		/* (No encoding has a single character >16 bytes.) */
+		if ((size_t)(p - buff) >= (size_t)(sizeof(buff) - 16)) {
+			*p = '\0';
+			archive_strcat(as, buff);
+			p = buff;
+		}
+		n = (*func)(p, *w++, &shift_state);
+		if (n == (size_t)-1)
+			return (-1);
+		p += n;
+	}
+	*p = '\0';
+	archive_strcat(as, buff);
+	return (0);
+}
+
+/*
+ * Translates a wide character string into UTF-8 and appends
+ * to the archive_string.  Note: returns NULL if conversion fails.
+ */
+struct archive_string *
+__archive_strappend_w_utf8(struct archive_string *as, const wchar_t *w)
+{
+	if (my_wcstombs(as, w, my_wcrtomb_utf8))
+		return (NULL);
+	return (as);
+}
+
+/*
+ * Translates a wide character string into current locale character set
+ * and appends to the archive_string.  Note: returns NULL if conversion
+ * fails.
+ *
+ * TODO: use my_wcrtomb_utf8 if !HAVE_WCRTOMB (add configure logic first!)
+ */
+struct archive_string *
+__archive_strappend_w_mbs(struct archive_string *as, const wchar_t *w)
+{
+	if (my_wcstombs(as, w, wcrtomb))
+		return (NULL);
+	return (as);
+}
+
+
+/*
+ * Home-grown mbrtowc for UTF-8.  Some systems lack UTF-8
+ * (or even lack mbrtowc()) and we need UTF-8 support for pax
+ * format.  So please don't replace this with a call to the
+ * standard mbrtowc() function!
+ */
+static size_t
+my_mbrtowc_utf8(wchar_t *pwc, const char *s, size_t n, mbstate_t *ps)
+{
+        int ch;
+
+	/*
+	 * This argument is here to make the prototype identical to the
+	 * standard mbrtowc(), so I can build generic string processors
+	 * that just accept a pointer to a suitable mbrtowc() function.
+	 */
+	(void)ps; /* UNUSED */
+
+	/* Standard behavior:  a NULL value for 's' just resets shift state. */
+        if (s == NULL)
+                return (0);
+	/* If length argument is zero, don't look at the first character. */
+	if (n <= 0)
+		return ((size_t)-2);
+
+        /*
+	 * Decode 1-4 bytes depending on the value of the first byte.
+	 */
+        ch = (unsigned char)*s;
+	if (ch == 0) {
+		return (0); /* Standard:  return 0 for end-of-string. */
+	}
+	if ((ch & 0x80) == 0) {
+                *pwc = ch & 0x7f;
+		return (1);
+        }
+	if ((ch & 0xe0) == 0xc0) {
+		if (n < 2)
+			return ((size_t)-2);
+		if ((s[1] & 0xc0) != 0x80) return (size_t)-1;
+                *pwc = ((ch & 0x1f) << 6) | (s[1] & 0x3f);
+		return (2);
+        }
+	if ((ch & 0xf0) == 0xe0) {
+		if (n < 3)
+			return ((size_t)-2);
+		if ((s[1] & 0xc0) != 0x80) return (size_t)-1;
+		if ((s[2] & 0xc0) != 0x80) return (size_t)-1;
+                *pwc = ((ch & 0x0f) << 12)
+		    | ((s[1] & 0x3f) << 6)
+		    | (s[2] & 0x3f);
+		return (3);
+        }
+	if ((ch & 0xf8) == 0xf0) {
+		if (n < 4)
+			return ((size_t)-2);
+		if ((s[1] & 0xc0) != 0x80) return (size_t)-1;
+		if ((s[2] & 0xc0) != 0x80) return (size_t)-1;
+		if ((s[3] & 0xc0) != 0x80) return (size_t)-1;
+                *pwc = ((ch & 0x07) << 18)
+		    | ((s[1] & 0x3f) << 12)
+		    | ((s[2] & 0x3f) << 6)
+		    | (s[3] & 0x3f);
+		return (4);
+        }
+	/* Invalid first byte. */
+	return ((size_t)-1);
+}
+
+/*
+ * Return a wide-character string by converting this archive_string
+ * from UTF-8.
+ */
+wchar_t *
+__archive_string_utf8_w(struct archive_string *as)
+{
+	wchar_t *ws, *dest;
+	const char *src;
+	size_t n;
+	int err;
+
+	ws = (wchar_t *)malloc((as->length + 1) * sizeof(wchar_t));
+	if (ws == NULL)
+		__archive_errx(1, "Out of memory");
+	err = 0;
+	dest = ws;
+	src = as->s;
+	while (*src != '\0') {
+		n = my_mbrtowc_utf8(dest, src, 8, NULL);
+		if (n == 0)
+			break;
+		if (n == (size_t)-1 || n == (size_t)-2) {
+			free(ws);
+			return (NULL);
+		}
+		dest++;
+		src += n;
+	}
+	*dest++ = L'\0';
+	return (ws);
+}