When writing "pax" format, readers are supposed to ignore fields

in the regular ustar header that are overridden by the pax
extended attributes.  As a result, it makes perfect sense to
use numeric extensions in the regular ustar header so that readers
that don't understand pax extensions but do understand some other
extensions can still get useful information out of it.

This is especially important for filesizes, as the failure to
read a file size correctly can get the reader out of sync.

This commit introduces a "non-strict" option into the internal
function to format a ustar header.  In non-strict mode, the formatter
will use longer octal values (overwriting terminators) or binary
("base-256") values as needed to ensure that large file sizes,
negative mtimes, etc, have the correct values stored in the regular
ustar header.

5 files changed, 117 insertions, 25 deletions

diff --git a/lib/libarchive/Makefile b/lib/libarchive/Makefile
index 406dded..7c72f95 100644
--- a/lib/libarchive/Makefile
+++ b/lib/libarchive/Makefile
@@ -2,7 +2,7 @@
 
 LIB=	archive
 SHLIB_MAJOR=	1
-VERSION=	1.00.000
+VERSION=	1.00.002
 CFLAGS+=	-DPACKAGE_NAME=\"lib${LIB}\"
 CFLAGS+=	-DPACKAGE_VERSION=\"${VERSION}\"
 
diff --git a/lib/libarchive/Makefile.freebsd b/lib/libarchive/Makefile.freebsd
index 406dded..7c72f95 100644
--- a/lib/libarchive/Makefile.freebsd
+++ b/lib/libarchive/Makefile.freebsd
@@ -2,7 +2,7 @@
 
 LIB=	archive
 SHLIB_MAJOR=	1
-VERSION=	1.00.000
+VERSION=	1.00.002
 CFLAGS+=	-DPACKAGE_NAME=\"lib${LIB}\"
 CFLAGS+=	-DPACKAGE_VERSION=\"${VERSION}\"
 
diff --git a/lib/libarchive/archive_private.h b/lib/libarchive/archive_private.h
index 730974c..5fb4d9b 100644
--- a/lib/libarchive/archive_private.h
+++ b/lib/libarchive/archive_private.h
@@ -198,10 +198,15 @@ struct archive {
 };
 
 
-/* Utility function to format a USTAR header into a buffer. */
+/*
+ * Utility function to format a USTAR header into a buffer.  If
+ * "strict" is set, this tries to create the absolutely most portable
+ * version of a ustar header.  If "strict" is set to 0, then it will
+ * relax certain requirements.
+ */
 int
 __archive_write_format_header_ustar(struct archive *, char buff[512],
-    struct archive_entry *, int tartype);
+    struct archive_entry *, int tartype, int strict);
 
 #define	ARCHIVE_STATE_ANY	0xFFFFU
 #define	ARCHIVE_STATE_NEW	1U
diff --git a/lib/libarchive/archive_write_set_format_pax.c b/lib/libarchive/archive_write_set_format_pax.c
index cc4b82e..a8e5be8 100644
--- a/lib/libarchive/archive_write_set_format_pax.c
+++ b/lib/libarchive/archive_write_set_format_pax.c
@@ -500,7 +500,6 @@ archive_write_pax_header(struct archive *a,
 	if (!need_extension && p != NULL  &&  *p != '\0')
 		need_extension = 1;
 
-
 	/* If there are non-trivial ACL entries, we need an extension. */
 	if (!need_extension && archive_entry_acl_count(entry_original,
 		ARCHIVE_ENTRY_ACL_TYPE_ACCESS) > 0)
@@ -595,9 +594,33 @@ archive_write_pax_header(struct archive *a,
 	if (hardlink != NULL)
 		archive_entry_set_size(entry_main, 0);
 
-	/* Format 'ustar' header for main entry. */
-	/* We don't care if this returns an error. */
-	__archive_write_format_header_ustar(a, ustarbuff, entry_main, -1);
+	/* Format 'ustar' header for main entry.
+	 *
+	 * The trouble with file size: If the reader can't understand
+	 * the file size, they may not be able to locate the next
+	 * entry and the rest of the archive is toast.  Pax-compliant
+	 * readers are supposed to ignore the file size in the main
+	 * header, so the question becomes how to maximize portability
+	 * for readers that don't support pax attribute extensions.
+	 * For maximum compatibility, I permit numeric extensions in
+	 * the main header so that the file size stored will always be
+	 * correct, even if it's in a format that only some
+	 * implementations understand.  The technique used here is:
+	 *
+	 *  a) If possible, follow the standard exactly.  This handles
+	 *  files up to 8 gigabytes minus 1.
+	 *
+	 *  b) If that fails, try octal but omit the field terminator.
+	 *  That handles files up to 64 gigabytes minus 1.
+	 *
+	 *  c) Otherwise, use base-256 extensions.  That handles files
+	 *  up to 2^63 in this implementation, with the potential to
+	 *  go up to 2^94.  That should hold us for a while. ;-)
+	 *
+	 * The non-strict formatter uses similar logic for other
+	 * numeric fields, though they're less critical.
+	 */
+	__archive_write_format_header_ustar(a, ustarbuff, entry_main, -1, 0);
 
 	/* If we built any extended attributes, write that entry first. */
 	ret = 0;
@@ -624,7 +647,7 @@ archive_write_pax_header(struct archive *a,
 		    archive_entry_gname(entry_main));
 
 		ret = __archive_write_format_header_ustar(a, paxbuff,
-		    pax_attr_entry, 'x');
+		    pax_attr_entry, 'x', 1);
 
 		archive_entry_free(pax_attr_entry);
 		archive_string_free(&pax_entry_name);
diff --git a/lib/libarchive/archive_write_set_format_ustar.c b/lib/libarchive/archive_write_set_format_ustar.c
index 68892eb..9e49596 100644
--- a/lib/libarchive/archive_write_set_format_ustar.c
+++ b/lib/libarchive/archive_write_set_format_ustar.c
@@ -102,6 +102,8 @@ static int	archive_write_ustar_finish(struct archive *);
 static int	archive_write_ustar_finish_entry(struct archive *);
 static int	archive_write_ustar_header(struct archive *,
 		    struct archive_entry *entry);
+static int	format_256(int64_t, char *, int);
+static int	format_number(int64_t, char *, int size, int max, int strict);
 static int	format_octal(int64_t, char *, int);
 static int	write_nulls(struct archive *a, size_t);
 
@@ -151,7 +153,7 @@ archive_write_ustar_header(struct archive *a, struct archive_entry *entry)
 	    !S_ISREG(archive_entry_mode(entry)))
 		archive_entry_set_size(entry, 0);
 
-	ret = __archive_write_format_header_ustar(a, buff, entry, -1);
+	ret = __archive_write_format_header_ustar(a, buff, entry, -1, 1);
 	if (ret != ARCHIVE_OK)
 		return (ret);
 	ret = (a->compression_write)(a, buff, 512);
@@ -168,12 +170,14 @@ archive_write_ustar_header(struct archive *a, struct archive_entry *entry)
  *
  * Returns -1 if format failed (due to field overflow).
  * Note that this always formats as much of the header as possible.
+ * If "strict" is set to zero, it will extend numeric fields as
+ * necessary (overwriting terminators or using base-256 extensions).
  *
  * This is exported so that other 'tar' formats can use it.
  */
 int
 __archive_write_format_header_ustar(struct archive *a, char buff[512],
-				    struct archive_entry *entry, int tartype)
+    struct archive_entry *entry, int tartype, int strict)
 {
 	unsigned int checksum;
 	struct archive_entry_header_ustar *h;
@@ -185,6 +189,11 @@ __archive_write_format_header_ustar(struct archive *a, char buff[512],
 
 	ret = 0;
 	mytartype = -1;
+	/*
+	 * The "template header" already includes the "ustar"
+	 * signature, various end-of-field markers and other required
+	 * elements.
+	 */
 	memcpy(buff, &template_header, 512);
 
 	h = (struct archive_entry_header_ustar *)buff;
@@ -262,42 +271,42 @@ __archive_write_format_header_ustar(struct archive *a, char buff[512],
 
 	st = archive_entry_stat(entry);
 
-	if (format_octal(st->st_mode & 07777, h->mode, sizeof(h->mode))) {
+	if (format_number(st->st_mode & 07777, h->mode, sizeof(h->mode), 8, strict)) {
 		archive_set_error(a, ERANGE, "Numeric mode too large");
 		ret = ARCHIVE_WARN;
 	}
 
-	if (format_octal(st->st_uid, h->uid, sizeof(h->uid))) {
+	if (format_number(st->st_uid, h->uid, sizeof(h->uid), 8, strict)) {
 		archive_set_error(a, ERANGE, "Numeric user ID too large");
 		ret = ARCHIVE_WARN;
 	}
 
-	if (format_octal(st->st_gid, h->gid, sizeof(h->gid))) {
+	if (format_number(st->st_gid, h->gid, sizeof(h->gid), 8, strict)) {
 		archive_set_error(a, ERANGE, "Numeric group ID too large");
 		ret = ARCHIVE_WARN;
 	}
 
-	if (format_octal(st->st_size, h->size, sizeof(h->size))) {
-		archive_set_error(a, ERANGE, "File size too large");
+	if (format_number(st->st_size, h->size, sizeof(h->size), 12, strict)) {
+		archive_set_error(a, ERANGE, "File size out of range");
 		ret = ARCHIVE_WARN;
 	}
 
-	if (format_octal(st->st_mtime, h->mtime, sizeof(h->mtime))) {
+	if (format_number(st->st_mtime, h->mtime, sizeof(h->mtime), 12, strict)) {
 		archive_set_error(a, ERANGE,
 		    "File modification time too large");
 		ret = ARCHIVE_WARN;
 	}
 
 	if (S_ISBLK(st->st_mode) || S_ISCHR(st->st_mode)) {
-		if (format_octal(major(st->st_rdev), h->rdevmajor,
-			sizeof(h->rdevmajor))) {
+		if (format_number(major(st->st_rdev), h->rdevmajor,
+			sizeof(h->rdevmajor), 8, strict)) {
 			archive_set_error(a, ERANGE,
 			    "Major device number too large");
 			ret = ARCHIVE_WARN;
 		}
 
-		if (format_octal(minor(st->st_rdev), h->rdevminor,
-			sizeof(h->rdevminor))) {
+		if (format_number(minor(st->st_rdev), h->rdevminor,
+			sizeof(h->rdevminor), 8, strict)) {
 			archive_set_error(a, ERANGE,
 			    "Minor device number too large");
 			ret = ARCHIVE_WARN;
@@ -331,13 +340,61 @@ __archive_write_format_header_ustar(struct archive *a, char buff[512],
 	checksum = 0;
 	for (i = 0; i < 512; i++)
 		checksum += 255 & (unsigned int)buff[i];
-	h->checksum[6] = '\0';
-	h->checksum[7] = ' ';
+	h->checksum[6] = '\0'; /* Can't be pre-set in the template. */
+	/* h->checksum[7] = ' '; */ /* This is pre-set in the template. */
 	format_octal(checksum, h->checksum, 6);
 	return (ret);
 }
 
 /*
+ * Format a number into a field, with some intelligence.
+ */
+static int
+format_number(int64_t v, char *p, int s, int maxsize, int strict)
+{
+	int64_t limit;
+
+	limit = ((int64_t)1 << (s*3));
+
+	/* "Strict" only permits octal values with proper termination. */
+	if (strict)
+		return (format_octal(v, p, s));
+
+	/*
+	 * In non-strict mode, we allow the number to overwrite one or
+	 * more bytes of the field termination.  Even old tar
+	 * implementations should be able to handle this with no
+	 * problem.
+	 */
+	if (v >= 0) {
+		while (s <= maxsize) {
+			if (v < limit)
+				return (format_octal(v, p, s));
+			s++;
+			limit <<= 3;
+		}
+	}
+
+	/* Base-256 can handle any number, positive or negative. */
+	return (format_256(v, p, maxsize));
+}
+
+/*
+ * Format a number into the specified field using base-256.
+ */
+static int
+format_256(int64_t v, char *p, int s)
+{
+	p += s;
+	while (s-- > 0) {
+		*--p = (char)(v & 0xff);
+		v >>= 8;
+	}
+	*p |= 0x80; /* Set the base-256 marker bit. */
+	return (0);
+}
+
+/*
  * Format a number into the specified field.
  */
 static int
@@ -345,9 +402,16 @@ format_octal(int64_t v, char *p, int s)
 {
 	int len;
 
-	p += s;		/* Start at the end and work backwards. */
-
 	len = s;
+
+	/* Octal values can't be negative, so use 0. */
+	if (v < 0) {
+		while (len-- > 0)
+			*p++ = '0';
+		return (-1);
+	}
+
+	p += s;		/* Start at the end and work backwards. */
 	while (s-- > 0) {
 		*--p = '0' + (v & 7);
 		v >>= 3;