Fix ARCHIVE_EXTRACT_SPARSE handling in libarchive.

Add a test to exercise this feature.
This should fix --sparse/-S support in tar.

Thanks to:      Daichi GOTO
MFC after:	1 week

3 files changed, 305 insertions, 22 deletions

diff --git a/lib/libarchive/archive_write_disk.c b/lib/libarchive/archive_write_disk.c
index 807c78f..ddac630 100644
--- a/lib/libarchive/archive_write_disk.c
+++ b/lib/libarchive/archive_write_disk.c
@@ -178,6 +178,8 @@ struct archive_write_disk {
 	int			 fd;
 	/* Current offset for writing data to the file. */
 	off_t			 offset;
+	/* Last offset actually written to disk. */
+	off_t			 fd_offset;
 	/* Maximum size of file, -1 if unknown. */
 	off_t			 filesize;
 	/* Dir we were in before this restore; only for deep paths. */
@@ -187,8 +189,6 @@ struct archive_write_disk {
 	/* UID/GID to use in restoring this entry. */
 	uid_t			 uid;
 	gid_t			 gid;
-	/* Last offset written to disk. */
-	off_t			 last_offset;
 };
 
 /*
@@ -235,7 +235,7 @@ static struct fixup_entry *sort_dir_list(struct fixup_entry *p);
 static gid_t	trivial_lookup_gid(void *, const char *, gid_t);
 static uid_t	trivial_lookup_uid(void *, const char *, uid_t);
 static ssize_t	write_data_block(struct archive_write_disk *,
-		    const char *, size_t, off_t);
+		    const char *, size_t);
 
 static struct archive_vtable *archive_write_disk_vtable(void);
 
@@ -337,7 +337,7 @@ _archive_write_header(struct archive *_a, struct archive_entry *entry)
 	}
 	a->entry = archive_entry_clone(entry);
 	a->fd = -1;
-	a->last_offset = 0;
+	a->fd_offset = 0;
 	a->offset = 0;
 	a->uid = a->user_uid;
 	a->mode = archive_entry_mode(a->entry);
@@ -513,9 +513,9 @@ archive_write_disk_set_skip_file(struct archive *_a, dev_t d, ino_t i)
 }
 
 static ssize_t
-write_data_block(struct archive_write_disk *a,
-    const char *buff, size_t size, off_t offset)
+write_data_block(struct archive_write_disk *a, const char *buff, size_t size)
 {
+	uint64_t start_size = size;
 	ssize_t bytes_written = 0;
 	ssize_t block_size = 0, bytes_to_write;
 
@@ -538,8 +538,9 @@ write_data_block(struct archive_write_disk *a,
 #endif
 	}
 
-	if (a->filesize >= 0 && (off_t)(offset + size) > a->filesize)
-		size = (size_t)(a->filesize - offset);
+	/* If this write would run beyond the file size, truncate it. */
+	if (a->filesize >= 0 && (off_t)(a->offset + size) > a->filesize)
+		start_size = size = (size_t)(a->filesize - a->offset);
 
 	/* Write the data. */
 	while (size > 0) {
@@ -555,7 +556,7 @@ write_data_block(struct archive_write_disk *a,
 				if (*p != '\0')
 					break;
 			}
-			offset += p - buff;
+			a->offset += p - buff;
 			size -= p - buff;
 			buff = p;
 			if (size == 0)
@@ -563,22 +564,25 @@ write_data_block(struct archive_write_disk *a,
 
 			/* Calculate next block boundary after offset. */
 			block_end
-			    = (offset / block_size) * block_size + block_size;
+			    = (a->offset / block_size + 1) * block_size;
 
 			/* If the adjusted write would cross block boundary,
 			 * truncate it to the block boundary. */
 			bytes_to_write = size;
-			if (offset + bytes_to_write > block_end)
-				bytes_to_write = block_end - offset;
+			if (a->offset + bytes_to_write > block_end)
+				bytes_to_write = block_end - a->offset;
 		}
 
 		/* Seek if necessary to the specified offset. */
-		if (offset != a->last_offset) {
-			if (lseek(a->fd, offset, SEEK_SET) < 0) {
+		if (a->offset != a->fd_offset) {
+			if (lseek(a->fd, a->offset, SEEK_SET) < 0) {
 				archive_set_error(&a->archive, errno,
 				    "Seek failed");
 				return (ARCHIVE_FATAL);
 			}
+			a->fd_offset = a->offset;
+			a->archive.file_position = a->offset;
+			a->archive.raw_position = a->offset;
  		}
 		bytes_written = write(a->fd, buff, bytes_to_write);
 		if (bytes_written < 0) {
@@ -587,12 +591,12 @@ write_data_block(struct archive_write_disk *a,
 		}
 		buff += bytes_written;
 		size -= bytes_written;
-		offset += bytes_written;
+		a->offset += bytes_written;
 		a->archive.file_position += bytes_written;
 		a->archive.raw_position += bytes_written;
-		a->last_offset = a->offset = offset;
+		a->fd_offset = a->offset;
 	}
-	return (bytes_written);
+	return (start_size - size);
 }
 
 static ssize_t
@@ -605,9 +609,9 @@ _archive_write_data_block(struct archive *_a,
 	__archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC,
 	    ARCHIVE_STATE_DATA, "archive_write_disk_block");
 
-	r = write_data_block(a, buff, size, offset);
-
-	if (r < 0)
+	a->offset = offset;
+	r = write_data_block(a, buff, size);
+	if (r < ARCHIVE_OK)
 		return (r);
 	if ((size_t)r < size) {
 		archive_set_error(&a->archive, 0,
@@ -625,7 +629,7 @@ _archive_write_data(struct archive *_a, const void *buff, size_t size)
 	__archive_check_magic(&a->archive, ARCHIVE_WRITE_DISK_MAGIC,
 	    ARCHIVE_STATE_DATA, "archive_write_data");
 
-	return (write_data_block(a, buff, size, a->offset));
+	return (write_data_block(a, buff, size));
 }
 
 static int
@@ -646,7 +650,7 @@ _archive_write_finish_entry(struct archive *_a)
 		/* There's no file. */
 	} else if (a->filesize < 0) {
 		/* File size is unknown, so we can't set the size. */
-	} else if (a->last_offset == a->filesize) {
+	} else if (a->fd_offset == a->filesize) {
 		/* Last write ended at exactly the filesize; we're done. */
 		/* Hopefully, this is the common case. */
 	} else {
diff --git a/lib/libarchive/test/Makefile b/lib/libarchive/test/Makefile
index 68a3c0c..fc8e3e9 100644
--- a/lib/libarchive/test/Makefile
+++ b/lib/libarchive/test/Makefile
@@ -62,6 +62,7 @@ TESTS= \
 	test_write_disk_hardlink.c		\
 	test_write_disk_perms.c			\
 	test_write_disk_secure.c		\
+	test_write_disk_sparse.c		\
 	test_write_disk_times.c			\
 	test_write_format_ar.c			\
 	test_write_format_cpio.c		\
diff --git a/lib/libarchive/test/test_write_disk_sparse.c b/lib/libarchive/test/test_write_disk_sparse.c
new file mode 100644
index 0000000..6179610
--- /dev/null
+++ b/lib/libarchive/test/test_write_disk_sparse.c
@@ -0,0 +1,278 @@
+/*-
+ * Copyright (c) 2003-2007 Tim Kientzle
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "test.h"
+__FBSDID("$FreeBSD$");
+
+/*
+ * Write a file using archive_write_data call, read the file
+ * back and verify the contents.  The data written includes large
+ * blocks of nulls, so it should exercise the sparsification logic
+ * if ARCHIVE_EXTRACT_SPARSE is enabled.
+ */
+static void
+verify_write_data(struct archive *a, int sparse)
+{
+	static const char data[]="abcdefghijklmnopqrstuvwxyz";
+	struct stat st;
+	struct archive_entry *ae;
+	size_t buff_size = 64 * 1024;
+	char *buff, *p;
+	const char *msg = sparse ? "sparse" : "non-sparse";
+	int fd;
+
+	buff = malloc(buff_size);
+	assert(buff != NULL);
+
+	ae = archive_entry_new();
+	assert(ae != NULL);
+	archive_entry_set_size(ae, 8 * buff_size);
+	archive_entry_set_pathname(ae, "test_write_data");
+	archive_entry_set_mode(ae, AE_IFREG | 0755);
+	assertEqualIntA(a, 0, archive_write_header(a, ae));
+
+	/* Use archive_write_data() to write three relatively sparse blocks. */
+
+	/* First has non-null data at beginning. */
+	memset(buff, 0, buff_size);
+	memcpy(buff, data, sizeof(data));
+	failure("%s", msg);
+	assertEqualInt(buff_size, archive_write_data(a, buff, buff_size));
+
+	/* Second has non-null data in the middle. */
+	memset(buff, 0, buff_size);
+	memcpy(buff + buff_size / 2 - 3, data, sizeof(data));
+	failure("%s", msg);
+	assertEqualInt(buff_size, archive_write_data(a, buff, buff_size));
+
+	/* Third has non-null data at the end. */
+	memset(buff, 0, buff_size);
+	memcpy(buff + buff_size - sizeof(data), data, sizeof(data));
+	failure("%s", msg);
+	assertEqualInt(buff_size, archive_write_data(a, buff, buff_size));
+
+	failure("%s", msg);
+	assertEqualIntA(a, 0, archive_write_finish_entry(a));
+
+	/* Test the entry on disk. */
+	assert(0 == stat(archive_entry_pathname(ae), &st));
+        assertEqualInt(st.st_size, 8 * buff_size);
+	fd = open(archive_entry_pathname(ae), O_RDONLY);
+	if (!assert(fd >= 0))
+		return;
+
+	/* Check first block. */
+	assertEqualInt(buff_size, read(fd, buff, buff_size));
+	failure("%s", msg);
+	assertEqualMem(buff, data, sizeof(data));
+	for (p = buff + sizeof(data); p < buff + buff_size; ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (!assertEqualInt(0, *p))
+			break;
+	}
+
+	/* Check second block. */
+	assertEqualInt(buff_size, read(fd, buff, buff_size));
+	for (p = buff; p < buff + buff_size; ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (p == buff + buff_size / 2 - 3) {
+			assertEqualMem(p, data, sizeof(data));
+			p += sizeof(data);
+		} else if (!assertEqualInt(0, *p))
+			break;
+	}
+
+	/* Check third block. */
+	assertEqualInt(buff_size, read(fd, buff, buff_size));
+	for (p = buff; p < buff + buff_size - sizeof(data); ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (!assertEqualInt(0, *p))
+			break;
+	}
+	failure("%s", msg);
+	assertEqualMem(buff + buff_size - sizeof(data), data, sizeof(data));
+
+	/* XXX more XXX */
+
+	assertEqualInt(0, close(fd));
+	free(buff);
+}
+
+/*
+ * As above, but using the archive_write_data_block() call.
+ */
+static void
+verify_write_data_block(struct archive *a, int sparse)
+{
+	static const char data[]="abcdefghijklmnopqrstuvwxyz";
+	struct stat st;
+	struct archive_entry *ae;
+	size_t buff_size = 64 * 1024;
+	char *buff, *p;
+	const char *msg = sparse ? "sparse" : "non-sparse";
+	int fd;
+
+	buff = malloc(buff_size);
+	assert(buff != NULL);
+
+	ae = archive_entry_new();
+	assert(ae != NULL);
+	archive_entry_set_size(ae, 8 * buff_size);
+	archive_entry_set_pathname(ae, "test_write_data_block");
+	archive_entry_set_mode(ae, AE_IFREG | 0755);
+	assertEqualIntA(a, 0, archive_write_header(a, ae));
+
+	/* Use archive_write_data_block() to write three
+	   relatively sparse blocks. */
+
+	/* First has non-null data at beginning. */
+	memset(buff, 0, buff_size);
+	memcpy(buff, data, sizeof(data));
+	failure("%s", msg);
+	assertEqualInt(ARCHIVE_OK,
+	    archive_write_data_block(a, buff, buff_size, 100));
+
+	/* Second has non-null data in the middle. */
+	memset(buff, 0, buff_size);
+	memcpy(buff + buff_size / 2 - 3, data, sizeof(data));
+	failure("%s", msg);
+	assertEqualInt(ARCHIVE_OK,
+	    archive_write_data_block(a, buff, buff_size, buff_size + 200));
+
+	/* Third has non-null data at the end. */
+	memset(buff, 0, buff_size);
+	memcpy(buff + buff_size - sizeof(data), data, sizeof(data));
+	failure("%s", msg);
+	assertEqualInt(ARCHIVE_OK,
+	    archive_write_data_block(a, buff, buff_size, buff_size * 2 + 300));
+
+	failure("%s", msg);
+	assertEqualIntA(a, 0, archive_write_finish_entry(a));
+
+	/* Test the entry on disk. */
+	assert(0 == stat(archive_entry_pathname(ae), &st));
+        assertEqualInt(st.st_size, 8 * buff_size);
+	fd = open(archive_entry_pathname(ae), O_RDONLY);
+	if (!assert(fd >= 0))
+		return;
+
+	/* Check 100-byte gap at beginning */
+	assertEqualInt(100, read(fd, buff, 100));
+	failure("%s", msg);
+	for (p = buff; p < buff + 100; ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (!assertEqualInt(0, *p))
+			break;
+	}
+
+	/* Check first block. */
+	assertEqualInt(buff_size, read(fd, buff, buff_size));
+	failure("%s", msg);
+	assertEqualMem(buff, data, sizeof(data));
+	for (p = buff + sizeof(data); p < buff + buff_size; ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (!assertEqualInt(0, *p))
+			break;
+	}
+
+	/* Check 100-byte gap */
+	assertEqualInt(100, read(fd, buff, 100));
+	failure("%s", msg);
+	for (p = buff; p < buff + 100; ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (!assertEqualInt(0, *p))
+			break;
+	}
+
+	/* Check second block. */
+	assertEqualInt(buff_size, read(fd, buff, buff_size));
+	for (p = buff; p < buff + buff_size; ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (p == buff + buff_size / 2 - 3) {
+			assertEqualMem(p, data, sizeof(data));
+			p += sizeof(data);
+		} else if (!assertEqualInt(0, *p))
+			break;
+	}
+
+	/* Check 100-byte gap */
+	assertEqualInt(100, read(fd, buff, 100));
+	failure("%s", msg);
+	for (p = buff; p < buff + 100; ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (!assertEqualInt(0, *p))
+			break;
+	}
+
+	/* Check third block. */
+	assertEqualInt(buff_size, read(fd, buff, buff_size));
+	for (p = buff; p < buff + buff_size - sizeof(data); ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (!assertEqualInt(0, *p))
+			break;
+	}
+	failure("%s", msg);
+	assertEqualMem(buff + buff_size - sizeof(data), data, sizeof(data));
+
+	/* Check another block size beyond last we wrote. */
+	assertEqualInt(buff_size, read(fd, buff, buff_size));
+	failure("%s", msg);
+	for (p = buff; p < buff + buff_size; ++p) {
+		failure("offset: %d, %s", (int)(p - buff), msg);
+		if (!assertEqualInt(0, *p))
+			break;
+	}
+
+
+	/* XXX more XXX */
+
+	assertEqualInt(0, close(fd));
+	free(buff);
+}
+
+DEFINE_TEST(test_write_disk_sparse)
+{
+	struct archive *ad;
+
+
+	/*
+	 * The return values, etc, of the write data functions
+	 * shouldn't change regardless of whether we've requested
+	 * sparsification.  (The performance and pattern of actual
+	 * write calls to the disk should vary, of course, but the
+	 * client program shouldn't see any difference.)
+	 */
+	assert((ad = archive_write_disk_new()) != NULL);
+        archive_write_disk_set_options(ad, 0);
+	verify_write_data(ad, 0);
+	verify_write_data_block(ad, 0);
+	assertEqualInt(0, archive_write_finish(ad));
+
+	assert((ad = archive_write_disk_new()) != NULL);
+        archive_write_disk_set_options(ad, ARCHIVE_EXTRACT_SPARSE);
+	verify_write_data(ad, 1);
+	verify_write_data_block(ad, 1);
+	assertEqualInt(0, archive_write_finish(ad));
+
+}