Merge r364, r378, r379, r393, and r539 from libarchive.googlecode.com:

This is the last phase of the "big decompression refactor" that
puts a lazy reblocking layer between each pair of read filters.
I've also changed the terminology for this area---the two kinds
of objects are now called "read filters" and "read filter bidders"---and
moved ownership of these objects to the archive_read core.

This greatly simplifies implementing new read filters, which
can now use peek/consume I/O semantics both for bidding (arbitrary
look-ahead!) and for reading streams (look-ahead simplifies handling
concatenated streams, for instance).

The first merge here is the overhaul proper; the remainder are small
fixes to correct errors in the initial implementation.

7 files changed, 506 insertions, 613 deletions

diff --git a/lib/libarchive/archive_read.c b/lib/libarchive/archive_read.c
index 6909980..4e307ae 100644
--- a/lib/libarchive/archive_read.c
+++ b/lib/libarchive/archive_read.c
@@ -75,16 +75,6 @@ archive_read_new(void)
 	a->archive.state = ARCHIVE_STATE_NEW;
 	a->entry = archive_entry_new();
 
-	/* Initialize reblocking logic. */
-	a->buffer_size = 64 * 1024; /* 64k */
-	a->buffer = (char *)malloc(a->buffer_size);
-	a->next = a->buffer;
-	if (a->buffer == NULL) {
-		archive_entry_free(a->entry);
-		free(a);
-		return (NULL);
-	}
-
 	return (&a->archive);
 }
 
@@ -117,28 +107,33 @@ archive_read_open(struct archive *a, void *client_data,
 }
 
 static ssize_t
-client_read_proxy(struct archive_read_source *self, const void **buff)
+client_read_proxy(struct archive_read_filter *self, const void **buff)
 {
-	return (self->archive->client.reader)((struct archive *)self->archive,
+	ssize_t r;
+	r = (self->archive->client.reader)(&self->archive->archive,
 	    self->data, buff);
+	self->archive->archive.raw_position += r;
+	return (r);
 }
 
 static int64_t
-client_skip_proxy(struct archive_read_source *self, int64_t request)
+client_skip_proxy(struct archive_read_filter *self, int64_t request)
 {
-	return (self->archive->client.skipper)((struct archive *)self->archive,
+	int64_t r;
+	r = (self->archive->client.skipper)(&self->archive->archive,
 	    self->data, request);
+	self->archive->archive.raw_position += r;
+	return (r);
 }
 
 static int
-client_close_proxy(struct archive_read_source *self)
+client_close_proxy(struct archive_read_filter *self)
 {
 	int r = ARCHIVE_OK;
 
 	if (self->archive->client.closer != NULL)
 		r = (self->archive->client.closer)((struct archive *)self->archive,
 		    self->data);
-	free(self);
 	return (r);
 }
 
@@ -151,6 +146,7 @@ archive_read_open2(struct archive *_a, void *client_data,
     archive_close_callback *client_closer)
 {
 	struct archive_read *a = (struct archive_read *)_a;
+	struct archive_read_filter *filter;
 	int e;
 
 	__archive_check_magic(_a, ARCHIVE_READ_MAGIC, ARCHIVE_STATE_NEW,
@@ -172,27 +168,21 @@ archive_read_open2(struct archive *_a, void *client_data,
 	}
 
 	/* Save the client functions and mock up the initial source. */
-	a->client.opener = client_opener; /* Do we need to remember this? */
 	a->client.reader = client_reader;
 	a->client.skipper = client_skipper;
 	a->client.closer = client_closer;
-	a->client.data = client_data;
 
-	{
-		struct archive_read_source *source;
-
-		source = calloc(1, sizeof(*source));
-		if (source == NULL)
-			return (ARCHIVE_FATAL);
-		source->reader = NULL;
-		source->upstream = NULL;
-		source->archive = a;
-		source->data = client_data;
-		source->read = client_read_proxy;
-		source->skip = client_skip_proxy;
-		source->close = client_close_proxy;
-		a->source = source;
-	}
+	filter = calloc(1, sizeof(*filter));
+	if (filter == NULL)
+		return (ARCHIVE_FATAL);
+	filter->bidder = NULL;
+	filter->upstream = NULL;
+	filter->archive = a;
+	filter->data = client_data;
+	filter->read = client_read_proxy;
+	filter->skip = client_skip_proxy;
+	filter->close = client_close_proxy;
+	a->filter = filter;
 
 	/* In case there's no filter. */
 	a->archive.compression_code = ARCHIVE_COMPRESSION_NONE;
@@ -214,60 +204,49 @@ archive_read_open2(struct archive *_a, void *client_data,
 static int
 build_stream(struct archive_read *a)
 {
-	int number_readers, i, bid, best_bid;
-	struct archive_reader *reader, *best_reader;
-	struct archive_read_source *source;
-	const void *block;
-	ssize_t bytes_read;
+	int number_bidders, i, bid, best_bid;
+	struct archive_read_filter_bidder *bidder, *best_bidder;
+	struct archive_read_filter *filter;
+	int r;
 
-	/* Read first block now for compress format detection. */
-	bytes_read = (a->source->read)(a->source, &block);
-	if (bytes_read < 0) {
-		/* If the first read fails, close before returning error. */
-		if (a->source->close != NULL) {
-			(a->source->close)(a->source);
-			a->source = NULL;
+	for (;;) {
+		number_bidders = sizeof(a->bidders) / sizeof(a->bidders[0]);
+
+		best_bid = 0;
+		best_bidder = NULL;
+
+		bidder = a->bidders;
+		for (i = 0; i < number_bidders; i++, bidder++) {
+			if (bidder->bid != NULL) {
+				bid = (bidder->bid)(bidder, a->filter);
+				if (bid > best_bid) {
+					best_bid = bid;
+					best_bidder = bidder;
+				}
+			}
 		}
-		/* source->read should have already set error information. */
-		return (ARCHIVE_FATAL);
-	}
 
-	number_readers = sizeof(a->readers) / sizeof(a->readers[0]);
-
-	best_bid = 0;
-	best_reader = NULL;
-
-	reader = a->readers;
-	for (i = 0, reader = a->readers; i < number_readers; i++, reader++) {
-		if (reader->bid != NULL) {
-			bid = (reader->bid)(reader, block, bytes_read);
-			if (bid > best_bid) {
-				best_bid = bid;
-				best_reader = reader;
-			}
+		/* If no bidder, we're done. */
+		if (best_bidder == NULL) {
+			a->archive.compression_name = a->filter->name;
+			a->archive.compression_code = a->filter->code;
+			return (ARCHIVE_OK);
 		}
-	}
 
-	/*
-	 * If we have a winner, it becomes the next stage in the pipeline.
-	 */
-	if (best_reader != NULL) {
-		source = (best_reader->init)(a, best_reader, a->source,
-		    block, bytes_read);
-		if (source == NULL)
+		filter
+		    = (struct archive_read_filter *)calloc(1, sizeof(*filter));
+		if (filter == NULL)
 			return (ARCHIVE_FATAL);
-		/* Record the best decompressor for this stream. */
-		a->source = source;
-		/* Recurse to get next pipeline stage. */
-		return (build_stream(a));
+		filter->bidder = best_bidder;
+		filter->archive = a;
+		filter->upstream = a->filter;
+		r = (best_bidder->init)(filter);
+		if (r != ARCHIVE_OK) {
+			free(filter);
+			return (r);
+		}
+		a->filter = filter;
 	}
-
-	/* Save first block of data. */
-	a->client_buff = block;
-	a->client_total = bytes_read;
-	a->client_next = a->client_buff;
-	a->client_avail = a->client_total;
-	return (ARCHIVE_OK);
 }
 
 /*
@@ -594,20 +573,24 @@ archive_read_close(struct archive *_a)
 
 	/* TODO: Clean up the formatters. */
 
-	/* Clean up the stream pipeline. */
-	while (a->source != NULL) {
-		struct archive_read_source *t = a->source->upstream;
-		r1 = (a->source->close)(a->source);
-		if (r1 < r)
-			r = r1;
-		a->source = t;
+	/* Clean up the filter pipeline. */
+	while (a->filter != NULL) {
+		struct archive_read_filter *t = a->filter->upstream;
+		if (a->filter->close != NULL) {
+			r1 = (a->filter->close)(a->filter);
+			if (r1 < r)
+				r = r1;
+		}
+		free(a->filter->buffer);
+		free(a->filter);
+		a->filter = t;
 	}
 
-	/* Release the reader objects. */
-	n = sizeof(a->readers)/sizeof(a->readers[0]);
+	/* Release the bidder objects. */
+	n = sizeof(a->bidders)/sizeof(a->bidders[0]);
 	for (i = 0; i < n; i++) {
-		if (a->readers[i].free != NULL) {
-			r1 = (a->readers[i].free)(&a->readers[i]);
+		if (a->bidders[i].free != NULL) {
+			r1 = (a->bidders[i].free)(&a->bidders[i]);
 			if (r1 < r)
 				r = r1;
 		}
@@ -649,7 +632,6 @@ archive_read_finish(struct archive *_a)
 	if (a->entry)
 		archive_entry_free(a->entry);
 	a->archive.magic = 0;
-	free(a->buffer);
 	free(a);
 #if ARCHIVE_API_VERSION > 1
 	return (r);
@@ -699,20 +681,20 @@ __archive_read_register_format(struct archive_read *a,
  * Used internally by decompression routines to register their bid and
  * initialization functions.
  */
-struct archive_reader *
-__archive_read_get_reader(struct archive_read *a)
+struct archive_read_filter_bidder *
+__archive_read_get_bidder(struct archive_read *a)
 {
 	int i, number_slots;
 
 	__archive_check_magic(&a->archive,
 	    ARCHIVE_READ_MAGIC, ARCHIVE_STATE_NEW,
-	    "__archive_read_get_reader");
+	    "__archive_read_get_bidder");
 
-	number_slots = sizeof(a->readers) / sizeof(a->readers[0]);
+	number_slots = sizeof(a->bidders) / sizeof(a->bidders[0]);
 
 	for (i = 0; i < number_slots; i++) {
-		if (a->readers[i].bid == NULL)
-			return (a->readers + i);
+		if (a->bidders[i].bid == NULL)
+			return (a->bidders + i);
 	}
 
 	__archive_errx(1, "Not enough slots for compression registration");
@@ -725,7 +707,7 @@ __archive_read_get_reader(struct archive_read *a)
  * flexible read-ahead and allows the I/O code to operate in a
  * zero-copy manner most of the time.
  *
- * In the ideal case, block providers give the I/O code blocks of data
+ * In the ideal case, filters generate blocks of data
  * and __archive_read_ahead() just returns pointers directly into
  * those blocks.  Then __archive_read_consume() just bumps those
  * pointers.  Only if your request would span blocks does the I/O
@@ -738,7 +720,7 @@ __archive_read_get_reader(struct archive_read *a)
  *  * "I just want some data."  Ask for 1 byte and pay attention to
  *    the "number of bytes available" from __archive_read_ahead().
  *    You can consume more than you asked for; you just can't consume
- *    more than is available right now.  If you consume everything that's
+ *    more than is available.  If you consume everything that's
  *    immediately available, the next read_ahead() call will pull
  *    the next block.
  *  * "I want to output a large block of data."  As above, ask for 1 byte,
@@ -790,10 +772,17 @@ __archive_read_get_reader(struct archive_read *a)
 const void *
 __archive_read_ahead(struct archive_read *a, size_t min, ssize_t *avail)
 {
+	return (__archive_read_filter_ahead(a->filter, min, avail));
+}
+
+const void *
+__archive_read_filter_ahead(struct archive_read_filter *filter,
+    size_t min, ssize_t *avail)
+{
 	ssize_t bytes_read;
 	size_t tocopy;
 
-	if (a->fatal) {
+	if (filter->fatal) {
 		if (avail)
 			*avail = ARCHIVE_FATAL;
 		return (NULL);
@@ -807,68 +796,68 @@ __archive_read_ahead(struct archive_read *a, size_t min, ssize_t *avail)
 		/*
 		 * If we can satisfy from the copy buffer, we're done.
 		 */
-		if (a->avail >= min) {
+		if (filter->avail >= min) {
 			if (avail != NULL)
-				*avail = a->avail;
-			return (a->next);
+				*avail = filter->avail;
+			return (filter->next);
 		}
 
 		/*
 		 * We can satisfy directly from client buffer if everything
 		 * currently in the copy buffer is still in the client buffer.
 		 */
-		if (a->client_total >= a->client_avail + a->avail
-		    && a->client_avail + a->avail >= min) {
+		if (filter->client_total >= filter->client_avail + filter->avail
+		    && filter->client_avail + filter->avail >= min) {
 			/* "Roll back" to client buffer. */
-			a->client_avail += a->avail;
-			a->client_next -= a->avail;
+			filter->client_avail += filter->avail;
+			filter->client_next -= filter->avail;
 			/* Copy buffer is now empty. */
-			a->avail = 0;
-			a->next = a->buffer;
+			filter->avail = 0;
+			filter->next = filter->buffer;
 			/* Return data from client buffer. */
 			if (avail != NULL)
-				*avail = a->client_avail;
-			return (a->client_next);
+				*avail = filter->client_avail;
+			return (filter->client_next);
 		}
 
 		/* Move data forward in copy buffer if necessary. */
-		if (a->next > a->buffer &&
-		    a->next + min > a->buffer + a->buffer_size) {
-			if (a->avail > 0)
-				memmove(a->buffer, a->next, a->avail);
-			a->next = a->buffer;
+		if (filter->next > filter->buffer &&
+		    filter->next + min > filter->buffer + filter->buffer_size) {
+			if (filter->avail > 0)
+				memmove(filter->buffer, filter->next, filter->avail);
+			filter->next = filter->buffer;
 		}
 
 		/* If we've used up the client data, get more. */
-		if (a->client_avail <= 0) {
-			if (a->end_of_file) {
+		if (filter->client_avail <= 0) {
+			if (filter->end_of_file) {
 				if (avail != NULL)
 					*avail = 0;
 				return (NULL);
 			}
-			bytes_read = (a->source->read)(a->source,
-			    &a->client_buff);
+			bytes_read = (filter->read)(filter,
+			    &filter->client_buff);
 			if (bytes_read < 0) {		/* Read error. */
-				a->client_total = a->client_avail = 0;
-				a->client_next = a->client_buff = NULL;
-				a->fatal = 1;
+				filter->client_total = filter->client_avail = 0;
+				filter->client_next = filter->client_buff = NULL;
+				filter->fatal = 1;
 				if (avail != NULL)
 					*avail = ARCHIVE_FATAL;
 				return (NULL);
 			}
 			if (bytes_read == 0) {	/* Premature end-of-file. */
-				a->client_total = a->client_avail = 0;
-				a->client_next = a->client_buff = NULL;
-				a->end_of_file = 1;
+				filter->client_total = filter->client_avail = 0;
+				filter->client_next = filter->client_buff = NULL;
+				filter->end_of_file = 1;
 				/* Return whatever we do have. */
 				if (avail != NULL)
-					*avail = a->avail;
+					*avail = filter->avail;
 				return (NULL);
 			}
-			a->archive.raw_position += bytes_read;
-			a->client_total = bytes_read;
-			a->client_avail = a->client_total;
-			a->client_next = a->client_buff;
+			filter->position += bytes_read;
+			filter->client_total = bytes_read;
+			filter->client_avail = filter->client_total;
+			filter->client_next = filter->client_buff;
 		}
 		else
 		{
@@ -880,19 +869,22 @@ __archive_read_ahead(struct archive_read *a, size_t min, ssize_t *avail)
 			 */
 
 			/* Ensure the buffer is big enough. */
-			if (min > a->buffer_size) {
+			if (min > filter->buffer_size) {
 				size_t s, t;
 				char *p;
 
 				/* Double the buffer; watch for overflow. */
-				s = t = a->buffer_size;
+				s = t = filter->buffer_size;
+				if (s == 0)
+					s = min;
 				while (s < min) {
 					t *= 2;
 					if (t <= s) { /* Integer overflow! */
-						archive_set_error(&a->archive,
-						    ENOMEM,
+						archive_set_error(
+							&filter->archive->archive,
+							ENOMEM,
 						    "Unable to allocate copy buffer");
-						a->fatal = 1;
+						filter->fatal = 1;
 						if (avail != NULL)
 							*avail = ARCHIVE_FATAL;
 						return (NULL);
@@ -902,39 +894,41 @@ __archive_read_ahead(struct archive_read *a, size_t min, ssize_t *avail)
 				/* Now s >= min, so allocate a new buffer. */
 				p = (char *)malloc(s);
 				if (p == NULL) {
-					archive_set_error(&a->archive, ENOMEM,
+					archive_set_error(
+						&filter->archive->archive,
+						ENOMEM,
 					    "Unable to allocate copy buffer");
-					a->fatal = 1;
+					filter->fatal = 1;
 					if (avail != NULL)
 						*avail = ARCHIVE_FATAL;
 					return (NULL);
 				}
 				/* Move data into newly-enlarged buffer. */
-				if (a->avail > 0)
-					memmove(p, a->next, a->avail);
-				free(a->buffer);
-				a->next = a->buffer = p;
-				a->buffer_size = s;
+				if (filter->avail > 0)
+					memmove(p, filter->next, filter->avail);
+				free(filter->buffer);
+				filter->next = filter->buffer = p;
+				filter->buffer_size = s;
 			}
 
 			/* We can add client data to copy buffer. */
 			/* First estimate: copy to fill rest of buffer. */
-			tocopy = (a->buffer + a->buffer_size)
-			    - (a->next + a->avail);
+			tocopy = (filter->buffer + filter->buffer_size)
+			    - (filter->next + filter->avail);
 			/* Don't waste time buffering more than we need to. */
-			if (tocopy + a->avail > min)
-				tocopy = min - a->avail;
+			if (tocopy + filter->avail > min)
+				tocopy = min - filter->avail;
 			/* Don't copy more than is available. */
-			if (tocopy > a->client_avail)
-				tocopy = a->client_avail;
+			if (tocopy > filter->client_avail)
+				tocopy = filter->client_avail;
 
-			memcpy(a->next + a->avail, a->client_next,
+			memcpy(filter->next + filter->avail, filter->client_next,
 			    tocopy);
 			/* Remove this data from client buffer. */
-			a->client_next += tocopy;
-			a->client_avail -= tocopy;
+			filter->client_next += tocopy;
+			filter->client_avail -= tocopy;
 			/* add it to copy buffer. */
-			a->avail += tocopy;
+			filter->avail += tocopy;
 		}
 	}
 }
@@ -953,16 +947,25 @@ __archive_read_ahead(struct archive_read *a, size_t min, ssize_t *avail)
 ssize_t
 __archive_read_consume(struct archive_read *a, size_t request)
 {
-	if (a->avail > 0) {
+	ssize_t r;
+	r = __archive_read_filter_consume(a->filter, request);
+	a->archive.file_position += r;
+	return (r);
+}
+
+ssize_t
+__archive_read_filter_consume(struct archive_read_filter * filter,
+    size_t request)
+{
+	if (filter->avail > 0) {
 		/* Read came from copy buffer. */
-		a->next += request;
-		a->avail -= request;
+		filter->next += request;
+		filter->avail -= request;
 	} else {
 		/* Read came from client buffer. */
-		a->client_next += request;
-		a->client_avail -= request;
+		filter->client_next += request;
+		filter->client_avail -= request;
 	}
-	a->archive.file_position += request;
 	return (request);
 }
 
@@ -976,23 +979,29 @@ __archive_read_consume(struct archive_read *a, size_t request)
 int64_t
 __archive_read_skip(struct archive_read *a, int64_t request)
 {
+	return (__archive_read_filter_skip(a->filter, request));
+}
+
+int64_t
+__archive_read_filter_skip(struct archive_read_filter *filter, int64_t request)
+{
 	off_t bytes_skipped, total_bytes_skipped = 0;
 	size_t min;
 
-	if (a->fatal)
+	if (filter->fatal)
 		return (-1);
 	/*
 	 * If there is data in the buffers already, use that first.
 	 */
-	if (a->avail > 0) {
-		min = minimum(request, (off_t)a->avail);
-		bytes_skipped = __archive_read_consume(a, min);
+	if (filter->avail > 0) {
+		min = minimum(request, (off_t)filter->avail);
+		bytes_skipped = __archive_read_consume(filter->archive, min);
 		request -= bytes_skipped;
 		total_bytes_skipped += bytes_skipped;
 	}
-	if (a->client_avail > 0) {
-		min = minimum(request, (off_t)a->client_avail);
-		bytes_skipped = __archive_read_consume(a, min);
+	if (filter->client_avail > 0) {
+		min = minimum(request, (off_t)filter->client_avail);
+		bytes_skipped = __archive_read_consume(filter->archive, min);
 		request -= bytes_skipped;
 		total_bytes_skipped += bytes_skipped;
 	}
@@ -1002,23 +1011,22 @@ __archive_read_skip(struct archive_read *a, int64_t request)
 	 * If a client_skipper was provided, try that first.
 	 */
 #if ARCHIVE_API_VERSION < 2
-	if ((a->source->skip != NULL) && (request < SSIZE_MAX)) {
+	if ((filter->skip != NULL) && (request < SSIZE_MAX)) {
 #else
-	if (a->source->skip != NULL) {
+	if (filter->skip != NULL) {
 #endif
-		bytes_skipped = (a->source->skip)(a->source, request);
+		bytes_skipped = (filter->skip)(filter, request);
 		if (bytes_skipped < 0) {	/* error */
-			a->client_total = a->client_avail = 0;
-			a->client_next = a->client_buff = NULL;
-			a->fatal = 1;
+			filter->client_total = filter->client_avail = 0;
+			filter->client_next = filter->client_buff = NULL;
+			filter->fatal = 1;
 			return (bytes_skipped);
 		}
+		filter->archive->archive.file_position += bytes_skipped;
 		total_bytes_skipped += bytes_skipped;
-		a->archive.file_position += bytes_skipped;
 		request -= bytes_skipped;
-		a->client_next = a->client_buff;
-		a->archive.raw_position += bytes_skipped;
-		a->client_avail = a->client_total = 0;
+		filter->client_next = filter->client_buff;
+		filter->client_avail = filter->client_total = 0;
 	}
 	/*
 	 * Note that client_skipper will usually not satisfy the
@@ -1029,18 +1037,20 @@ __archive_read_skip(struct archive_read *a, int64_t request)
 	while (request > 0) {
 		const void* dummy_buffer;
 		ssize_t bytes_read;
-		dummy_buffer = __archive_read_ahead(a, 1, &bytes_read);
+		dummy_buffer = __archive_read_ahead(filter->archive,
+		    1, &bytes_read);
 		if (bytes_read < 0)
 			return (bytes_read);
 		if (bytes_read == 0) {
 			/* We hit EOF before we satisfied the skip request. */
-			archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+			archive_set_error(&filter->archive->archive,
+			    ARCHIVE_ERRNO_MISC,
 			    "Truncated input file (need to skip %jd bytes)",
 			    (intmax_t)request);
 			return (ARCHIVE_FATAL);
 		}
 		min = (size_t)(minimum(bytes_read, request));
-		bytes_read = __archive_read_consume(a, min);
+		bytes_read = __archive_read_consume(filter->archive, min);
 		total_bytes_skipped += bytes_read;
 		request -= bytes_read;
 	}
diff --git a/lib/libarchive/archive_read_private.h b/lib/libarchive/archive_read_private.h
index da0b83f..f1438a17 100644
--- a/lib/libarchive/archive_read_private.h
+++ b/lib/libarchive/archive_read_private.h
@@ -33,72 +33,80 @@
 #include "archive_private.h"
 
 struct archive_read;
-struct archive_reader;
-struct archive_read_source;
+struct archive_read_filter_bidder;
+struct archive_read_filter;
 
 /*
- * A "reader" knows how to provide blocks.  That can include something
- * that reads blocks from disk or socket or a transformation layer
- * that reads blocks from another source and transforms them.  This
- * includes decompression and decryption filters.
- *
- * How bidding works:
+ * How bidding works for filters:
  *   * The bid manager reads the first block from the current source.
  *   * It shows that block to each registered bidder.
- *   * The winning bidder is initialized (with the block and information
- *     about the source)
- *   * The winning bidder becomes the new source and the process repeats
- * This ends only when no reader provides a non-zero bid.
+ *   * The bid manager creates a new filter structure for the winning
+ *     bidder and gives the winning bidder a chance to initialize it.
+ *   * The new filter becomes the top filter in the archive_read structure
+ *     and we repeat the process.
+ * This ends only when no bidder provides a non-zero bid.
  */
-struct archive_reader {
-	/* Configuration data for the reader. */
+struct archive_read_filter_bidder {
+	/* Configuration data for the bidder. */
 	void *data;
-	/* Bidder is handed the initial block from its source. */
-	int (*bid)(struct archive_reader *, const void *buff, size_t);
-	/* Init() is given the archive, upstream source, and the initial
-	 * block above.  It returns a populated source structure. */
-	struct archive_read_source *(*init)(struct archive_read *,
-	    struct archive_reader *, struct archive_read_source *source,
-	    const void *, size_t);
-	/* Release the reader and any configuration data it allocated. */
-	int (*free)(struct archive_reader *);
+	/* Taste the upstream filter to see if we handle this. */
+	int (*bid)(struct archive_read_filter_bidder *,
+	    struct archive_read_filter *);
+	/* Initialize a newly-created filter. */
+	int (*init)(struct archive_read_filter *);
+	/* Release the bidder's configuration data. */
+	int (*free)(struct archive_read_filter_bidder *);
 };
 
 /*
- * A "source" is an instance of a reader.  This structure is
- * allocated and initialized by the init() method of a reader
- * above.
+ * This structure is allocated within the archive_read core
+ * and initialized by archive_read and the init() method of the
+ * corresponding bidder above.
  */
-struct archive_read_source {
-	/* Essentially all sources will need these values, so
+struct archive_read_filter {
+	/* Essentially all filters will need these values, so
 	 * just declare them here. */
-	struct archive_reader *reader; /* Reader that I'm an instance of. */
-	struct archive_read_source *upstream; /* Who I get blocks from. */
-	struct archive_read *archive; /* associated archive. */
+	struct archive_read_filter_bidder *bidder; /* My bidder. */
+	struct archive_read_filter *upstream; /* Who I read from. */
+	struct archive_read *archive; /* Associated archive. */
 	/* Return next block. */
-	ssize_t (*read)(struct archive_read_source *, const void **);
+	ssize_t (*read)(struct archive_read_filter *, const void **);
 	/* Skip forward this many bytes. */
-	int64_t (*skip)(struct archive_read_source *self, int64_t request);
-	/* Close (recursively) and free(self). */
-	int (*close)(struct archive_read_source *self);
+	int64_t (*skip)(struct archive_read_filter *self, int64_t request);
+	/* Close (just this filter) and free(self). */
+	int (*close)(struct archive_read_filter *self);
 	/* My private data. */
 	void *data;
+
+	const char	*name;
+	int		 code;
+
+	/* Used by reblocking logic. */
+	char		*buffer;
+	size_t		 buffer_size;
+	char		*next;		/* Current read location. */
+	size_t		 avail;		/* Bytes in my buffer. */
+	const void	*client_buff;	/* Client buffer information. */
+	size_t		 client_total;
+	const char	*client_next;
+	size_t		 client_avail;
+	int64_t		 position;
+	char		 end_of_file;
+	char		 fatal;
 };
 
 /*
- * The client source is almost the same as an internal source.
+ * The client looks a lot like a filter, so we just wrap it here.
  *
- * TODO: Make archive_read_source and archive_read_client identical so
+ * TODO: Make archive_read_filter and archive_read_client identical so
  * that users of the library can easily register their own
  * transformation filters.  This will probably break the API/ABI and
- * so should be deferred until libarchive 3.0.
+ * so should be deferred at least until libarchive 3.0.
  */
 struct archive_read_client {
-	archive_open_callback	*opener;
 	archive_read_callback	*reader;
 	archive_skip_callback	*skipper;
 	archive_close_callback	*closer;
-	void			*data;
 };
 
 struct archive_read {
@@ -122,28 +130,15 @@ struct archive_read {
 	/* Callbacks to open/read/write/close client archive stream. */
 	struct archive_read_client client;
 
-	/* Registered readers. */
-	struct archive_reader readers[8];
+	/* Registered filter bidders. */
+	struct archive_read_filter_bidder bidders[8];
 
-	/* Source */
-	struct archive_read_source *source;
+	/* Last filter in chain */
+	struct archive_read_filter *filter;
 
 	/* File offset of beginning of most recently-read header. */
 	off_t		  header_position;
 
-
-	/* Used by reblocking logic. */
-	char		*buffer;
-	size_t		 buffer_size;
-	char		*next;		/* Current read location. */
-	size_t		 avail;		/* Bytes in my buffer. */
-	const void	*client_buff;	/* Client buffer information. */
-	size_t		 client_total;
-	const char	*client_next;
-	size_t		 client_avail;
-	char		 end_of_file;
-	char		 fatal;
-
 	/*
 	 * Format detection is mostly the same as compression
 	 * detection, with one significant difference: The bidders
@@ -177,13 +172,14 @@ int	__archive_read_register_format(struct archive_read *a,
 	    int (*read_data_skip)(struct archive_read *),
 	    int (*cleanup)(struct archive_read *));
 
-struct archive_reader
-	*__archive_read_get_reader(struct archive_read *a);
+struct archive_read_filter_bidder
+	*__archive_read_get_bidder(struct archive_read *a);
 
-const void
-	*__archive_read_ahead(struct archive_read *, size_t, ssize_t *);
-ssize_t
-	__archive_read_consume(struct archive_read *, size_t);
-int64_t
-	__archive_read_skip(struct archive_read *, int64_t);
+const void *__archive_read_ahead(struct archive_read *, size_t, ssize_t *);
+const void *__archive_read_filter_ahead(struct archive_read_filter *,
+    size_t, ssize_t *);
+ssize_t	__archive_read_consume(struct archive_read *, size_t);
+ssize_t	__archive_read_filter_consume(struct archive_read_filter *, size_t);
+int64_t	__archive_read_skip(struct archive_read *, int64_t);
+int64_t	__archive_read_filter_skip(struct archive_read_filter *, int64_t);
 #endif
diff --git a/lib/libarchive/archive_read_support_compression_bzip2.c b/lib/libarchive/archive_read_support_compression_bzip2.c
index 6849771..ea39cfe 100644
--- a/lib/libarchive/archive_read_support_compression_bzip2.c
+++ b/lib/libarchive/archive_read_support_compression_bzip2.c
@@ -57,9 +57,9 @@ struct private_data {
 	char		 eof; /* True = found end of compressed data. */
 };
 
-/* Bzip2 source */
-static ssize_t	bzip2_source_read(struct archive_read_source *, const void **);
-static int	bzip2_source_close(struct archive_read_source *);
+/* Bzip2 filter */
+static ssize_t	bzip2_filter_read(struct archive_read_filter *, const void **);
+static int	bzip2_filter_close(struct archive_read_filter *);
 #endif
 
 /*
@@ -68,17 +68,15 @@ static int	bzip2_source_close(struct archive_read_source *);
  * error messages.)  So the bid framework here gets compiled even
  * if bzlib is unavailable.
  */
-static int	bzip2_reader_bid(struct archive_reader *, const void *, size_t);
-static struct archive_read_source *bzip2_reader_init(struct archive_read *,
-    struct archive_reader *, struct archive_read_source *,
-    const void *, size_t);
-static int	bzip2_reader_free(struct archive_reader *);
+static int	bzip2_reader_bid(struct archive_read_filter_bidder *, struct archive_read_filter *);
+static int	bzip2_reader_init(struct archive_read_filter *);
+static int	bzip2_reader_free(struct archive_read_filter_bidder *);
 
 int
 archive_read_support_compression_bzip2(struct archive *_a)
 {
 	struct archive_read *a = (struct archive_read *)_a;
-	struct archive_reader *reader = __archive_read_get_reader(a);
+	struct archive_read_filter_bidder *reader = __archive_read_get_bidder(a);
 
 	if (reader == NULL)
 		return (ARCHIVE_FATAL);
@@ -91,7 +89,7 @@ archive_read_support_compression_bzip2(struct archive *_a)
 }
 
 static int
-bzip2_reader_free(struct archive_reader *self){
+bzip2_reader_free(struct archive_read_filter_bidder *self){
 	(void)self; /* UNUSED */
 	return (ARCHIVE_OK);
 }
@@ -104,61 +102,38 @@ bzip2_reader_free(struct archive_reader *self){
  * from verifying as much as we would like.
  */
 static int
-bzip2_reader_bid(struct archive_reader *self, const void *buff, size_t len)
+bzip2_reader_bid(struct archive_read_filter_bidder *self, struct archive_read_filter *filter)
 {
 	const unsigned char *buffer;
+	size_t avail;
 	int bits_checked;
 
 	(void)self; /* UNUSED */
 
-	if (len < 1)
+	/* Minimal bzip2 archive is 14 bytes. */
+	buffer = __archive_read_filter_ahead(filter, 14, &avail);
+	if (buffer == NULL)
 		return (0);
 
-	buffer = (const unsigned char *)buff;
+	/* First three bytes must be "BZh" */
 	bits_checked = 0;
-	if (buffer[0] != 'B')	/* Verify first ID byte. */
+	if (buffer[0] != 'B' || buffer[1] != 'Z' || buffer[2] != 'h')
 		return (0);
-	bits_checked += 8;
-	if (len < 2)
-		return (bits_checked);
-
-	if (buffer[1] != 'Z')	/* Verify second ID byte. */
-		return (0);
-	bits_checked += 8;
-	if (len < 3)
-		return (bits_checked);
-
-	if (buffer[2] != 'h')	/* Verify third ID byte. */
-		return (0);
-	bits_checked += 8;
-	if (len < 4)
-		return (bits_checked);
+	bits_checked += 24;
 
+	/* Next follows a compression flag which must be an ASCII digit. */
 	if (buffer[3] < '1' || buffer[3] > '9')
 		return (0);
 	bits_checked += 5;
-	if (len < 5)
-		return (bits_checked);
 
 	/* After BZh[1-9], there must be either a data block
 	 * which begins with 0x314159265359 or an end-of-data
 	 * marker of 0x177245385090. */
-
-	if (buffer[4] == 0x31) {
-		/* Verify the data block signature. */
-		size_t s = len;
-		if (s > 10) s = 10;
-		if (memcmp(buffer + 4, "\x31\x41\x59\x26\x53\x59", s - 4) != 0)
-			return (0);
-		bits_checked += 8 * (s - 4);
-	} else if (buffer[4] == 0x17) {
-		/* Verify the end-of-data marker. */
-		size_t s = len;
-		if (s > 10) s = 10;
-		if (memcmp(buffer + 4, "\x17\x72\x45\x38\x50\x90", s - 4) != 0)
-			return (0);
-		bits_checked += 8 * (s - 4);
-	} else
+	if (memcmp(buffer + 4, "\x31\x41\x59\x26\x53\x59", 6) == 0)
+		bits_checked += 48;
+	else if (memcmp(buffer + 4, "\x17\x72\x45\x38\x50\x90", 6) == 0)
+		bits_checked += 48;
+	else
 		return (0);
 
 	return (bits_checked);
@@ -171,19 +146,13 @@ bzip2_reader_bid(struct archive_reader *self, const void *buff, size_t len)
  * decompression.  We can, however, still detect compressed archives
  * and emit a useful message.
  */
-static struct archive_read_source *
-bzip2_reader_init(struct archive_read *a, struct archive_reader *reader,
-    struct archive_read_source *upstream, const void *buff, size_t n)
+static int
+bzip2_reader_init(struct archive_read_filter *self)
 {
-	(void)a;	/* UNUSED */
-	(void)reader;	/* UNUSED */
-	(void)upstream; /* UNUSED */
-	(void)buff;	/* UNUSED */
-	(void)n;	/* UNUSED */
 
-	archive_set_error(&a->archive, -1,
+	archive_set_error(&self->archive->archive, -1,
 	    "This version of libarchive was compiled without bzip2 support");
-	return (NULL);
+	return (ARCHIVE_FATAL);
 }
 
 
@@ -192,67 +161,45 @@ bzip2_reader_init(struct archive_read *a, struct archive_reader *reader,
 /*
  * Setup the callbacks.
  */
-static struct archive_read_source *
-bzip2_reader_init(struct archive_read *a, struct archive_reader *reader,
-    struct archive_read_source *upstream, const void *buff, size_t n)
+static int
+bzip2_reader_init(struct archive_read_filter *self)
 {
 	static const size_t out_block_size = 64 * 1024;
 	void *out_block;
-	struct archive_read_source *self;
 	struct private_data *state;
 
-	(void)reader; /* UNUSED */
+	self->code = ARCHIVE_COMPRESSION_BZIP2;
+	self->name = "bzip2";
 
-	a->archive.compression_code = ARCHIVE_COMPRESSION_BZIP2;
-	a->archive.compression_name = "bzip2";
-
-	self = calloc(sizeof(*self), 1);
 	state = (struct private_data *)calloc(sizeof(*state), 1);
 	out_block = (unsigned char *)malloc(out_block_size);
 	if (self == NULL || state == NULL || out_block == NULL) {
-		archive_set_error(&a->archive, ENOMEM,
-		    "Can't allocate data for %s decompression",
-		    a->archive.compression_name);
+		archive_set_error(&self->archive->archive, ENOMEM,
+		    "Can't allocate data for bzip2 decompression");
 		free(out_block);
 		free(state);
-		free(self);
-		return (NULL);
+		return (ARCHIVE_FATAL);
 	}
 
-
-	self->archive = a;
 	self->data = state;
 	state->out_block_size = out_block_size;
 	state->out_block = out_block;
-	self->upstream = upstream;
-	self->read = bzip2_source_read;
+	self->read = bzip2_filter_read;
 	self->skip = NULL; /* not supported */
-	self->close = bzip2_source_close;
-
-	/*
-	 * A bug in bzlib.h: stream.next_in should be marked 'const'
-	 * but isn't (the library never alters data through the
-	 * next_in pointer, only reads it).  The result: this ugly
-	 * cast to remove 'const'.
-	 */
-	state->stream.next_in = (char *)(uintptr_t)(const void *)buff;
-	state->stream.avail_in = n;
-
-	state->stream.next_out = state->out_block;
-	state->stream.avail_out = state->out_block_size;
+	self->close = bzip2_filter_close;
 
-	return (self);
+	return (ARCHIVE_OK);
 }
 
 /*
  * Return the next block of decompressed data.
  */
 static ssize_t
-bzip2_source_read(struct archive_read_source *self, const void **p)
+bzip2_filter_read(struct archive_read_filter *self, const void **p)
 {
 	struct private_data *state;
 	size_t read_avail, decompressed;
-	const void *read_buf;
+	unsigned char *read_buf;
 	int ret;
 
 	state = (struct private_data *)self->data;
@@ -269,29 +216,8 @@ bzip2_source_read(struct archive_read_source *self, const void **p)
 
 	/* Try to fill the output buffer. */
 	for (;;) {
-		/* If the last upstream block is done, get another one. */
-		if (state->stream.avail_in == 0) {
-			ret = (self->upstream->read)(self->upstream,
-			    &read_buf);
-			/* stream.next_in is really const, but bzlib
-			 * doesn't declare it so. <sigh> */
-			state->stream.next_in
-			    = (unsigned char *)(uintptr_t)read_buf;
-			if (ret < 0)
-				return (ARCHIVE_FATAL);
-			/* There is no more data, return whatever we have. */
-			if (ret == 0) {
-				state->eof = 1;
-				*p = state->out_block;
-				decompressed = state->stream.next_out
-				    - state->out_block;
-				return (decompressed);
-			}
-			state->stream.avail_in = ret;
-		}
-
 		if (!state->valid) {
-			if (state->stream.next_in[0] != 'B') {
+			if (bzip2_reader_bid(self->bidder, self->upstream) == 0) {
 				state->eof = 1;
 				*p = state->out_block;
 				decompressed = state->stream.next_out
@@ -333,8 +259,28 @@ bzip2_source_read(struct archive_read_source *self, const void **p)
 			state->valid = 1;
 		}
 
+		/* stream.next_in is really const, but bzlib
+		 * doesn't declare it so. <sigh> */
+		read_buf = (unsigned char *)(uintptr_t)
+		    __archive_read_filter_ahead(self->upstream, 1, &ret);
+		if (read_buf == NULL)
+			return (ARCHIVE_FATAL);
+		state->stream.next_in = read_buf;
+		state->stream.avail_in = ret;
+		/* There is no more data, return whatever we have. */
+		if (ret == 0) {
+			state->eof = 1;
+			*p = state->out_block;
+			decompressed = state->stream.next_out
+			    - state->out_block;
+			return (decompressed);
+		}
+
 		/* Decompress as much as we can in one pass. */
 		ret = BZ2_bzDecompress(&(state->stream));
+		__archive_read_filter_consume(self->upstream,
+		    (unsigned char *)state->stream.next_in - read_buf);
+
 		switch (ret) {
 		case BZ_STREAM_END: /* Found end of stream. */
 			switch (BZ2_bzDecompressEnd(&(state->stream))) {
@@ -369,7 +315,7 @@ bzip2_source_read(struct archive_read_source *self, const void **p)
  * Clean up the decompressor.
  */
 static int
-bzip2_source_close(struct archive_read_source *self)
+bzip2_filter_close(struct archive_read_filter *self)
 {
 	struct private_data *state;
 	int ret = ARCHIVE_OK;
@@ -390,7 +336,6 @@ bzip2_source_close(struct archive_read_source *self)
 
 	free(state->out_block);
 	free(state);
-	free(self);
 	return (ARCHIVE_OK);
 }
 
diff --git a/lib/libarchive/archive_read_support_compression_compress.c b/lib/libarchive/archive_read_support_compression_compress.c
index 8e2d0d5..2a4383a 100644
--- a/lib/libarchive/archive_read_support_compression_compress.c
+++ b/lib/libarchive/archive_read_support_compression_compress.c
@@ -130,31 +130,29 @@ struct private_data {
 	unsigned char		 stack[65300];
 };
 
-static int	compress_reader_bid(struct archive_reader *, const void *, size_t);
-static struct archive_read_source *compress_reader_init(struct archive_read *,
-    struct archive_reader *, struct archive_read_source *,
-    const void *, size_t);
-static int	compress_reader_free(struct archive_reader *);
+static int	compress_bidder_bid(struct archive_read_filter_bidder *, struct archive_read_filter *);
+static int	compress_bidder_init(struct archive_read_filter *);
+static int	compress_bidder_free(struct archive_read_filter_bidder *);
 
-static ssize_t	compress_source_read(struct archive_read_source *, const void **);
-static int	compress_source_close(struct archive_read_source *);
+static ssize_t	compress_filter_read(struct archive_read_filter *, const void **);
+static int	compress_filter_close(struct archive_read_filter *);
 
-static int	getbits(struct archive_read_source *, int n);
-static int	next_code(struct archive_read_source *);
+static int	getbits(struct archive_read_filter *, int n);
+static int	next_code(struct archive_read_filter *);
 
 int
 archive_read_support_compression_compress(struct archive *_a)
 {
 	struct archive_read *a = (struct archive_read *)_a;
-	struct archive_reader *reader = __archive_read_get_reader(a);
+	struct archive_read_filter_bidder *bidder = __archive_read_get_bidder(a);
 
-	if (reader == NULL)
+	if (bidder == NULL)
 		return (ARCHIVE_FATAL);
 
-	reader->data = NULL;
-	reader->bid = compress_reader_bid;
-	reader->init = compress_reader_init;
-	reader->free = compress_reader_free;
+	bidder->data = NULL;
+	bidder->bid = compress_bidder_bid;
+	bidder->init = compress_bidder_init;
+	bidder->free = compress_bidder_free;
 	return (ARCHIVE_OK);
 }
 
@@ -166,29 +164,28 @@ archive_read_support_compression_compress(struct archive *_a)
  * from verifying as much as we would like.
  */
 static int
-compress_reader_bid(struct archive_reader *self, const void *buff, size_t len)
+compress_bidder_bid(struct archive_read_filter_bidder *self,
+    struct archive_read_filter *filter)
 {
 	const unsigned char *buffer;
+	size_t avail;
 	int bits_checked;
 
 	(void)self; /* UNUSED */
 
-	if (len < 1)
+	buffer = __archive_read_filter_ahead(filter, 2, &avail);
+
+	if (buffer == NULL)
 		return (0);
 
-	buffer = (const unsigned char *)buff;
 	bits_checked = 0;
 	if (buffer[0] != 037)	/* Verify first ID byte. */
 		return (0);
 	bits_checked += 8;
-	if (len < 2)
-		return (bits_checked);
 
 	if (buffer[1] != 0235)	/* Verify second ID byte. */
 		return (0);
 	bits_checked += 8;
-	if (len < 3)
-		return (bits_checked);
 
 	/*
 	 * TODO: Verify more.
@@ -200,66 +197,39 @@ compress_reader_bid(struct archive_reader *self, const void *buff, size_t len)
 /*
  * Setup the callbacks.
  */
-static struct archive_read_source *
-compress_reader_init(struct archive_read *a, struct archive_reader *reader,
-    struct archive_read_source *upstream, const void *buff, size_t n)
+static int
+compress_bidder_init(struct archive_read_filter *self)
 {
-	struct archive_read_source *self;
 	struct private_data *state;
+	static const size_t out_block_size = 64 * 1024;
+	void *out_block;
 	int code;
 
-	(void)reader; /* UNUSED */
-
-	a->archive.compression_code = ARCHIVE_COMPRESSION_COMPRESS;
-	a->archive.compression_name = "compress (.Z)";
-
-	self = calloc(sizeof(*self), 1);
-	if (self == NULL)
-		return (NULL);
-
-	self->read = compress_source_read;
-	self->skip = NULL; /* not supported */
-	self->close = compress_source_close;
-	self->upstream = upstream;
-	self->archive = a;
+	self->code = ARCHIVE_COMPRESSION_COMPRESS;
+	self->name = "compress (.Z)";
 
 	state = (struct private_data *)calloc(sizeof(*state), 1);
-	if (state == NULL) {
-		archive_set_error(&a->archive, ENOMEM,
+	out_block = malloc(out_block_size);
+	if (state == NULL || out_block == NULL) {
+		free(out_block);
+		free(state);
+		archive_set_error(&self->archive->archive, ENOMEM,
 		    "Can't allocate data for %s decompression",
-		    a->archive.compression_name);
-		free(self);
-		return (NULL);
-	}
-	self->data = state;
-
-	state->out_block_size = 64 * 1024;
-	state->out_block = malloc(state->out_block_size);
-
-	if (state->out_block == NULL) {
-		archive_set_error(&a->archive, ENOMEM,
-		    "Can't allocate %s decompression buffers",
-		    a->archive.compression_name);
-		goto fatal;
+		    self->name);
+		return (ARCHIVE_FATAL);
 	}
 
-	state->next_in = (const unsigned char *)buff;
-	state->avail_in = n;
+	self->data = state;
+	state->out_block_size = out_block_size;
+	state->out_block = out_block;
+	self->read = compress_filter_read;
+	self->skip = NULL; /* not supported */
+	self->close = compress_filter_close;
 
-	code = getbits(self, 8);
-	if (code != 037) /* This should be impossible. */
-		goto fatal;
+	/* XXX MOVE THE FOLLOWING OUT OF INIT() XXX */
 
-	code = getbits(self, 8);
-	if (code != 0235) {
-		/* This can happen if the library is receiving 1-byte
-		 * blocks and gzip and compress are both enabled.
-		 * You can't distinguish gzip and compress only from
-		 * the first byte. */
-		archive_set_error(&a->archive, ARCHIVE_ERRNO_FILE_FORMAT,
-		    "Compress signature did not match.");
-		goto fatal;
-	}
+	code = getbits(self, 8); /* Skip first signature byte. */
+	code = getbits(self, 8); /* Skip second signature byte. */
 
 	code = getbits(self, 8);
 	state->maxcode_bits = code & 0x1f;
@@ -279,11 +249,8 @@ compress_reader_init(struct archive_read *a, struct archive_reader *reader,
 		state->suffix[code] = code;
 	}
 	next_code(self);
-	return (self);
 
-fatal:
-	compress_source_close(self);
-	return (NULL);
+	return (ARCHIVE_OK);
 }
 
 /*
@@ -291,7 +258,7 @@ fatal:
  * as necessary.
  */
 static ssize_t
-compress_source_read(struct archive_read_source *self, const void **pblock)
+compress_filter_read(struct archive_read_filter *self, const void **pblock)
 {
 	struct private_data *state;
 	unsigned char *p, *start, *end;
@@ -325,23 +292,22 @@ compress_source_read(struct archive_read_source *self, const void **pblock)
  * Clean up the reader.
  */
 static int
-compress_reader_free(struct archive_reader *self)
+compress_bidder_free(struct archive_read_filter_bidder *self)
 {
 	self->data = NULL;
 	return (ARCHIVE_OK);
 }
 
 /*
- * Close and release a source.
+ * Close and release the filter.
  */
 static int
-compress_source_close(struct archive_read_source *self)
+compress_filter_close(struct archive_read_filter *self)
 {
 	struct private_data *state = (struct private_data *)self->data;
 
 	free(state->out_block);
 	free(state);
-	free(self);
 	return (ARCHIVE_OK);
 }
 
@@ -351,7 +317,7 @@ compress_source_close(struct archive_read_source *self)
  * format error, ARCHIVE_EOF if we hit end of data, ARCHIVE_OK otherwise.
  */
 static int
-next_code(struct archive_read_source *self)
+next_code(struct archive_read_filter *self)
 {
 	struct private_data *state = (struct private_data *)self->data;
 	int code, newcode;
@@ -441,7 +407,7 @@ next_code(struct archive_read_source *self)
  * -1 indicates end of available data.
  */
 static int
-getbits(struct archive_read_source *self, int n)
+getbits(struct archive_read_filter *self, int n)
 {
 	struct private_data *state = (struct private_data *)self->data;
 	int code, ret;
@@ -449,19 +415,18 @@ getbits(struct archive_read_source *self, int n)
 		0x00, 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f, 0x7f, 0xff,
 		0x1ff, 0x3ff, 0x7ff, 0xfff, 0x1fff, 0x3fff, 0x7fff, 0xffff
 	};
-	const void *read_buf;
 
 	while (state->bits_avail < n) {
 		if (state->avail_in <= 0) {
-			read_buf = state->next_in;
-			ret = (self->upstream->read)(self->upstream, &read_buf);
-			state->next_in = read_buf;
-			if (ret < 0)
-				return (ARCHIVE_FATAL);
+			state->next_in
+			    = __archive_read_filter_ahead(self->upstream,
+				1, &ret);
 			if (ret == 0)
 				return (ARCHIVE_EOF);
-/* TODO: Fix this			a->archive.raw_position += ret; */
+			if (ret < 0 || state->next_in == NULL)
+				return (ARCHIVE_FATAL);
 			state->avail_in = ret;
+			__archive_read_filter_consume(self->upstream, ret);
 		}
 		state->bit_buffer |= *state->next_in++ << state->bits_avail;
 		state->avail_in--;
diff --git a/lib/libarchive/archive_read_support_compression_gzip.c b/lib/libarchive/archive_read_support_compression_gzip.c
index 4669ea4..1c7473d 100644
--- a/lib/libarchive/archive_read_support_compression_gzip.c
+++ b/lib/libarchive/archive_read_support_compression_gzip.c
@@ -62,9 +62,9 @@ struct private_data {
 	char		 eof; /* True = found end of compressed data. */
 };
 
-/* Gzip Source. */
-static ssize_t	gzip_source_read(struct archive_read_source *, const void **);
-static int	gzip_source_close(struct archive_read_source *);
+/* Gzip Filter. */
+static ssize_t	gzip_filter_read(struct archive_read_filter *, const void **);
+static int	gzip_filter_close(struct archive_read_filter *);
 #endif
 
 /*
@@ -73,30 +73,28 @@ static int	gzip_source_close(struct archive_read_source *);
  * error messages.)  So the bid framework here gets compiled even
  * if zlib is unavailable.
  */
-static int	gzip_reader_bid(struct archive_reader *, const void *, size_t);
-static struct archive_read_source *gzip_reader_init(struct archive_read *,
-    struct archive_reader *, struct archive_read_source *,
-    const void *, size_t);
-static int	gzip_reader_free(struct archive_reader *);
+static int	gzip_bidder_bid(struct archive_read_filter_bidder *, struct archive_read_filter *);
+static int	gzip_bidder_init(struct archive_read_filter *);
+static int	gzip_bidder_free(struct archive_read_filter_bidder *);
 
 int
 archive_read_support_compression_gzip(struct archive *_a)
 {
 	struct archive_read *a = (struct archive_read *)_a;
-	struct archive_reader *reader = __archive_read_get_reader(a);
+	struct archive_read_filter_bidder *bidder = __archive_read_get_bidder(a);
 
-	if (reader == NULL)
+	if (bidder == NULL)
 		return (ARCHIVE_FATAL);
 
-	reader->data = NULL;
-	reader->bid = gzip_reader_bid;
-	reader->init = gzip_reader_init;
-	reader->free = gzip_reader_free;
+	bidder->data = NULL;
+	bidder->bid = gzip_bidder_bid;
+	bidder->init = gzip_bidder_init;
+	bidder->free = gzip_bidder_free;
 	return (ARCHIVE_OK);
 }
 
 static int
-gzip_reader_free(struct archive_reader *self){
+gzip_bidder_free(struct archive_read_filter_bidder *self){
 	(void)self; /* UNUSED */
 	return (ARCHIVE_OK);
 }
@@ -109,41 +107,35 @@ gzip_reader_free(struct archive_reader *self){
  * from verifying as much as we would like.
  */
 static int
-gzip_reader_bid(struct archive_reader *self, const void *buff, size_t len)
+gzip_bidder_bid(struct archive_read_filter_bidder *self,
+    struct archive_read_filter *filter)
 {
 	const unsigned char *buffer;
+	size_t avail;
 	int bits_checked;
 
 	(void)self; /* UNUSED */
 
-	if (len < 1)
+	buffer = __archive_read_filter_ahead(filter, 8, &avail);
+	if (buffer == NULL)
 		return (0);
 
-	buffer = (const unsigned char *)buff;
 	bits_checked = 0;
 	if (buffer[0] != 037)	/* Verify first ID byte. */
 		return (0);
 	bits_checked += 8;
-	if (len < 2)
-		return (bits_checked);
 
 	if (buffer[1] != 0213)	/* Verify second ID byte. */
 		return (0);
 	bits_checked += 8;
-	if (len < 3)
-		return (bits_checked);
 
 	if (buffer[2] != 8)	/* Compression must be 'deflate'. */
 		return (0);
 	bits_checked += 8;
-	if (len < 4)
-		return (bits_checked);
 
 	if ((buffer[3] & 0xE0)!= 0)	/* No reserved flags set. */
 		return (0);
 	bits_checked += 3;
-	if (len < 5)
-		return (bits_checked);
 
 	/*
 	 * TODO: Verify more; in particular, gzip has an optional
@@ -163,77 +155,56 @@ gzip_reader_bid(struct archive_reader *self, const void *buff, size_t len)
  * decompression.  We can, however, still detect compressed archives
  * and emit a useful message.
  */
-static struct archive_read_source *
-gzip_reader_init(struct archive_read *a, struct archive_reader *reader,
-    struct archive_read_source *upstream, const void *buff, size_t n)
+static int
+gzip_bidder_init(struct archive_read_filter *filter)
 {
-	(void)a;	/* UNUSED */
-	(void)buff;	/* UNUSED */
-	(void)n;	/* UNUSED */
 
-	archive_set_error(&a->archive, -1,
+	archive_set_error(&filter->archive->archive, -1,
 	    "This version of libarchive was compiled without gzip support");
-	return (NULL);
+	return (ARCHIVE_FATAL);
 }
 
 #else
 
 /*
- * Initialize the source object.
+ * Initialize the filter object.
  */
-static struct archive_read_source *
-gzip_reader_init(struct archive_read *a, struct archive_reader *reader,
-    struct archive_read_source *upstream, const void *buff, size_t n)
+static int
+gzip_bidder_init(struct archive_read_filter *self)
 {
+	struct private_data *state;
 	static const size_t out_block_size = 64 * 1024;
 	void *out_block;
-	struct archive_read_source *self;
-	struct private_data *state;
 
-	(void)reader; /* UNUSED */
+	self->code = ARCHIVE_COMPRESSION_GZIP;
+	self->name = "gzip";
 
-	a->archive.compression_code = ARCHIVE_COMPRESSION_GZIP;
-	a->archive.compression_name = "gzip";
-
-	self = calloc(sizeof(*self), 1);
 	state = (struct private_data *)calloc(sizeof(*state), 1);
 	out_block = (unsigned char *)malloc(out_block_size);
-	if (self == NULL || state == NULL || out_block == NULL) {
-		archive_set_error(&a->archive, ENOMEM,
-		    "Can't allocate data for %s decompression",
-		    a->archive.compression_name);
+	if (state == NULL || out_block == NULL) {
 		free(out_block);
 		free(state);
-		free(self);
-		return (NULL);
+		archive_set_error(&self->archive->archive, ENOMEM,
+		    "Can't allocate data for %s decompression",
+		    self->name);
+		return (ARCHIVE_FATAL);
 	}
 
-	self->archive = a;
 	self->data = state;
 	state->out_block_size = out_block_size;
 	state->out_block = out_block;
-	self->upstream = upstream;
-	self->read = gzip_source_read;
+	self->read = gzip_filter_read;
 	self->skip = NULL; /* not supported */
-	self->close = gzip_source_close;
+	self->close = gzip_filter_close;
 
 	state->crc = crc32(0L, NULL, 0);
 	state->header_done = 0; /* We've not yet begun to parse header... */
 
-	/*
-	 * A bug in zlib.h: stream.next_in should be marked 'const'
-	 * but isn't (the library never alters data through the
-	 * next_in pointer, only reads it).  The result: this ugly
-	 * cast to remove 'const'.
-	 */
-	state->stream.next_in = (Bytef *)(uintptr_t)(const void *)buff;
-	state->stream.avail_in = n;
-
-	return (self);
+	return (ARCHIVE_OK);
 }
 
 static int
-header(struct archive_read_source *self)
+header(struct archive_read_filter *self)
 {
 	struct private_data *state;
 	int ret, b;
@@ -372,7 +343,7 @@ header(struct archive_read_source *self)
 }
 
 static ssize_t
-gzip_source_read(struct archive_read_source *self, const void **p)
+gzip_filter_read(struct archive_read_filter *self, const void **p)
 {
 	struct private_data *state;
 	size_t read_avail, decompressed;
@@ -390,20 +361,21 @@ gzip_source_read(struct archive_read_source *self, const void **p)
 	while (state->stream.avail_out > 0 && !state->eof) {
 		/* If the last upstream block is done, get another one. */
 		if (state->stream.avail_in == 0) {
-			ret = (self->upstream->read)(self->upstream,
-			    &read_buf);
+			read_buf = __archive_read_filter_ahead(self->upstream,
+			    1, &ret);
+			if (read_buf == NULL)
+				return (ARCHIVE_FATAL);
 			/* stream.next_in is really const, but zlib
 			 * doesn't declare it so. <sigh> */
 			state->stream.next_in
 			    = (unsigned char *)(uintptr_t)read_buf;
-			if (ret < 0)
-				return (ARCHIVE_FATAL);
 			state->stream.avail_in = ret;
 			/* There is no more data, return whatever we have. */
 			if (ret == 0) {
 				state->eof = 1;
 				break;
 			}
+			__archive_read_filter_consume(self->upstream, ret);
 		}
 
 		/* If we're still parsing header bytes, walk through those. */
@@ -461,7 +433,7 @@ gzip_source_read(struct archive_read_source *self, const void **p)
  * Clean up the decompressor.
  */
 static int
-gzip_source_close(struct archive_read_source *self)
+gzip_filter_close(struct archive_read_filter *self)
 {
 	struct private_data *state;
 	int ret;
@@ -484,7 +456,6 @@ gzip_source_close(struct archive_read_source *self)
 
 	free(state->out_block);
 	free(state);
-	free(self);
 	return (ret);
 }
 
diff --git a/lib/libarchive/archive_read_support_compression_program.c b/lib/libarchive/archive_read_support_compression_program.c
index a9d6c5f..d5cac14 100644
--- a/lib/libarchive/archive_read_support_compression_program.c
+++ b/lib/libarchive/archive_read_support_compression_program.c
@@ -75,12 +75,12 @@ archive_read_support_compression_program(struct archive *_a, const char *cmd)
 
 #include "filter_fork.h"
 
-struct program_reader {
+struct program_bidder {
 	char *cmd;
 	int bid;
 };
 
-struct program_source {
+struct program_filter {
 	char		*description;
 	pid_t		 child;
 	int		 child_stdin, child_stdout;
@@ -92,45 +92,45 @@ struct program_source {
 	size_t		 child_in_buf_avail;
 };
 
-static int	program_reader_bid(struct archive_reader *,
-		    const void *, size_t);
-static struct archive_read_source *program_reader_init(struct archive_read *,
-    struct archive_reader *, struct archive_read_source *,
-    const void *, size_t);
-static int	program_reader_free(struct archive_reader *);
+static int	program_bidder_bid(struct archive_read_filter_bidder *,
+		    struct archive_read_filter *upstream);
+static int	program_bidder_init(struct archive_read_filter *);
+static int	program_bidder_free(struct archive_read_filter_bidder *);
 
-static ssize_t	program_source_read(struct archive_read_source *,
-    const void **);
-static int	program_source_close(struct archive_read_source *);
+static ssize_t	program_filter_read(struct archive_read_filter *,
+		    const void **);
+static int	program_filter_close(struct archive_read_filter *);
 
 
 int
 archive_read_support_compression_program(struct archive *_a, const char *cmd)
 {
 	struct archive_read *a = (struct archive_read *)_a;
-	struct archive_reader *reader = __archive_read_get_reader(a);
-	struct program_reader *state;
+	struct archive_read_filter_bidder *bidder = __archive_read_get_bidder(a);
+	struct program_bidder *state;
 
-	state = (struct program_reader *)calloc(sizeof (*state), 1);
+	state = (struct program_bidder *)calloc(sizeof (*state), 1);
 
 	if (state == NULL)
 		return (ARCHIVE_FATAL);
-	if (reader == NULL)
+	if (bidder == NULL)
 		return (ARCHIVE_FATAL);
 
 	state->cmd = strdup(cmd);
 	state->bid = INT_MAX;
 
-	reader->data = state;
-	reader->bid = program_reader_bid;
-	reader->init = program_reader_init;
-	reader->free = program_reader_free;
+	bidder->data = state;
+	bidder->bid = program_bidder_bid;
+	bidder->init = program_bidder_init;
+	bidder->free = program_bidder_free;
 	return (ARCHIVE_OK);
 }
 
 static int
-program_reader_free(struct archive_reader *self)
+program_bidder_free(struct archive_read_filter_bidder *self)
 {
+	struct program_bidder *state = (struct program_bidder *)self->data;
+	free(state->cmd);
 	free(self->data);
 	return (ARCHIVE_OK);
 }
@@ -142,13 +142,13 @@ program_reader_free(struct archive_reader *self)
  * an infinite pipeline.
  */
 static int
-program_reader_bid(struct archive_reader *self, const void *buff, size_t len)
+program_bidder_bid(struct archive_read_filter_bidder *self,
+    struct archive_read_filter *upstream)
 {
-	struct program_reader *state = self->data;
+	struct program_bidder *state = self->data;
 	int bid = state->bid;
 
-	(void)buff; /* UNUSED */
-	(void)len; /* UNUSED */
+	(void)upstream; /* UNUSED */
 
 	state->bid = 0; /* Don't bid again on this pipeline. */
 
@@ -160,9 +160,9 @@ program_reader_bid(struct archive_reader *self, const void *buff, size_t len)
  */
 
 static ssize_t
-child_read(struct archive_read_source *self, char *buf, size_t buf_len)
+child_read(struct archive_read_filter *self, char *buf, size_t buf_len)
 {
-	struct program_source *state = self->data;
+	struct program_filter *state = self->data;
 	ssize_t ret, requested;
 	const void *child_buf;
 
@@ -191,7 +191,8 @@ restart_read:
 
 	if (state->child_in_buf_avail == 0) {
 		child_buf = state->child_in_buf;
-		ret = (self->upstream->read)(self->upstream, &child_buf);
+		child_buf = __archive_read_filter_ahead(self->upstream,
+			    1, &ret);
 		state->child_in_buf = (const char *)child_buf;
 
 		if (ret < 0) {
@@ -206,6 +207,7 @@ restart_read:
 			fcntl(state->child_stdout, F_SETFL, 0);
 			goto restart_read;
 		}
+		__archive_read_filter_consume(self->upstream, ret);
 		state->child_in_buf_avail = ret;
 	}
 
@@ -240,79 +242,66 @@ restart_read:
 	}
 }
 
-static struct archive_read_source *
-program_reader_init(struct archive_read *a, struct archive_reader *reader,
-    struct archive_read_source *upstream, const void *buff, size_t n)
+static int
+program_bidder_init(struct archive_read_filter *self)
 {
-	struct program_source	*state;
-	struct program_reader   *reader_state;
-	struct archive_read_source *self;
+	struct program_filter	*state;
+	struct program_bidder   *bidder_state;
 	static const size_t out_buf_len = 65536;
 	char *out_buf;
 	char *description;
 	const char *prefix = "Program: ";
 
 
-	reader_state = (struct program_reader *)reader->data;
+	bidder_state = (struct program_bidder *)self->bidder->data;
 
-	self = (struct archive_read_source *)malloc(sizeof(*self));
-	state = (struct program_source *)malloc(sizeof(*state));
+	state = (struct program_filter *)malloc(sizeof(*state));
 	out_buf = (char *)malloc(out_buf_len);
-	description = (char *)malloc(strlen(prefix) + strlen(reader_state->cmd) + 1);
-	if (self == NULL
-	    || state == NULL
-	    || out_buf == NULL
-	    || description == NULL)
-	{
-		archive_set_error(&a->archive, ENOMEM,
+	description = (char *)malloc(strlen(prefix) + strlen(bidder_state->cmd) + 1);
+	if (state == NULL || out_buf == NULL || description == NULL) {
+		archive_set_error(&self->archive->archive, ENOMEM,
 		    "Can't allocate input data");
-		free(self);
 		free(state);
 		free(out_buf);
 		free(description);
-		return (NULL);
+		return (ARCHIVE_FATAL);
 	}
 
-	a->archive.compression_code = ARCHIVE_COMPRESSION_PROGRAM;
+	self->code = ARCHIVE_COMPRESSION_PROGRAM;
 	state->description = description;
 	strcpy(state->description, prefix);
-	strcat(state->description, reader_state->cmd);
-	a->archive.compression_name = state->description;
+	strcat(state->description, bidder_state->cmd);
+	self->name = state->description;
 
 	state->out_buf = out_buf;
 	state->out_buf_len = out_buf_len;
 
-	state->child_in_buf = buff;
-	state->child_in_buf_avail = n;
-
-	if ((state->child = __archive_create_child(reader_state->cmd,
+	if ((state->child = __archive_create_child(bidder_state->cmd,
 		 &state->child_stdin, &state->child_stdout)) == -1) {
 		free(state->out_buf);
 		free(state);
-		archive_set_error(&a->archive, EINVAL,
+		archive_set_error(&self->archive->archive, EINVAL,
 		    "Can't initialise filter");
-		return (NULL);
+		return (ARCHIVE_FATAL);
 	}
 
 	self->data = state;
-	self->read = program_source_read;
+	self->read = program_filter_read;
 	self->skip = NULL;
-	self->close = program_source_close;
-	self->upstream = upstream;
-	self->archive = a;
+	self->close = program_filter_close;
 
 	/* XXX Check that we can read at least one byte? */
-	return (self);
+	return (ARCHIVE_OK);
 }
 
 static ssize_t
-program_source_read(struct archive_read_source *self, const void **buff)
+program_filter_read(struct archive_read_filter *self, const void **buff)
 {
-	struct program_source *state;
+	struct program_filter *state;
 	ssize_t bytes, total;
 	char *p;
 
-	state = (struct program_source *)self->data;
+	state = (struct program_filter *)self->data;
 
 	total = 0;
 	p = state->out_buf;
@@ -323,7 +312,6 @@ program_source_read(struct archive_read_source *self, const void **buff)
 		if (bytes == 0)
 			break;
 		total += bytes;
-/* TODO: fix this */ /*	a->archive.raw_position += bytes_read; */
 	}
 
 	*buff = state->out_buf;
@@ -331,12 +319,12 @@ program_source_read(struct archive_read_source *self, const void **buff)
 }
 
 static int
-program_source_close(struct archive_read_source *self)
+program_filter_close(struct archive_read_filter *self)
 {
-	struct program_source	*state;
+	struct program_filter	*state;
 	int status;
 
-	state = (struct program_source *)self->data;
+	state = (struct program_filter *)self->data;
 
 	/* Shut down the child. */
 	if (state->child_stdin != -1)
@@ -350,7 +338,6 @@ program_source_close(struct archive_read_source *self)
 	free(state->out_buf);
 	free(state->description);
 	free(state);
-	free(self);
 
 	return (ARCHIVE_OK);
 }
diff --git a/lib/libarchive/test/test_read_position.c b/lib/libarchive/test/test_read_position.c
index c700c8c..bc7b53b 100644
--- a/lib/libarchive/test/test_read_position.c
+++ b/lib/libarchive/test/test_read_position.c
@@ -25,8 +25,8 @@
 #include "test.h"
 __FBSDID("$FreeBSD$");
 
-static unsigned char nulls[10000000];
-static unsigned char buff[10000000];
+static unsigned char nulls[10000];
+static unsigned char  buff[10000000];
 
 /* Check that header_position tracks correctly on read. */
 DEFINE_TEST(test_read_position)
@@ -34,42 +34,61 @@ DEFINE_TEST(test_read_position)
 	struct archive *a;
 	struct archive_entry *ae;
 	size_t write_pos;
-	const size_t data_size = 1000000;
+	intmax_t read_position;
+	size_t i, j;
+	size_t data_sizes[] = {0, 5, 511, 512, 513};
 
-	/* Create a simple archive_entry. */
-	assert((ae = archive_entry_new()) != NULL);
-	archive_entry_set_pathname(ae, "testfile");
-	archive_entry_set_mode(ae, S_IFREG);
-	archive_entry_set_size(ae, data_size);
+	/* Sanity test */
+	assert(sizeof(nulls) + 512 + 1024 <= sizeof(buff));
 
+	/* Create an archive. */
 	assert(NULL != (a = archive_write_new()));
 	assertA(0 == archive_write_set_format_pax_restricted(a));
 	assertA(0 == archive_write_set_bytes_per_block(a, 512));
 	assertA(0 == archive_write_open_memory(a, buff, sizeof(buff), &write_pos));
-	assertA(0 == archive_write_header(a, ae));
-	archive_entry_free(ae);
-	assertA(data_size == (size_t)archive_write_data(a, nulls, sizeof(nulls)));
-#if ARCHIVE_VERSION_NUMBER < 2000000
+
+	for (i = 0; i < sizeof(data_sizes)/sizeof(data_sizes[0]); ++i) {
+		/* Create a simple archive_entry. */
+		assert((ae = archive_entry_new()) != NULL);
+		archive_entry_set_pathname(ae, "testfile");
+		archive_entry_set_mode(ae, S_IFREG);
+		archive_entry_set_size(ae, data_sizes[i]);
+		assertA(0 == archive_write_header(a, ae));
+		archive_entry_free(ae);
+		assertA(data_sizes[i]
+		    == (size_t)archive_write_data(a, nulls, sizeof(nulls)));
+	}
 	assertA(0 == archive_write_close(a));
-	archive_write_finish(a);
-#else
 	assertA(0 == archive_write_finish(a));
-#endif
-	/* 512-byte header + data_size (rounded up) + 1024 end-of-archive */
-	assert(write_pos == ((512 + data_size + 1024 + 511)/512)*512);
 
 	/* Read the archive back. */
 	assert(NULL != (a = archive_read_new()));
 	assertA(0 == archive_read_support_format_tar(a));
 	assertA(0 == archive_read_open_memory2(a, buff, sizeof(buff), 512));
-	assert((intmax_t)0 == (intmax_t)archive_read_header_position(a));
-	assertA(0 == archive_read_next_header(a, &ae));
-	assert((intmax_t)0 == (intmax_t)archive_read_header_position(a));
-	assertA(0 == archive_read_data_skip(a));
-	assert((intmax_t)0 == (intmax_t)archive_read_header_position(a));
+
+	read_position = 0;
+	/* Initial header position is zero. */
+	assert(read_position == (intmax_t)archive_read_header_position(a));
+	for (j = 0; j < i; ++j) {
+		assertA(0 == archive_read_next_header(a, &ae));
+		assert(read_position
+		    == (intmax_t)archive_read_header_position(a));
+		/* Every other entry: read, then skip */
+		if (j & 1)
+			assertEqualInt(ARCHIVE_OK,
+			    archive_read_data_into_buffer(a, buff, 1));
+		assertA(0 == archive_read_data_skip(a));
+		/* read_data_skip() doesn't change header_position */
+		assert(read_position
+		    == (intmax_t)archive_read_header_position(a));
+
+		read_position += 512; /* Size of header. */
+		read_position += (data_sizes[j] + 511) & ~511;
+	}
+
 	assertA(1 == archive_read_next_header(a, &ae));
-	assert((intmax_t)((data_size + 511 + 512)/512)*512 == (intmax_t)archive_read_header_position(a));
+	assert(read_position == (intmax_t)archive_read_header_position(a));
 	assertA(0 == archive_read_close(a));
-	assert((intmax_t)((data_size + 511 + 512)/512)*512 == (intmax_t)archive_read_header_position(a));
+	assert(read_position == (intmax_t)archive_read_header_position(a));
 	archive_read_finish(a);
 }