diff options
Diffstat (limited to 'contrib/xz/src/liblzma/common')
52 files changed, 7681 insertions, 0 deletions
diff --git a/contrib/xz/src/liblzma/common/alone_decoder.c b/contrib/xz/src/liblzma/common/alone_decoder.c new file mode 100644 index 0000000..039b428 --- /dev/null +++ b/contrib/xz/src/liblzma/common/alone_decoder.c @@ -0,0 +1,232 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file alone_decoder.c +/// \brief Decoder for LZMA_Alone files +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "alone_decoder.h" +#include "lzma_decoder.h" +#include "lz_decoder.h" + + +struct lzma_coder_s { + lzma_next_coder next; + + enum { + SEQ_PROPERTIES, + SEQ_DICTIONARY_SIZE, + SEQ_UNCOMPRESSED_SIZE, + SEQ_CODER_INIT, + SEQ_CODE, + } sequence; + + /// Position in the header fields + size_t pos; + + /// Uncompressed size decoded from the header + lzma_vli uncompressed_size; + + /// Memory usage limit + uint64_t memlimit; + + /// Amount of memory actually needed (only an estimate) + uint64_t memusage; + + /// Options decoded from the header needed to initialize + /// the LZMA decoder + lzma_options_lzma options; +}; + + +static lzma_ret +alone_decode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action) +{ + while (*out_pos < out_size + && (coder->sequence == SEQ_CODE || *in_pos < in_size)) + switch (coder->sequence) { + case SEQ_PROPERTIES: + if (lzma_lzma_lclppb_decode(&coder->options, in[*in_pos])) + return LZMA_FORMAT_ERROR; + + coder->sequence = SEQ_DICTIONARY_SIZE; + ++*in_pos; + break; + + case SEQ_DICTIONARY_SIZE: + coder->options.dict_size + |= (size_t)(in[*in_pos]) << (coder->pos * 8); + + if (++coder->pos == 4) { + if (coder->options.dict_size != UINT32_MAX) { + // A hack to ditch tons of false positives: + // We allow only dictionary sizes that are + // 2^n or 2^n + 2^(n-1). LZMA_Alone created + // only files with 2^n, but accepts any + // dictionary size. If someone complains, this + // will be reconsidered. + uint32_t d = coder->options.dict_size - 1; + d |= d >> 2; + d |= d >> 3; + d |= d >> 4; + d |= d >> 8; + d |= d >> 16; + ++d; + + if (d != coder->options.dict_size) + return LZMA_FORMAT_ERROR; + } + + coder->pos = 0; + coder->sequence = SEQ_UNCOMPRESSED_SIZE; + } + + ++*in_pos; + break; + + case SEQ_UNCOMPRESSED_SIZE: + coder->uncompressed_size + |= (lzma_vli)(in[*in_pos]) << (coder->pos * 8); + ++*in_pos; + if (++coder->pos < 8) + break; + + // Another hack to ditch false positives: Assume that + // if the uncompressed size is known, it must be less + // than 256 GiB. Again, if someone complains, this + // will be reconsidered. + if (coder->uncompressed_size != LZMA_VLI_UNKNOWN + && coder->uncompressed_size + >= (LZMA_VLI_C(1) << 38)) + return LZMA_FORMAT_ERROR; + + // Calculate the memory usage so that it is ready + // for SEQ_CODER_INIT. + coder->memusage = lzma_lzma_decoder_memusage(&coder->options) + + LZMA_MEMUSAGE_BASE; + + coder->pos = 0; + coder->sequence = SEQ_CODER_INIT; + + // Fall through + + case SEQ_CODER_INIT: { + if (coder->memusage > coder->memlimit) + return LZMA_MEMLIMIT_ERROR; + + lzma_filter_info filters[2] = { + { + .init = &lzma_lzma_decoder_init, + .options = &coder->options, + }, { + .init = NULL, + } + }; + + const lzma_ret ret = lzma_next_filter_init(&coder->next, + allocator, filters); + if (ret != LZMA_OK) + return ret; + + // Use a hack to set the uncompressed size. + lzma_lz_decoder_uncompressed(coder->next.coder, + coder->uncompressed_size); + + coder->sequence = SEQ_CODE; + break; + } + + case SEQ_CODE: { + return coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + } + + default: + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +alone_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +alone_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + *memusage = coder->memusage; + *old_memlimit = coder->memlimit; + + if (new_memlimit != 0) { + if (new_memlimit < coder->memusage) + return LZMA_MEMLIMIT_ERROR; + + coder->memlimit = new_memlimit; + } + + return LZMA_OK; +} + + +extern lzma_ret +lzma_alone_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + uint64_t memlimit) +{ + lzma_next_coder_init(&lzma_alone_decoder_init, next, allocator); + + if (memlimit == 0) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &alone_decode; + next->end = &alone_decoder_end; + next->memconfig = &alone_decoder_memconfig; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + next->coder->sequence = SEQ_PROPERTIES; + next->coder->pos = 0; + next->coder->options.dict_size = 0; + next->coder->options.preset_dict = NULL; + next->coder->options.preset_dict_size = 0; + next->coder->uncompressed_size = 0; + next->coder->memlimit = memlimit; + next->coder->memusage = LZMA_MEMUSAGE_BASE; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_alone_decoder(lzma_stream *strm, uint64_t memlimit) +{ + lzma_next_strm_init(lzma_alone_decoder_init, strm, memlimit); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/alone_decoder.h b/contrib/xz/src/liblzma/common/alone_decoder.h new file mode 100644 index 0000000..70d0d2a --- /dev/null +++ b/contrib/xz/src/liblzma/common/alone_decoder.h @@ -0,0 +1,22 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file alone_decoder.h +/// \brief Decoder for LZMA_Alone files +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_ALONE_DECODER_H +#define LZMA_ALONE_DECODER_H + +#include "common.h" + + +extern lzma_ret lzma_alone_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, uint64_t memlimit); + +#endif diff --git a/contrib/xz/src/liblzma/common/alone_encoder.c b/contrib/xz/src/liblzma/common/alone_encoder.c new file mode 100644 index 0000000..d8c0170 --- /dev/null +++ b/contrib/xz/src/liblzma/common/alone_encoder.c @@ -0,0 +1,157 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file alone_decoder.c +/// \brief Decoder for LZMA_Alone files +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "lzma_encoder.h" + + +#define ALONE_HEADER_SIZE (1 + 4 + 8) + + +struct lzma_coder_s { + lzma_next_coder next; + + enum { + SEQ_HEADER, + SEQ_CODE, + } sequence; + + size_t header_pos; + uint8_t header[ALONE_HEADER_SIZE]; +}; + + +static lzma_ret +alone_encode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action) +{ + while (*out_pos < out_size) + switch (coder->sequence) { + case SEQ_HEADER: + lzma_bufcpy(coder->header, &coder->header_pos, + ALONE_HEADER_SIZE, + out, out_pos, out_size); + if (coder->header_pos < ALONE_HEADER_SIZE) + return LZMA_OK; + + coder->sequence = SEQ_CODE; + break; + + case SEQ_CODE: + return coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + + default: + assert(0); + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +alone_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +// At least for now, this is not used by any internal function. +static lzma_ret +alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_options_lzma *options) +{ + lzma_next_coder_init(&alone_encoder_init, next, allocator); + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &alone_encode; + next->end = &alone_encoder_end; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + // Basic initializations + next->coder->sequence = SEQ_HEADER; + next->coder->header_pos = 0; + + // Encode the header: + // - Properties (1 byte) + if (lzma_lzma_lclppb_encode(options, next->coder->header)) + return LZMA_OPTIONS_ERROR; + + // - Dictionary size (4 bytes) + if (options->dict_size < LZMA_DICT_SIZE_MIN) + return LZMA_OPTIONS_ERROR; + + // Round up to to the next 2^n or 2^n + 2^(n - 1) depending on which + // one is the next unless it is UINT32_MAX. While the header would + // allow any 32-bit integer, we do this to keep the decoder of liblzma + // accepting the resulting files. + uint32_t d = options->dict_size - 1; + d |= d >> 2; + d |= d >> 3; + d |= d >> 4; + d |= d >> 8; + d |= d >> 16; + if (d != UINT32_MAX) + ++d; + + unaligned_write32le(next->coder->header + 1, d); + + // - Uncompressed size (always unknown and using EOPM) + memset(next->coder->header + 1 + 4, 0xFF, 8); + + // Initialize the LZMA encoder. + const lzma_filter_info filters[2] = { + { + .init = &lzma_lzma_encoder_init, + .options = (void *)(options), + }, { + .init = NULL, + } + }; + + return lzma_next_filter_init(&next->coder->next, allocator, filters); +} + + +/* +extern lzma_ret +lzma_alone_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_options_alone *options) +{ + lzma_next_coder_init(&alone_encoder_init, next, allocator, options); +} +*/ + + +extern LZMA_API(lzma_ret) +lzma_alone_encoder(lzma_stream *strm, const lzma_options_lzma *options) +{ + lzma_next_strm_init(alone_encoder_init, strm, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/auto_decoder.c b/contrib/xz/src/liblzma/common/auto_decoder.c new file mode 100644 index 0000000..ae6c3e7 --- /dev/null +++ b/contrib/xz/src/liblzma/common/auto_decoder.c @@ -0,0 +1,186 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file auto_decoder.c +/// \brief Autodetect between .xz Stream and .lzma (LZMA_Alone) formats +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_decoder.h" +#include "alone_decoder.h" + + +struct lzma_coder_s { + /// Stream decoder or LZMA_Alone decoder + lzma_next_coder next; + + uint64_t memlimit; + uint32_t flags; + + enum { + SEQ_INIT, + SEQ_CODE, + SEQ_FINISH, + } sequence; +}; + + +static lzma_ret +auto_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + switch (coder->sequence) { + case SEQ_INIT: + if (*in_pos >= in_size) + return LZMA_OK; + + // Update the sequence now, because we want to continue from + // SEQ_CODE even if we return some LZMA_*_CHECK. + coder->sequence = SEQ_CODE; + + // Detect the file format. For now this is simple, since if + // it doesn't start with 0xFD (the first magic byte of the + // new format), it has to be LZMA_Alone, or something that + // we don't support at all. + if (in[*in_pos] == 0xFD) { + return_if_error(lzma_stream_decoder_init( + &coder->next, allocator, + coder->memlimit, coder->flags)); + } else { + return_if_error(lzma_alone_decoder_init(&coder->next, + allocator, coder->memlimit)); + + // If the application wants to know about missing + // integrity check or about the check in general, we + // need to handle it here, because LZMA_Alone decoder + // doesn't accept any flags. + if (coder->flags & LZMA_TELL_NO_CHECK) + return LZMA_NO_CHECK; + + if (coder->flags & LZMA_TELL_ANY_CHECK) + return LZMA_GET_CHECK; + } + + // Fall through + + case SEQ_CODE: { + const lzma_ret ret = coder->next.code( + coder->next.coder, allocator, + in, in_pos, in_size, + out, out_pos, out_size, action); + if (ret != LZMA_STREAM_END + || (coder->flags & LZMA_CONCATENATED) == 0) + return ret; + + coder->sequence = SEQ_FINISH; + } + + // Fall through + + case SEQ_FINISH: + // When LZMA_DECODE_CONCATENATED was used and we were decoding + // LZMA_Alone file, we need to check check that there is no + // trailing garbage and wait for LZMA_FINISH. + if (*in_pos < in_size) + return LZMA_DATA_ERROR; + + return action == LZMA_FINISH ? LZMA_STREAM_END : LZMA_OK; + + default: + assert(0); + return LZMA_PROG_ERROR; + } +} + + +static void +auto_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_check +auto_decoder_get_check(const lzma_coder *coder) +{ + // It is LZMA_Alone if get_check is NULL. + return coder->next.get_check == NULL ? LZMA_CHECK_NONE + : coder->next.get_check(coder->next.coder); +} + + +static lzma_ret +auto_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + lzma_ret ret; + + if (coder->next.memconfig != NULL) { + ret = coder->next.memconfig(coder->next.coder, + memusage, old_memlimit, new_memlimit); + assert(*old_memlimit == coder->memlimit); + } else { + // No coder is configured yet. Use the base value as + // the current memory usage. + *memusage = LZMA_MEMUSAGE_BASE; + *old_memlimit = coder->memlimit; + ret = LZMA_OK; + } + + if (ret == LZMA_OK && new_memlimit != 0) + coder->memlimit = new_memlimit; + + return ret; +} + + +static lzma_ret +auto_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags) +{ + lzma_next_coder_init(&auto_decoder_init, next, allocator); + + if (memlimit == 0) + return LZMA_PROG_ERROR; + + if (flags & ~LZMA_SUPPORTED_FLAGS) + return LZMA_OPTIONS_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &auto_decode; + next->end = &auto_decoder_end; + next->get_check = &auto_decoder_get_check; + next->memconfig = &auto_decoder_memconfig; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + next->coder->memlimit = memlimit; + next->coder->flags = flags; + next->coder->sequence = SEQ_INIT; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_auto_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) +{ + lzma_next_strm_init(auto_decoder_init, strm, memlimit, flags); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/block_buffer_decoder.c b/contrib/xz/src/liblzma/common/block_buffer_decoder.c new file mode 100644 index 0000000..ff27a11 --- /dev/null +++ b/contrib/xz/src/liblzma/common/block_buffer_decoder.c @@ -0,0 +1,80 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_buffer_decoder.c +/// \brief Single-call .xz Block decoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "block_decoder.h" + + +extern LZMA_API(lzma_ret) +lzma_block_buffer_decode(lzma_block *block, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + if (in_pos == NULL || (in == NULL && *in_pos != in_size) + || *in_pos > in_size || out_pos == NULL + || (out == NULL && *out_pos != out_size) + || *out_pos > out_size) + return LZMA_PROG_ERROR; + + // Initialize the Block decoder. + lzma_next_coder block_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_block_decoder_init( + &block_decoder, allocator, block); + + if (ret == LZMA_OK) { + // Save the positions so that we can restore them in case + // an error occurs. + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; + + // Do the actual decoding. + ret = block_decoder.code(block_decoder.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, + LZMA_FINISH); + + if (ret == LZMA_STREAM_END) { + ret = LZMA_OK; + } else { + if (ret == LZMA_OK) { + // Either the input was truncated or the + // output buffer was too small. + assert(*in_pos == in_size + || *out_pos == out_size); + + // If all the input was consumed, then the + // input is truncated, even if the output + // buffer is also full. This is because + // processing the last byte of the Block + // never produces output. + // + // NOTE: This assumption may break when new + // filters are added, if the end marker of + // the filter doesn't consume at least one + // complete byte. + if (*in_pos == in_size) + ret = LZMA_DATA_ERROR; + else + ret = LZMA_BUF_ERROR; + } + + // Restore the positions. + *in_pos = in_start; + *out_pos = out_start; + } + } + + // Free the decoder memory. This needs to be done even if + // initialization fails, because the internal API doesn't + // require the initialization function to free its memory on error. + lzma_next_end(&block_decoder, allocator); + + return ret; +} diff --git a/contrib/xz/src/liblzma/common/block_buffer_encoder.c b/contrib/xz/src/liblzma/common/block_buffer_encoder.c new file mode 100644 index 0000000..4d90fee --- /dev/null +++ b/contrib/xz/src/liblzma/common/block_buffer_encoder.c @@ -0,0 +1,299 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_buffer_encoder.c +/// \brief Single-call .xz Block encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "block_encoder.h" +#include "filter_encoder.h" +#include "lzma2_encoder.h" +#include "check.h" + + +/// Estimate the maximum size of the Block Header and Check fields for +/// a Block that uses LZMA2 uncompressed chunks. We could use +/// lzma_block_header_size() but this is simpler. +/// +/// Block Header Size + Block Flags + Compressed Size +/// + Uncompressed Size + Filter Flags for LZMA2 + CRC32 + Check +/// and round up to the next multiple of four to take Header Padding +/// into account. +#define HEADERS_BOUND ((1 + 1 + 2 * LZMA_VLI_BYTES_MAX + 3 + 4 \ + + LZMA_CHECK_SIZE_MAX + 3) & ~3) + + +static lzma_vli +lzma2_bound(lzma_vli uncompressed_size) +{ + // Prevent integer overflow in overhead calculation. + if (uncompressed_size > COMPRESSED_SIZE_MAX) + return 0; + + // Calculate the exact overhead of the LZMA2 headers: Round + // uncompressed_size up to the next multiple of LZMA2_CHUNK_MAX, + // multiply by the size of per-chunk header, and add one byte for + // the end marker. + const lzma_vli overhead = ((uncompressed_size + LZMA2_CHUNK_MAX - 1) + / LZMA2_CHUNK_MAX) + * LZMA2_HEADER_UNCOMPRESSED + 1; + + // Catch the possible integer overflow. + if (COMPRESSED_SIZE_MAX - overhead < uncompressed_size) + return 0; + + return uncompressed_size + overhead; +} + + +extern LZMA_API(size_t) +lzma_block_buffer_bound(size_t uncompressed_size) +{ + // For now, if the data doesn't compress, we always use uncompressed + // chunks of LZMA2. In future we may use Subblock filter too, but + // but for simplicity we probably will still use the same bound + // calculation even though Subblock filter would have slightly less + // overhead. + lzma_vli lzma2_size = lzma2_bound(uncompressed_size); + if (lzma2_size == 0) + return 0; + + // Take Block Padding into account. + lzma2_size = (lzma2_size + 3) & ~LZMA_VLI_C(3); + +#if SIZE_MAX < LZMA_VLI_MAX + // Catch the possible integer overflow on 32-bit systems. There's no + // overflow on 64-bit systems, because lzma2_bound() already takes + // into account the size of the headers in the Block. + if (SIZE_MAX - HEADERS_BOUND < lzma2_size) + return 0; +#endif + + return HEADERS_BOUND + lzma2_size; +} + + +static lzma_ret +block_encode_uncompressed(lzma_block *block, const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // TODO: Figure out if the last filter is LZMA2 or Subblock and use + // that filter to encode the uncompressed chunks. + + // Use LZMA2 uncompressed chunks. We wouldn't need a dictionary at + // all, but LZMA2 always requires a dictionary, so use the minimum + // value to minimize memory usage of the decoder. + lzma_options_lzma lzma2 = { + .dict_size = LZMA_DICT_SIZE_MIN, + }; + + lzma_filter filters[2]; + filters[0].id = LZMA_FILTER_LZMA2; + filters[0].options = &lzma2; + filters[1].id = LZMA_VLI_UNKNOWN; + + // Set the above filter options to *block temporarily so that we can + // encode the Block Header. + lzma_filter *filters_orig = block->filters; + block->filters = filters; + + if (lzma_block_header_size(block) != LZMA_OK) { + block->filters = filters_orig; + return LZMA_PROG_ERROR; + } + + // Check that there's enough output space. The caller has already + // set block->compressed_size to what lzma2_bound() has returned, + // so we can reuse that value. We know that compressed_size is a + // known valid VLI and header_size is a small value so their sum + // will never overflow. + assert(block->compressed_size == lzma2_bound(in_size)); + if (out_size - *out_pos + < block->header_size + block->compressed_size) { + block->filters = filters_orig; + return LZMA_BUF_ERROR; + } + + if (lzma_block_header_encode(block, out + *out_pos) != LZMA_OK) { + block->filters = filters_orig; + return LZMA_PROG_ERROR; + } + + block->filters = filters_orig; + *out_pos += block->header_size; + + // Encode the data using LZMA2 uncompressed chunks. + size_t in_pos = 0; + uint8_t control = 0x01; // Dictionary reset + + while (in_pos < in_size) { + // Control byte: Indicate uncompressed chunk, of which + // the first resets the dictionary. + out[(*out_pos)++] = control; + control = 0x02; // No dictionary reset + + // Size of the uncompressed chunk + const size_t copy_size + = MIN(in_size - in_pos, LZMA2_CHUNK_MAX); + out[(*out_pos)++] = (copy_size - 1) >> 8; + out[(*out_pos)++] = (copy_size - 1) & 0xFF; + + // The actual data + assert(*out_pos + copy_size <= out_size); + memcpy(out + *out_pos, in + in_pos, copy_size); + + in_pos += copy_size; + *out_pos += copy_size; + } + + // End marker + out[(*out_pos)++] = 0x00; + assert(*out_pos <= out_size); + + return LZMA_OK; +} + + +static lzma_ret +block_encode_normal(lzma_block *block, lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // Find out the size of the Block Header. + block->compressed_size = lzma2_bound(in_size); + if (block->compressed_size == 0) + return LZMA_DATA_ERROR; + + block->uncompressed_size = in_size; + return_if_error(lzma_block_header_size(block)); + + // Reserve space for the Block Header and skip it for now. + if (out_size - *out_pos <= block->header_size) + return LZMA_BUF_ERROR; + + const size_t out_start = *out_pos; + *out_pos += block->header_size; + + // Limit out_size so that we stop encoding if the output would grow + // bigger than what uncompressed Block would be. + if (out_size - *out_pos > block->compressed_size) + out_size = *out_pos + block->compressed_size; + + // TODO: In many common cases this could be optimized to use + // significantly less memory. + lzma_next_coder raw_encoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_raw_encoder_init( + &raw_encoder, allocator, block->filters); + + if (ret == LZMA_OK) { + size_t in_pos = 0; + ret = raw_encoder.code(raw_encoder.coder, allocator, + in, &in_pos, in_size, out, out_pos, out_size, + LZMA_FINISH); + } + + // NOTE: This needs to be run even if lzma_raw_encoder_init() failed. + lzma_next_end(&raw_encoder, allocator); + + if (ret == LZMA_STREAM_END) { + // Compression was successful. Write the Block Header. + block->compressed_size + = *out_pos - (out_start + block->header_size); + ret = lzma_block_header_encode(block, out + out_start); + if (ret != LZMA_OK) + ret = LZMA_PROG_ERROR; + + } else if (ret == LZMA_OK) { + // Output buffer became full. + ret = LZMA_BUF_ERROR; + } + + // Reset *out_pos if something went wrong. + if (ret != LZMA_OK) + *out_pos = out_start; + + return ret; +} + + +extern LZMA_API(lzma_ret) +lzma_block_buffer_encode(lzma_block *block, lzma_allocator *allocator, + const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // Sanity checks + if (block == NULL || block->filters == NULL + || (in == NULL && in_size != 0) || out == NULL + || out_pos == NULL || *out_pos > out_size) + return LZMA_PROG_ERROR; + + // Check the version field. + if (block->version != 0) + return LZMA_OPTIONS_ERROR; + + // Size of a Block has to be a multiple of four, so limit the size + // here already. This way we don't need to check it again when adding + // Block Padding. + out_size -= (out_size - *out_pos) & 3; + + // Get the size of the Check field. + const size_t check_size = lzma_check_size(block->check); + if (check_size == UINT32_MAX) + return LZMA_PROG_ERROR; + + // Reserve space for the Check field. + if (out_size - *out_pos <= check_size) + return LZMA_BUF_ERROR; + + out_size -= check_size; + + // Do the actual compression. + const lzma_ret ret = block_encode_normal(block, allocator, + in, in_size, out, out_pos, out_size); + if (ret != LZMA_OK) { + // If the error was something else than output buffer + // becoming full, return the error now. + if (ret != LZMA_BUF_ERROR) + return ret; + + // The data was uncompressible (at least with the options + // given to us) or the output buffer was too small. Use the + // uncompressed chunks of LZMA2 to wrap the data into a valid + // Block. If we haven't been given enough output space, even + // this may fail. + return_if_error(block_encode_uncompressed(block, in, in_size, + out, out_pos, out_size)); + } + + assert(*out_pos <= out_size); + + // Block Padding. No buffer overflow here, because we already adjusted + // out_size so that (out_size - out_start) is a multiple of four. + // Thus, if the buffer is full, the loop body can never run. + for (size_t i = (size_t)(block->compressed_size); i & 3; ++i) { + assert(*out_pos < out_size); + out[(*out_pos)++] = 0x00; + } + + // If there's no Check field, we are done now. + if (check_size > 0) { + // Calculate the integrity check. We reserved space for + // the Check field earlier so we don't need to check for + // available output space here. + lzma_check_state check; + lzma_check_init(&check, block->check); + lzma_check_update(&check, block->check, in, in_size); + lzma_check_finish(&check, block->check); + + memcpy(block->raw_check, check.buffer.u8, check_size); + memcpy(out + *out_pos, check.buffer.u8, check_size); + *out_pos += check_size; + } + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/block_decoder.c b/contrib/xz/src/liblzma/common/block_decoder.c new file mode 100644 index 0000000..a3ce6f4 --- /dev/null +++ b/contrib/xz/src/liblzma/common/block_decoder.c @@ -0,0 +1,242 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_decoder.c +/// \brief Decodes .xz Blocks +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "block_decoder.h" +#include "filter_decoder.h" +#include "check.h" + + +struct lzma_coder_s { + enum { + SEQ_CODE, + SEQ_PADDING, + SEQ_CHECK, + } sequence; + + /// The filters in the chain; initialized with lzma_raw_decoder_init(). + lzma_next_coder next; + + /// Decoding options; we also write Compressed Size and Uncompressed + /// Size back to this structure when the decoding has been finished. + lzma_block *block; + + /// Compressed Size calculated while decoding + lzma_vli compressed_size; + + /// Uncompressed Size calculated while decoding + lzma_vli uncompressed_size; + + /// Maximum allowed Compressed Size; this takes into account the + /// size of the Block Header and Check fields when Compressed Size + /// is unknown. + lzma_vli compressed_limit; + + /// Position when reading the Check field + size_t check_pos; + + /// Check of the uncompressed data + lzma_check_state check; +}; + + +static inline bool +update_size(lzma_vli *size, lzma_vli add, lzma_vli limit) +{ + if (limit > LZMA_VLI_MAX) + limit = LZMA_VLI_MAX; + + if (limit < *size || limit - *size < add) + return true; + + *size += add; + + return false; +} + + +static inline bool +is_size_valid(lzma_vli size, lzma_vli reference) +{ + return reference == LZMA_VLI_UNKNOWN || reference == size; +} + + +static lzma_ret +block_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + switch (coder->sequence) { + case SEQ_CODE: { + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; + + const lzma_ret ret = coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + + const size_t in_used = *in_pos - in_start; + const size_t out_used = *out_pos - out_start; + + // NOTE: We compare to compressed_limit here, which prevents + // the total size of the Block growing past LZMA_VLI_MAX. + if (update_size(&coder->compressed_size, in_used, + coder->compressed_limit) + || update_size(&coder->uncompressed_size, + out_used, + coder->block->uncompressed_size)) + return LZMA_DATA_ERROR; + + lzma_check_update(&coder->check, coder->block->check, + out + out_start, out_used); + + if (ret != LZMA_STREAM_END) + return ret; + + // Compressed and Uncompressed Sizes are now at their final + // values. Verify that they match the values given to us. + if (!is_size_valid(coder->compressed_size, + coder->block->compressed_size) + || !is_size_valid(coder->uncompressed_size, + coder->block->uncompressed_size)) + return LZMA_DATA_ERROR; + + // Copy the values into coder->block. The caller + // may use this information to construct Index. + coder->block->compressed_size = coder->compressed_size; + coder->block->uncompressed_size = coder->uncompressed_size; + + coder->sequence = SEQ_PADDING; + } + + // Fall through + + case SEQ_PADDING: + // Compressed Data is padded to a multiple of four bytes. + while (coder->compressed_size & 3) { + if (*in_pos >= in_size) + return LZMA_OK; + + // We use compressed_size here just get the Padding + // right. The actual Compressed Size was stored to + // coder->block already, and won't be modified by + // us anymore. + ++coder->compressed_size; + + if (in[(*in_pos)++] != 0x00) + return LZMA_DATA_ERROR; + } + + if (coder->block->check == LZMA_CHECK_NONE) + return LZMA_STREAM_END; + + lzma_check_finish(&coder->check, coder->block->check); + coder->sequence = SEQ_CHECK; + + // Fall through + + case SEQ_CHECK: { + const size_t check_size = lzma_check_size(coder->block->check); + lzma_bufcpy(in, in_pos, in_size, coder->block->raw_check, + &coder->check_pos, check_size); + if (coder->check_pos < check_size) + return LZMA_OK; + + // Validate the Check only if we support it. + // coder->check.buffer may be uninitialized + // when the Check ID is not supported. + if (lzma_check_is_supported(coder->block->check) + && memcmp(coder->block->raw_check, + coder->check.buffer.u8, + check_size) != 0) + return LZMA_DATA_ERROR; + + return LZMA_STREAM_END; + } + } + + return LZMA_PROG_ERROR; +} + + +static void +block_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +extern lzma_ret +lzma_block_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_block *block) +{ + lzma_next_coder_init(&lzma_block_decoder_init, next, allocator); + + // Validate the options. lzma_block_unpadded_size() does that for us + // except for Uncompressed Size and filters. Filters are validated + // by the raw decoder. + if (lzma_block_unpadded_size(block) == 0 + || !lzma_vli_is_valid(block->uncompressed_size)) + return LZMA_PROG_ERROR; + + // Allocate and initialize *next->coder if needed. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &block_decode; + next->end = &block_decoder_end; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + // Basic initializations + next->coder->sequence = SEQ_CODE; + next->coder->block = block; + next->coder->compressed_size = 0; + next->coder->uncompressed_size = 0; + + // If Compressed Size is not known, we calculate the maximum allowed + // value so that encoded size of the Block (including Block Padding) + // is still a valid VLI and a multiple of four. + next->coder->compressed_limit + = block->compressed_size == LZMA_VLI_UNKNOWN + ? (LZMA_VLI_MAX & ~LZMA_VLI_C(3)) + - block->header_size + - lzma_check_size(block->check) + : block->compressed_size; + + // Initialize the check. It's caller's problem if the Check ID is not + // supported, and the Block decoder cannot verify the Check field. + // Caller can test lzma_check_is_supported(block->check). + next->coder->check_pos = 0; + lzma_check_init(&next->coder->check, block->check); + + // Initialize the filter chain. + return lzma_raw_decoder_init(&next->coder->next, allocator, + block->filters); +} + + +extern LZMA_API(lzma_ret) +lzma_block_decoder(lzma_stream *strm, lzma_block *block) +{ + lzma_next_strm_init(lzma_block_decoder_init, strm, block); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/block_decoder.h b/contrib/xz/src/liblzma/common/block_decoder.h new file mode 100644 index 0000000..7da9df6 --- /dev/null +++ b/contrib/xz/src/liblzma/common/block_decoder.h @@ -0,0 +1,22 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_decoder.h +/// \brief Decodes .xz Blocks +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BLOCK_DECODER_H +#define LZMA_BLOCK_DECODER_H + +#include "common.h" + + +extern lzma_ret lzma_block_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_block *block); + +#endif diff --git a/contrib/xz/src/liblzma/common/block_encoder.c b/contrib/xz/src/liblzma/common/block_encoder.c new file mode 100644 index 0000000..ca51523 --- /dev/null +++ b/contrib/xz/src/liblzma/common/block_encoder.c @@ -0,0 +1,212 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_encoder.c +/// \brief Encodes .xz Blocks +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "block_encoder.h" +#include "filter_encoder.h" +#include "check.h" + + +struct lzma_coder_s { + /// The filters in the chain; initialized with lzma_raw_decoder_init(). + lzma_next_coder next; + + /// Encoding options; we also write Unpadded Size, Compressed Size, + /// and Uncompressed Size back to this structure when the encoding + /// has been finished. + lzma_block *block; + + enum { + SEQ_CODE, + SEQ_PADDING, + SEQ_CHECK, + } sequence; + + /// Compressed Size calculated while encoding + lzma_vli compressed_size; + + /// Uncompressed Size calculated while encoding + lzma_vli uncompressed_size; + + /// Position in the Check field + size_t pos; + + /// Check of the uncompressed data + lzma_check_state check; +}; + + +static lzma_ret +block_encode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + // Check that our amount of input stays in proper limits. + if (LZMA_VLI_MAX - coder->uncompressed_size < in_size - *in_pos) + return LZMA_DATA_ERROR; + + switch (coder->sequence) { + case SEQ_CODE: { + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; + + const lzma_ret ret = coder->next.code(coder->next.coder, + allocator, in, in_pos, in_size, + out, out_pos, out_size, action); + + const size_t in_used = *in_pos - in_start; + const size_t out_used = *out_pos - out_start; + + if (COMPRESSED_SIZE_MAX - coder->compressed_size < out_used) + return LZMA_DATA_ERROR; + + coder->compressed_size += out_used; + + // No need to check for overflow because we have already + // checked it at the beginning of this function. + coder->uncompressed_size += in_used; + + lzma_check_update(&coder->check, coder->block->check, + in + in_start, in_used); + + if (ret != LZMA_STREAM_END || action == LZMA_SYNC_FLUSH) + return ret; + + assert(*in_pos == in_size); + assert(action == LZMA_FINISH); + + // Copy the values into coder->block. The caller + // may use this information to construct Index. + coder->block->compressed_size = coder->compressed_size; + coder->block->uncompressed_size = coder->uncompressed_size; + + coder->sequence = SEQ_PADDING; + } + + // Fall through + + case SEQ_PADDING: + // Pad Compressed Data to a multiple of four bytes. We can + // use coder->compressed_size for this since we don't need + // it for anything else anymore. + while (coder->compressed_size & 3) { + if (*out_pos >= out_size) + return LZMA_OK; + + out[*out_pos] = 0x00; + ++*out_pos; + ++coder->compressed_size; + } + + if (coder->block->check == LZMA_CHECK_NONE) + return LZMA_STREAM_END; + + lzma_check_finish(&coder->check, coder->block->check); + + coder->sequence = SEQ_CHECK; + + // Fall through + + case SEQ_CHECK: { + const size_t check_size = lzma_check_size(coder->block->check); + lzma_bufcpy(coder->check.buffer.u8, &coder->pos, check_size, + out, out_pos, out_size); + if (coder->pos < check_size) + return LZMA_OK; + + memcpy(coder->block->raw_check, coder->check.buffer.u8, + check_size); + return LZMA_STREAM_END; + } + } + + return LZMA_PROG_ERROR; +} + + +static void +block_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_end(&coder->next, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +block_encoder_update(lzma_coder *coder, lzma_allocator *allocator, + const lzma_filter *filters lzma_attribute((unused)), + const lzma_filter *reversed_filters) +{ + if (coder->sequence != SEQ_CODE) + return LZMA_PROG_ERROR; + + return lzma_next_filter_update( + &coder->next, allocator, reversed_filters); +} + + +extern lzma_ret +lzma_block_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_block *block) +{ + lzma_next_coder_init(&lzma_block_encoder_init, next, allocator); + + if (block->version != 0) + return LZMA_OPTIONS_ERROR; + + // If the Check ID is not supported, we cannot calculate the check and + // thus not create a proper Block. + if ((unsigned int)(block->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + + if (!lzma_check_is_supported(block->check)) + return LZMA_UNSUPPORTED_CHECK; + + // Allocate and initialize *next->coder if needed. + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &block_encode; + next->end = &block_encoder_end; + next->update = &block_encoder_update; + next->coder->next = LZMA_NEXT_CODER_INIT; + } + + // Basic initializations + next->coder->sequence = SEQ_CODE; + next->coder->block = block; + next->coder->compressed_size = 0; + next->coder->uncompressed_size = 0; + next->coder->pos = 0; + + // Initialize the check + lzma_check_init(&next->coder->check, block->check); + + // Initialize the requested filters. + return lzma_raw_encoder_init(&next->coder->next, allocator, + block->filters); +} + + +extern LZMA_API(lzma_ret) +lzma_block_encoder(lzma_stream *strm, lzma_block *block) +{ + lzma_next_strm_init(lzma_block_encoder_init, strm, block); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/block_encoder.h b/contrib/xz/src/liblzma/common/block_encoder.h new file mode 100644 index 0000000..b9eff0b --- /dev/null +++ b/contrib/xz/src/liblzma/common/block_encoder.h @@ -0,0 +1,47 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_encoder.h +/// \brief Encodes .xz Blocks +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_BLOCK_ENCODER_H +#define LZMA_BLOCK_ENCODER_H + +#include "common.h" + + +/// \brief Biggest Compressed Size value that the Block encoder supports +/// +/// The maximum size of a single Block is limited by the maximum size of +/// a Stream, which in theory is 2^63 - 3 bytes (i.e. LZMA_VLI_MAX - 3). +/// While the size is really big and no one should hit it in practice, we +/// take it into account in some places anyway to catch some errors e.g. if +/// application passes insanely big value to some function. +/// +/// We could take into account the headers etc. to determine the exact +/// maximum size of the Compressed Data field, but the complexity would give +/// us nothing useful. Instead, limit the size of Compressed Data so that +/// even with biggest possible Block Header and Check fields the total +/// encoded size of the Block stays as a valid VLI. This doesn't guarantee +/// that the size of the Stream doesn't grow too big, but that problem is +/// taken care outside the Block handling code. +/// +/// ~LZMA_VLI_C(3) is to guarantee that if we need padding at the end of +/// the Compressed Data field, it will still stay in the proper limit. +/// +/// This constant is in this file because it is needed in both +/// block_encoder.c and block_buffer_encoder.c. +#define COMPRESSED_SIZE_MAX ((LZMA_VLI_MAX - LZMA_BLOCK_HEADER_SIZE_MAX \ + - LZMA_CHECK_SIZE_MAX) & ~LZMA_VLI_C(3)) + + +extern lzma_ret lzma_block_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, lzma_block *block); + +#endif diff --git a/contrib/xz/src/liblzma/common/block_header_decoder.c b/contrib/xz/src/liblzma/common/block_header_decoder.c new file mode 100644 index 0000000..2c9573e --- /dev/null +++ b/contrib/xz/src/liblzma/common/block_header_decoder.c @@ -0,0 +1,116 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_header_decoder.c +/// \brief Decodes Block Header from .xz files +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "check.h" + + +static void +free_properties(lzma_block *block, lzma_allocator *allocator) +{ + // Free allocated filter options. The last array member is not + // touched after the initialization in the beginning of + // lzma_block_header_decode(), so we don't need to touch that here. + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) { + lzma_free(block->filters[i].options, allocator); + block->filters[i].id = LZMA_VLI_UNKNOWN; + block->filters[i].options = NULL; + } + + return; +} + + +extern LZMA_API(lzma_ret) +lzma_block_header_decode(lzma_block *block, + lzma_allocator *allocator, const uint8_t *in) +{ + // NOTE: We consider the header to be corrupt not only when the + // CRC32 doesn't match, but also when variable-length integers + // are invalid or over 63 bits, or if the header is too small + // to contain the claimed information. + + // Initialize the filter options array. This way the caller can + // safely free() the options even if an error occurs in this function. + for (size_t i = 0; i <= LZMA_FILTERS_MAX; ++i) { + block->filters[i].id = LZMA_VLI_UNKNOWN; + block->filters[i].options = NULL; + } + + // Always zero for now. + block->version = 0; + + // Validate Block Header Size and Check type. The caller must have + // already set these, so it is a programming error if this test fails. + if (lzma_block_header_size_decode(in[0]) != block->header_size + || (unsigned int)(block->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + + // Exclude the CRC32 field. + const size_t in_size = block->header_size - 4; + + // Verify CRC32 + if (lzma_crc32(in, in_size, 0) != unaligned_read32le(in + in_size)) + return LZMA_DATA_ERROR; + + // Check for unsupported flags. + if (in[1] & 0x3C) + return LZMA_OPTIONS_ERROR; + + // Start after the Block Header Size and Block Flags fields. + size_t in_pos = 2; + + // Compressed Size + if (in[1] & 0x40) { + return_if_error(lzma_vli_decode(&block->compressed_size, + NULL, in, &in_pos, in_size)); + + // Validate Compressed Size. This checks that it isn't zero + // and that the total size of the Block is a valid VLI. + if (lzma_block_unpadded_size(block) == 0) + return LZMA_DATA_ERROR; + } else { + block->compressed_size = LZMA_VLI_UNKNOWN; + } + + // Uncompressed Size + if (in[1] & 0x80) + return_if_error(lzma_vli_decode(&block->uncompressed_size, + NULL, in, &in_pos, in_size)); + else + block->uncompressed_size = LZMA_VLI_UNKNOWN; + + // Filter Flags + const size_t filter_count = (in[1] & 3) + 1; + for (size_t i = 0; i < filter_count; ++i) { + const lzma_ret ret = lzma_filter_flags_decode( + &block->filters[i], allocator, + in, &in_pos, in_size); + if (ret != LZMA_OK) { + free_properties(block, allocator); + return ret; + } + } + + // Padding + while (in_pos < in_size) { + if (in[in_pos++] != 0x00) { + free_properties(block, allocator); + + // Possibly some new field present so use + // LZMA_OPTIONS_ERROR instead of LZMA_DATA_ERROR. + return LZMA_OPTIONS_ERROR; + } + } + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/block_header_encoder.c b/contrib/xz/src/liblzma/common/block_header_encoder.c new file mode 100644 index 0000000..707dd0c --- /dev/null +++ b/contrib/xz/src/liblzma/common/block_header_encoder.c @@ -0,0 +1,132 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_header_encoder.c +/// \brief Encodes Block Header for .xz files +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "check.h" + + +extern LZMA_API(lzma_ret) +lzma_block_header_size(lzma_block *block) +{ + if (block->version != 0) + return LZMA_OPTIONS_ERROR; + + // Block Header Size + Block Flags + CRC32. + uint32_t size = 1 + 1 + 4; + + // Compressed Size + if (block->compressed_size != LZMA_VLI_UNKNOWN) { + const uint32_t add = lzma_vli_size(block->compressed_size); + if (add == 0 || block->compressed_size == 0) + return LZMA_PROG_ERROR; + + size += add; + } + + // Uncompressed Size + if (block->uncompressed_size != LZMA_VLI_UNKNOWN) { + const uint32_t add = lzma_vli_size(block->uncompressed_size); + if (add == 0) + return LZMA_PROG_ERROR; + + size += add; + } + + // List of Filter Flags + if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) + return LZMA_PROG_ERROR; + + for (size_t i = 0; block->filters[i].id != LZMA_VLI_UNKNOWN; ++i) { + // Don't allow too many filters. + if (i == LZMA_FILTERS_MAX) + return LZMA_PROG_ERROR; + + uint32_t add; + return_if_error(lzma_filter_flags_size(&add, + block->filters + i)); + + size += add; + } + + // Pad to a multiple of four bytes. + block->header_size = (size + 3) & ~UINT32_C(3); + + // NOTE: We don't verify that the encoded size of the Block stays + // within limits. This is because it is possible that we are called + // with exaggerated Compressed Size (e.g. LZMA_VLI_MAX) to reserve + // space for Block Header, and later called again with lower, + // real values. + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_block_header_encode(const lzma_block *block, uint8_t *out) +{ + // Validate everything but filters. + if (lzma_block_unpadded_size(block) == 0 + || !lzma_vli_is_valid(block->uncompressed_size)) + return LZMA_PROG_ERROR; + + // Indicate the size of the buffer _excluding_ the CRC32 field. + const size_t out_size = block->header_size - 4; + + // Store the Block Header Size. + out[0] = out_size / 4; + + // We write Block Flags in pieces. + out[1] = 0x00; + size_t out_pos = 2; + + // Compressed Size + if (block->compressed_size != LZMA_VLI_UNKNOWN) { + return_if_error(lzma_vli_encode(block->compressed_size, NULL, + out, &out_pos, out_size)); + + out[1] |= 0x40; + } + + // Uncompressed Size + if (block->uncompressed_size != LZMA_VLI_UNKNOWN) { + return_if_error(lzma_vli_encode(block->uncompressed_size, NULL, + out, &out_pos, out_size)); + + out[1] |= 0x80; + } + + // Filter Flags + if (block->filters == NULL || block->filters[0].id == LZMA_VLI_UNKNOWN) + return LZMA_PROG_ERROR; + + size_t filter_count = 0; + do { + // There can be a maximum of four filters. + if (filter_count == LZMA_FILTERS_MAX) + return LZMA_PROG_ERROR; + + return_if_error(lzma_filter_flags_encode( + block->filters + filter_count, + out, &out_pos, out_size)); + + } while (block->filters[++filter_count].id != LZMA_VLI_UNKNOWN); + + out[1] |= filter_count - 1; + + // Padding + memzero(out + out_pos, out_size - out_pos); + + // CRC32 + unaligned_write32le(out + out_size, lzma_crc32(out, out_size, 0)); + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/block_util.c b/contrib/xz/src/liblzma/common/block_util.c new file mode 100644 index 0000000..cb9cde2 --- /dev/null +++ b/contrib/xz/src/liblzma/common/block_util.c @@ -0,0 +1,90 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file block_header.c +/// \brief Utility functions to handle lzma_block +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "index.h" + + +extern LZMA_API(lzma_ret) +lzma_block_compressed_size(lzma_block *block, lzma_vli total_size) +{ + // Validate everything but Uncompressed Size and filters. + if (lzma_block_unpadded_size(block) == 0) + return LZMA_PROG_ERROR; + + const uint32_t container_size = block->header_size + + lzma_check_size(block->check); + + // Validate that Compressed Size will be greater than zero. + if (container_size <= total_size) + return LZMA_DATA_ERROR; + + // Calculate what Compressed Size is supposed to be. + // If Compressed Size was present in Block Header, + // compare that the new value matches it. + const lzma_vli compressed_size = total_size - container_size; + if (block->compressed_size != LZMA_VLI_UNKNOWN + && block->compressed_size != compressed_size) + return LZMA_DATA_ERROR; + + block->compressed_size = compressed_size; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_vli) +lzma_block_unpadded_size(const lzma_block *block) +{ + // Validate the values that we are interested in i.e. all but + // Uncompressed Size and the filters. + // + // NOTE: This function is used for validation too, so it is + // essential that these checks are always done even if + // Compressed Size is unknown. + if (block == NULL || block->version != 0 + || block->header_size < LZMA_BLOCK_HEADER_SIZE_MIN + || block->header_size > LZMA_BLOCK_HEADER_SIZE_MAX + || (block->header_size & 3) + || !lzma_vli_is_valid(block->compressed_size) + || block->compressed_size == 0 + || (unsigned int)(block->check) > LZMA_CHECK_ID_MAX) + return 0; + + // If Compressed Size is unknown, return that we cannot know + // size of the Block either. + if (block->compressed_size == LZMA_VLI_UNKNOWN) + return LZMA_VLI_UNKNOWN; + + // Calculate Unpadded Size and validate it. + const lzma_vli unpadded_size = block->compressed_size + + block->header_size + + lzma_check_size(block->check); + + assert(unpadded_size >= UNPADDED_SIZE_MIN); + if (unpadded_size > UNPADDED_SIZE_MAX) + return 0; + + return unpadded_size; +} + + +extern LZMA_API(lzma_vli) +lzma_block_total_size(const lzma_block *block) +{ + lzma_vli unpadded_size = lzma_block_unpadded_size(block); + + if (unpadded_size != LZMA_VLI_UNKNOWN) + unpadded_size = vli_ceil4(unpadded_size); + + return unpadded_size; +} diff --git a/contrib/xz/src/liblzma/common/chunk_size.c b/contrib/xz/src/liblzma/common/chunk_size.c new file mode 100644 index 0000000..363f07e --- /dev/null +++ b/contrib/xz/src/liblzma/common/chunk_size.c @@ -0,0 +1,67 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file chunk_size.c +/// \brief Finds out the minimal reasonable chunk size for a filter chain +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +/** + * \brief Finds out the minimal reasonable chunk size for a filter chain + * + * This function helps determining the Uncompressed Sizes of the Blocks when + * doing multi-threaded encoding. + * + * When compressing a large file on a system having multiple CPUs or CPU + * cores, the file can be split into smaller chunks, that are compressed + * independently into separate Blocks in the same .lzma Stream. + * + * \return Minimum reasonable Uncompressed Size of a Block. The + * recommended minimum Uncompressed Size is between this value + * and the value times two. + + Zero if the Uncompressed Sizes of Blocks don't matter + */ +extern LZMA_API(size_t) +lzma_chunk_size(const lzma_options_filter *filters) +{ + while (filters->id != LZMA_VLI_UNKNOWN) { + switch (filters->id) { + // TODO LZMA_FILTER_SPARSE + + case LZMA_FILTER_COPY: + case LZMA_FILTER_SUBBLOCK: + case LZMA_FILTER_X86: + case LZMA_FILTER_POWERPC: + case LZMA_FILTER_IA64: + case LZMA_FILTER_ARM: + case LZMA_FILTER_ARMTHUMB: + case LZMA_FILTER_SPARC: + // These are very fast, thus there is no point in + // splitting the data into smaller blocks. + break; + + case LZMA_FILTER_LZMA1: + // The block sizes of the possible next filters in + // the chain are irrelevant after the LZMA filter. + return ((lzma_options_lzma *)(filters->options)) + ->dictionary_size; + + default: + // Unknown filters + return 0; + } + + ++filters; + } + + // Indicate that splitting would be useless. + return SIZE_MAX; +} diff --git a/contrib/xz/src/liblzma/common/common.c b/contrib/xz/src/liblzma/common/common.c new file mode 100644 index 0000000..2f185e4 --- /dev/null +++ b/contrib/xz/src/liblzma/common/common.c @@ -0,0 +1,374 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file common.h +/// \brief Common functions needed in many places in liblzma +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +///////////// +// Version // +///////////// + +extern LZMA_API(uint32_t) +lzma_version_number(void) +{ + return LZMA_VERSION; +} + + +extern LZMA_API(const char *) +lzma_version_string(void) +{ + return LZMA_VERSION_STRING; +} + + +/////////////////////// +// Memory allocation // +/////////////////////// + +extern void * lzma_attribute((malloc)) +lzma_alloc(size_t size, lzma_allocator *allocator) +{ + // Some malloc() variants return NULL if called with size == 0. + if (size == 0) + size = 1; + + void *ptr; + + if (allocator != NULL && allocator->alloc != NULL) + ptr = allocator->alloc(allocator->opaque, 1, size); + else + ptr = malloc(size); + + return ptr; +} + + +extern void +lzma_free(void *ptr, lzma_allocator *allocator) +{ + if (allocator != NULL && allocator->free != NULL) + allocator->free(allocator->opaque, ptr); + else + free(ptr); + + return; +} + + +////////// +// Misc // +////////// + +extern size_t +lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size) +{ + const size_t in_avail = in_size - *in_pos; + const size_t out_avail = out_size - *out_pos; + const size_t copy_size = MIN(in_avail, out_avail); + + memcpy(out + *out_pos, in + *in_pos, copy_size); + + *in_pos += copy_size; + *out_pos += copy_size; + + return copy_size; +} + + +extern lzma_ret +lzma_next_filter_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + lzma_next_coder_init(filters[0].init, next, allocator); + next->id = filters[0].id; + return filters[0].init == NULL + ? LZMA_OK : filters[0].init(next, allocator, filters); +} + + +extern lzma_ret +lzma_next_filter_update(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *reversed_filters) +{ + // Check that the application isn't trying to change the Filter ID. + // End of filters is indicated with LZMA_VLI_UNKNOWN in both + // reversed_filters[0].id and next->id. + if (reversed_filters[0].id != next->id) + return LZMA_PROG_ERROR; + + if (reversed_filters[0].id == LZMA_VLI_UNKNOWN) + return LZMA_OK; + + assert(next->update != NULL); + return next->update(next->coder, allocator, NULL, reversed_filters); +} + + +extern void +lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator) +{ + if (next->init != (uintptr_t)(NULL)) { + // To avoid tiny end functions that simply call + // lzma_free(coder, allocator), we allow leaving next->end + // NULL and call lzma_free() here. + if (next->end != NULL) + next->end(next->coder, allocator); + else + lzma_free(next->coder, allocator); + + // Reset the variables so the we don't accidentally think + // that it is an already initialized coder. + *next = LZMA_NEXT_CODER_INIT; + } + + return; +} + + +////////////////////////////////////// +// External to internal API wrapper // +////////////////////////////////////// + +extern lzma_ret +lzma_strm_init(lzma_stream *strm) +{ + if (strm == NULL) + return LZMA_PROG_ERROR; + + if (strm->internal == NULL) { + strm->internal = lzma_alloc(sizeof(lzma_internal), + strm->allocator); + if (strm->internal == NULL) + return LZMA_MEM_ERROR; + + strm->internal->next = LZMA_NEXT_CODER_INIT; + } + + strm->internal->supported_actions[LZMA_RUN] = false; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = false; + strm->internal->supported_actions[LZMA_FULL_FLUSH] = false; + strm->internal->supported_actions[LZMA_FINISH] = false; + strm->internal->sequence = ISEQ_RUN; + strm->internal->allow_buf_error = false; + + strm->total_in = 0; + strm->total_out = 0; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_code(lzma_stream *strm, lzma_action action) +{ + // Sanity checks + if ((strm->next_in == NULL && strm->avail_in != 0) + || (strm->next_out == NULL && strm->avail_out != 0) + || strm->internal == NULL + || strm->internal->next.code == NULL + || (unsigned int)(action) > LZMA_FINISH + || !strm->internal->supported_actions[action]) + return LZMA_PROG_ERROR; + + switch (strm->internal->sequence) { + case ISEQ_RUN: + switch (action) { + case LZMA_RUN: + break; + + case LZMA_SYNC_FLUSH: + strm->internal->sequence = ISEQ_SYNC_FLUSH; + break; + + case LZMA_FULL_FLUSH: + strm->internal->sequence = ISEQ_FULL_FLUSH; + break; + + case LZMA_FINISH: + strm->internal->sequence = ISEQ_FINISH; + break; + } + + break; + + case ISEQ_SYNC_FLUSH: + // The same action must be used until we return + // LZMA_STREAM_END, and the amount of input must not change. + if (action != LZMA_SYNC_FLUSH + || strm->internal->avail_in != strm->avail_in) + return LZMA_PROG_ERROR; + + break; + + case ISEQ_FULL_FLUSH: + if (action != LZMA_FULL_FLUSH + || strm->internal->avail_in != strm->avail_in) + return LZMA_PROG_ERROR; + + break; + + case ISEQ_FINISH: + if (action != LZMA_FINISH + || strm->internal->avail_in != strm->avail_in) + return LZMA_PROG_ERROR; + + break; + + case ISEQ_END: + return LZMA_STREAM_END; + + case ISEQ_ERROR: + default: + return LZMA_PROG_ERROR; + } + + size_t in_pos = 0; + size_t out_pos = 0; + lzma_ret ret = strm->internal->next.code( + strm->internal->next.coder, strm->allocator, + strm->next_in, &in_pos, strm->avail_in, + strm->next_out, &out_pos, strm->avail_out, action); + + strm->next_in += in_pos; + strm->avail_in -= in_pos; + strm->total_in += in_pos; + + strm->next_out += out_pos; + strm->avail_out -= out_pos; + strm->total_out += out_pos; + + strm->internal->avail_in = strm->avail_in; + + switch (ret) { + case LZMA_OK: + // Don't return LZMA_BUF_ERROR when it happens the first time. + // This is to avoid returning LZMA_BUF_ERROR when avail_out + // was zero but still there was no more data left to written + // to next_out. + if (out_pos == 0 && in_pos == 0) { + if (strm->internal->allow_buf_error) + ret = LZMA_BUF_ERROR; + else + strm->internal->allow_buf_error = true; + } else { + strm->internal->allow_buf_error = false; + } + break; + + case LZMA_STREAM_END: + if (strm->internal->sequence == ISEQ_SYNC_FLUSH + || strm->internal->sequence == ISEQ_FULL_FLUSH) + strm->internal->sequence = ISEQ_RUN; + else + strm->internal->sequence = ISEQ_END; + + // Fall through + + case LZMA_NO_CHECK: + case LZMA_UNSUPPORTED_CHECK: + case LZMA_GET_CHECK: + case LZMA_MEMLIMIT_ERROR: + // Something else than LZMA_OK, but not a fatal error, + // that is, coding may be continued (except if ISEQ_END). + strm->internal->allow_buf_error = false; + break; + + default: + // All the other errors are fatal; coding cannot be continued. + assert(ret != LZMA_BUF_ERROR); + strm->internal->sequence = ISEQ_ERROR; + break; + } + + return ret; +} + + +extern LZMA_API(void) +lzma_end(lzma_stream *strm) +{ + if (strm != NULL && strm->internal != NULL) { + lzma_next_end(&strm->internal->next, strm->allocator); + lzma_free(strm->internal, strm->allocator); + strm->internal = NULL; + } + + return; +} + + +extern LZMA_API(lzma_check) +lzma_get_check(const lzma_stream *strm) +{ + // Return LZMA_CHECK_NONE if we cannot know the check type. + // It's a bug in the application if this happens. + if (strm->internal->next.get_check == NULL) + return LZMA_CHECK_NONE; + + return strm->internal->next.get_check(strm->internal->next.coder); +} + + +extern LZMA_API(uint64_t) +lzma_memusage(const lzma_stream *strm) +{ + uint64_t memusage; + uint64_t old_memlimit; + + if (strm == NULL || strm->internal == NULL + || strm->internal->next.memconfig == NULL + || strm->internal->next.memconfig( + strm->internal->next.coder, + &memusage, &old_memlimit, 0) != LZMA_OK) + return 0; + + return memusage; +} + + +extern LZMA_API(uint64_t) +lzma_memlimit_get(const lzma_stream *strm) +{ + uint64_t old_memlimit; + uint64_t memusage; + + if (strm == NULL || strm->internal == NULL + || strm->internal->next.memconfig == NULL + || strm->internal->next.memconfig( + strm->internal->next.coder, + &memusage, &old_memlimit, 0) != LZMA_OK) + return 0; + + return old_memlimit; +} + + +extern LZMA_API(lzma_ret) +lzma_memlimit_set(lzma_stream *strm, uint64_t new_memlimit) +{ + // Dummy variables to simplify memconfig functions + uint64_t old_memlimit; + uint64_t memusage; + + if (strm == NULL || strm->internal == NULL + || strm->internal->next.memconfig == NULL) + return LZMA_PROG_ERROR; + + if (new_memlimit != 0 && new_memlimit < LZMA_MEMUSAGE_BASE) + return LZMA_MEMLIMIT_ERROR; + + return strm->internal->next.memconfig(strm->internal->next.coder, + &memusage, &old_memlimit, new_memlimit); +} diff --git a/contrib/xz/src/liblzma/common/common.h b/contrib/xz/src/liblzma/common/common.h new file mode 100644 index 0000000..7b7fbb1 --- /dev/null +++ b/contrib/xz/src/liblzma/common/common.h @@ -0,0 +1,290 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file common.h +/// \brief Definitions common to the whole liblzma library +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_COMMON_H +#define LZMA_COMMON_H + +#include "sysdefs.h" +#include "mythread.h" +#include "tuklib_integer.h" + +#if defined(_WIN32) || defined(__CYGWIN__) +# ifdef DLL_EXPORT +# define LZMA_API_EXPORT __declspec(dllexport) +# else +# define LZMA_API_EXPORT +# endif +// Don't use ifdef or defined() below. +#elif HAVE_VISIBILITY +# define LZMA_API_EXPORT __attribute__((__visibility__("default"))) +#else +# define LZMA_API_EXPORT +#endif + +#define LZMA_API(type) LZMA_API_EXPORT type LZMA_API_CALL + +#include "lzma.h" + +// These allow helping the compiler in some often-executed branches, whose +// result is almost always the same. +#ifdef __GNUC__ +# define likely(expr) __builtin_expect(expr, true) +# define unlikely(expr) __builtin_expect(expr, false) +#else +# define likely(expr) (expr) +# define unlikely(expr) (expr) +#endif + + +/// Size of temporary buffers needed in some filters +#define LZMA_BUFFER_SIZE 4096 + + +/// Starting value for memory usage estimates. Instead of calculating size +/// of _every_ structure and taking into account malloc() overhead etc., we +/// add a base size to all memory usage estimates. It's not very accurate +/// but should be easily good enough. +#define LZMA_MEMUSAGE_BASE (UINT64_C(1) << 15) + +/// Start of internal Filter ID space. These IDs must never be used +/// in Streams. +#define LZMA_FILTER_RESERVED_START (LZMA_VLI_C(1) << 62) + + +/// Internal helper filter used by Subblock decoder. It is mapped to an +/// otherwise invalid Filter ID, which is impossible to get from any input +/// file (even if malicious file). +#define LZMA_FILTER_SUBBLOCK_HELPER LZMA_VLI_C(0x7000000000000001) + + +/// Supported flags that can be passed to lzma_stream_decoder() +/// or lzma_auto_decoder(). +#define LZMA_SUPPORTED_FLAGS \ + ( LZMA_TELL_NO_CHECK \ + | LZMA_TELL_UNSUPPORTED_CHECK \ + | LZMA_TELL_ANY_CHECK \ + | LZMA_CONCATENATED ) + + +/// Type of encoder/decoder specific data; the actual structure is defined +/// differently in different coders. +typedef struct lzma_coder_s lzma_coder; + +typedef struct lzma_next_coder_s lzma_next_coder; + +typedef struct lzma_filter_info_s lzma_filter_info; + + +/// Type of a function used to initialize a filter encoder or decoder +typedef lzma_ret (*lzma_init_function)( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter_info *filters); + +/// Type of a function to do some kind of coding work (filters, Stream, +/// Block encoders/decoders etc.). Some special coders use don't use both +/// input and output buffers, but for simplicity they still use this same +/// function prototype. +typedef lzma_ret (*lzma_code_function)( + lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, + lzma_action action); + +/// Type of a function to free the memory allocated for the coder +typedef void (*lzma_end_function)( + lzma_coder *coder, lzma_allocator *allocator); + + +/// Raw coder validates and converts an array of lzma_filter structures to +/// an array of lzma_filter_info structures. This array is used with +/// lzma_next_filter_init to initialize the filter chain. +struct lzma_filter_info_s { + /// Filter ID. This is used only by the encoder + /// with lzma_filters_update(). + lzma_vli id; + + /// Pointer to function used to initialize the filter. + /// This is NULL to indicate end of array. + lzma_init_function init; + + /// Pointer to filter's options structure + void *options; +}; + + +/// Hold data and function pointers of the next filter in the chain. +struct lzma_next_coder_s { + /// Pointer to coder-specific data + lzma_coder *coder; + + /// Filter ID. This is LZMA_VLI_UNKNOWN when this structure doesn't + /// point to a filter coder. + lzma_vli id; + + /// "Pointer" to init function. This is never called here. + /// We need only to detect if we are initializing a coder + /// that was allocated earlier. See lzma_next_coder_init and + /// lzma_next_strm_init macros in this file. + uintptr_t init; + + /// Pointer to function to do the actual coding + lzma_code_function code; + + /// Pointer to function to free lzma_next_coder.coder. This can + /// be NULL; in that case, lzma_free is called to free + /// lzma_next_coder.coder. + lzma_end_function end; + + /// Pointer to function to return the type of the integrity check. + /// Most coders won't support this. + lzma_check (*get_check)(const lzma_coder *coder); + + /// Pointer to function to get and/or change the memory usage limit. + /// If new_memlimit == 0, the limit is not changed. + lzma_ret (*memconfig)(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit); + + /// Update the filter-specific options or the whole filter chain + /// in the encoder. + lzma_ret (*update)(lzma_coder *coder, lzma_allocator *allocator, + const lzma_filter *filters, + const lzma_filter *reversed_filters); +}; + + +/// Macro to initialize lzma_next_coder structure +#define LZMA_NEXT_CODER_INIT \ + (lzma_next_coder){ \ + .coder = NULL, \ + .init = (uintptr_t)(NULL), \ + .id = LZMA_VLI_UNKNOWN, \ + .code = NULL, \ + .end = NULL, \ + .get_check = NULL, \ + .memconfig = NULL, \ + .update = NULL, \ + } + + +/// Internal data for lzma_strm_init, lzma_code, and lzma_end. A pointer to +/// this is stored in lzma_stream. +struct lzma_internal_s { + /// The actual coder that should do something useful + lzma_next_coder next; + + /// Track the state of the coder. This is used to validate arguments + /// so that the actual coders can rely on e.g. that LZMA_SYNC_FLUSH + /// is used on every call to lzma_code until next.code has returned + /// LZMA_STREAM_END. + enum { + ISEQ_RUN, + ISEQ_SYNC_FLUSH, + ISEQ_FULL_FLUSH, + ISEQ_FINISH, + ISEQ_END, + ISEQ_ERROR, + } sequence; + + /// A copy of lzma_stream avail_in. This is used to verify that the + /// amount of input doesn't change once e.g. LZMA_FINISH has been + /// used. + size_t avail_in; + + /// Indicates which lzma_action values are allowed by next.code. + bool supported_actions[4]; + + /// If true, lzma_code will return LZMA_BUF_ERROR if no progress was + /// made (no input consumed and no output produced by next.code). + bool allow_buf_error; +}; + + +/// Allocates memory +extern void *lzma_alloc(size_t size, lzma_allocator *allocator) + lzma_attribute((malloc)); + +/// Frees memory +extern void lzma_free(void *ptr, lzma_allocator *allocator); + + +/// Allocates strm->internal if it is NULL, and initializes *strm and +/// strm->internal. This function is only called via lzma_next_strm_init macro. +extern lzma_ret lzma_strm_init(lzma_stream *strm); + +/// Initializes the next filter in the chain, if any. This takes care of +/// freeing the memory of previously initialized filter if it is different +/// than the filter being initialized now. This way the actual filter +/// initialization functions don't need to use lzma_next_coder_init macro. +extern lzma_ret lzma_next_filter_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_filter_info *filters); + +/// Update the next filter in the chain, if any. This checks that +/// the application is not trying to change the Filter IDs. +extern lzma_ret lzma_next_filter_update( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *reversed_filters); + +/// Frees the memory allocated for next->coder either using next->end or, +/// if next->end is NULL, using lzma_free. +extern void lzma_next_end(lzma_next_coder *next, lzma_allocator *allocator); + + +/// Copy as much data as possible from in[] to out[] and update *in_pos +/// and *out_pos accordingly. Returns the number of bytes copied. +extern size_t lzma_bufcpy(const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size); + + +/// \brief Return if expression doesn't evaluate to LZMA_OK +/// +/// There are several situations where we want to return immediately +/// with the value of expr if it isn't LZMA_OK. This macro shortens +/// the code a little. +#define return_if_error(expr) \ +do { \ + const lzma_ret ret_ = (expr); \ + if (ret_ != LZMA_OK) \ + return ret_; \ +} while (0) + + +/// If next isn't already initialized, free the previous coder. Then mark +/// that next is _possibly_ initialized for the coder using this macro. +/// "Possibly" means that if e.g. allocation of next->coder fails, the +/// structure isn't actually initialized for this coder, but leaving +/// next->init to func is still OK. +#define lzma_next_coder_init(func, next, allocator) \ +do { \ + if ((uintptr_t)(func) != (next)->init) \ + lzma_next_end(next, allocator); \ + (next)->init = (uintptr_t)(func); \ +} while (0) + + +/// Initializes lzma_strm and calls func() to initialize strm->internal->next. +/// (The function being called will use lzma_next_coder_init()). If +/// initialization fails, memory that wasn't freed by func() is freed +/// along strm->internal. +#define lzma_next_strm_init(func, strm, ...) \ +do { \ + return_if_error(lzma_strm_init(strm)); \ + const lzma_ret ret_ = func(&(strm)->internal->next, \ + (strm)->allocator, __VA_ARGS__); \ + if (ret_ != LZMA_OK) { \ + lzma_end(strm); \ + return ret_; \ + } \ +} while (0) + +#endif diff --git a/contrib/xz/src/liblzma/common/easy_buffer_encoder.c b/contrib/xz/src/liblzma/common/easy_buffer_encoder.c new file mode 100644 index 0000000..c4be34c --- /dev/null +++ b/contrib/xz/src/liblzma/common/easy_buffer_encoder.c @@ -0,0 +1,27 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file easy_buffer_encoder.c +/// \brief Easy single-call .xz Stream encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "easy_preset.h" + + +extern LZMA_API(lzma_ret) +lzma_easy_buffer_encode(uint32_t preset, lzma_check check, + lzma_allocator *allocator, const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + lzma_options_easy opt_easy; + if (lzma_easy_preset(&opt_easy, preset)) + return LZMA_OPTIONS_ERROR; + + return lzma_stream_buffer_encode(opt_easy.filters, check, + allocator, in, in_size, out, out_pos, out_size); +} diff --git a/contrib/xz/src/liblzma/common/easy_decoder_memusage.c b/contrib/xz/src/liblzma/common/easy_decoder_memusage.c new file mode 100644 index 0000000..20bcd5b --- /dev/null +++ b/contrib/xz/src/liblzma/common/easy_decoder_memusage.c @@ -0,0 +1,24 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file easy_decoder_memusage.c +/// \brief Decoder memory usage calculation to match easy encoder presets +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "easy_preset.h" + + +extern LZMA_API(uint64_t) +lzma_easy_decoder_memusage(uint32_t preset) +{ + lzma_options_easy opt_easy; + if (lzma_easy_preset(&opt_easy, preset)) + return UINT32_MAX; + + return lzma_raw_decoder_memusage(opt_easy.filters); +} diff --git a/contrib/xz/src/liblzma/common/easy_encoder.c b/contrib/xz/src/liblzma/common/easy_encoder.c new file mode 100644 index 0000000..d13ccd7 --- /dev/null +++ b/contrib/xz/src/liblzma/common/easy_encoder.c @@ -0,0 +1,25 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file easy_encoder.c +/// \brief Easy .xz Stream encoder initialization +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "easy_preset.h" +#include "stream_encoder.h" + + +extern LZMA_API(lzma_ret) +lzma_easy_encoder(lzma_stream *strm, uint32_t preset, lzma_check check) +{ + lzma_options_easy opt_easy; + if (lzma_easy_preset(&opt_easy, preset)) + return LZMA_OPTIONS_ERROR; + + return lzma_stream_encoder(strm, opt_easy.filters, check); +} diff --git a/contrib/xz/src/liblzma/common/easy_encoder_memusage.c b/contrib/xz/src/liblzma/common/easy_encoder_memusage.c new file mode 100644 index 0000000..e910575 --- /dev/null +++ b/contrib/xz/src/liblzma/common/easy_encoder_memusage.c @@ -0,0 +1,24 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file easy_encoder_memusage.c +/// \brief Easy .xz Stream encoder memory usage calculation +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "easy_preset.h" + + +extern LZMA_API(uint64_t) +lzma_easy_encoder_memusage(uint32_t preset) +{ + lzma_options_easy opt_easy; + if (lzma_easy_preset(&opt_easy, preset)) + return UINT32_MAX; + + return lzma_raw_encoder_memusage(opt_easy.filters); +} diff --git a/contrib/xz/src/liblzma/common/easy_preset.c b/contrib/xz/src/liblzma/common/easy_preset.c new file mode 100644 index 0000000..2f98598 --- /dev/null +++ b/contrib/xz/src/liblzma/common/easy_preset.c @@ -0,0 +1,27 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file easy_preset.c +/// \brief Preset handling for easy encoder and decoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "easy_preset.h" + + +extern bool +lzma_easy_preset(lzma_options_easy *opt_easy, uint32_t preset) +{ + if (lzma_lzma_preset(&opt_easy->opt_lzma, preset)) + return true; + + opt_easy->filters[0].id = LZMA_FILTER_LZMA2; + opt_easy->filters[0].options = &opt_easy->opt_lzma; + opt_easy->filters[1].id = LZMA_VLI_UNKNOWN; + + return false; +} diff --git a/contrib/xz/src/liblzma/common/easy_preset.h b/contrib/xz/src/liblzma/common/easy_preset.h new file mode 100644 index 0000000..382ade8 --- /dev/null +++ b/contrib/xz/src/liblzma/common/easy_preset.h @@ -0,0 +1,32 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file easy_preset.h +/// \brief Preset handling for easy encoder and decoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +typedef struct { + /// We need to keep the filters array available in case + /// LZMA_FULL_FLUSH is used. + lzma_filter filters[LZMA_FILTERS_MAX + 1]; + + /// Options for LZMA2 + lzma_options_lzma opt_lzma; + + // Options for more filters can be added later, so this struct + // is not ready to be put into the public API. + +} lzma_options_easy; + + +/// Set *easy to the settings given by the preset. Returns true on error, +/// false on success. +extern bool lzma_easy_preset(lzma_options_easy *easy, uint32_t preset); diff --git a/contrib/xz/src/liblzma/common/filter_buffer_decoder.c b/contrib/xz/src/liblzma/common/filter_buffer_decoder.c new file mode 100644 index 0000000..2d35ef8 --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_buffer_decoder.c @@ -0,0 +1,87 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_buffer_decoder.c +/// \brief Single-call raw decoding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_decoder.h" + + +extern LZMA_API(lzma_ret) +lzma_raw_buffer_decode(const lzma_filter *filters, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // Validate what isn't validated later in filter_common.c. + if (in == NULL || in_pos == NULL || *in_pos > in_size || out == NULL + || out_pos == NULL || *out_pos > out_size) + return LZMA_PROG_ERROR; + + // Initialize the decoer. + lzma_next_coder next = LZMA_NEXT_CODER_INIT; + return_if_error(lzma_raw_decoder_init(&next, allocator, filters)); + + // Store the positions so that we can restore them if something + // goes wrong. + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; + + // Do the actual decoding and free decoder's memory. + lzma_ret ret = next.code(next.coder, allocator, in, in_pos, in_size, + out, out_pos, out_size, LZMA_FINISH); + + if (ret == LZMA_STREAM_END) { + ret = LZMA_OK; + } else { + if (ret == LZMA_OK) { + // Either the input was truncated or the + // output buffer was too small. + assert(*in_pos == in_size || *out_pos == out_size); + + if (*in_pos != in_size) { + // Since input wasn't consumed completely, + // the output buffer became full and is + // too small. + ret = LZMA_BUF_ERROR; + + } else if (*out_pos != out_size) { + // Since output didn't became full, the input + // has to be truncated. + ret = LZMA_DATA_ERROR; + + } else { + // All the input was consumed and output + // buffer is full. Now we don't immediately + // know the reason for the error. Try + // decoding one more byte. If it succeeds, + // then the output buffer was too small. If + // we cannot get a new output byte, the input + // is truncated. + uint8_t tmp[1]; + size_t tmp_pos = 0; + (void)next.code(next.coder, allocator, + in, in_pos, in_size, + tmp, &tmp_pos, 1, LZMA_FINISH); + + if (tmp_pos == 1) + ret = LZMA_BUF_ERROR; + else + ret = LZMA_DATA_ERROR; + } + } + + // Restore the positions. + *in_pos = in_start; + *out_pos = out_start; + } + + lzma_next_end(&next, allocator); + + return ret; +} diff --git a/contrib/xz/src/liblzma/common/filter_buffer_encoder.c b/contrib/xz/src/liblzma/common/filter_buffer_encoder.c new file mode 100644 index 0000000..646e1b3 --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_buffer_encoder.c @@ -0,0 +1,54 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_buffer_encoder.c +/// \brief Single-call raw encoding +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_encoder.h" + + +extern LZMA_API(lzma_ret) +lzma_raw_buffer_encode(const lzma_filter *filters, lzma_allocator *allocator, + const uint8_t *in, size_t in_size, uint8_t *out, + size_t *out_pos, size_t out_size) +{ + // Validate what isn't validated later in filter_common.c. + if ((in == NULL && in_size != 0) || out == NULL + || out_pos == NULL || *out_pos > out_size) + return LZMA_PROG_ERROR; + + // Initialize the encoder + lzma_next_coder next = LZMA_NEXT_CODER_INIT; + return_if_error(lzma_raw_encoder_init(&next, allocator, filters)); + + // Store the output position so that we can restore it if + // something goes wrong. + const size_t out_start = *out_pos; + + // Do the actual encoding and free coder's memory. + size_t in_pos = 0; + lzma_ret ret = next.code(next.coder, allocator, in, &in_pos, in_size, + out, out_pos, out_size, LZMA_FINISH); + lzma_next_end(&next, allocator); + + if (ret == LZMA_STREAM_END) { + ret = LZMA_OK; + } else { + if (ret == LZMA_OK) { + // Output buffer was too small. + assert(*out_pos == out_size); + ret = LZMA_BUF_ERROR; + } + + // Restore the output position. + *out_pos = out_start; + } + + return ret; +} diff --git a/contrib/xz/src/liblzma/common/filter_common.c b/contrib/xz/src/liblzma/common/filter_common.c new file mode 100644 index 0000000..2322d7d --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_common.c @@ -0,0 +1,346 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_common.c +/// \brief Filter-specific stuff common for both encoder and decoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_common.h" + + +static const struct { + /// Filter ID + lzma_vli id; + + /// Size of the filter-specific options structure + size_t options_size; + + /// True if it is OK to use this filter as non-last filter in + /// the chain. + bool non_last_ok; + + /// True if it is OK to use this filter as the last filter in + /// the chain. + bool last_ok; + + /// True if the filter may change the size of the data (that is, the + /// amount of encoded output can be different than the amount of + /// uncompressed input). + bool changes_size; + +} features[] = { +#if defined (HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) + { + .id = LZMA_FILTER_LZMA1, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, + }, +#endif +#ifdef HAVE_DECODER_LZMA2 + { + .id = LZMA_FILTER_LZMA2, + .options_size = sizeof(lzma_options_lzma), + .non_last_ok = false, + .last_ok = true, + .changes_size = true, + }, +#endif +#if defined(HAVE_ENCODER_SUBBLOCK) || defined(HAVE_DECODER_SUBBLOCK) + { + .id = LZMA_FILTER_SUBBLOCK, + .options_size = sizeof(lzma_options_subblock), + .non_last_ok = true, + .last_ok = true, + .changes_size = true, + }, +#endif +#ifdef HAVE_DECODER_X86 + { + .id = LZMA_FILTER_X86, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) + { + .id = LZMA_FILTER_POWERPC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#ifdef HAVE_DECODER_IA64 + { + .id = LZMA_FILTER_IA64, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_ARM) || defined(HAVE_DECODER_ARM) + { + .id = LZMA_FILTER_ARM, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_ARMTHUMB) || defined(HAVE_DECODER_ARMTHUMB) + { + .id = LZMA_FILTER_ARMTHUMB, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_SPARC) || defined(HAVE_DECODER_SPARC) + { + .id = LZMA_FILTER_SPARC, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif +#if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) + { + .id = LZMA_FILTER_DELTA, + .options_size = sizeof(lzma_options_delta), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif + { + .id = LZMA_VLI_UNKNOWN + } +}; + + +extern LZMA_API(lzma_ret) +lzma_filters_copy(const lzma_filter *src, lzma_filter *dest, + lzma_allocator *allocator) +{ + if (src == NULL || dest == NULL) + return LZMA_PROG_ERROR; + + lzma_ret ret; + size_t i; + for (i = 0; src[i].id != LZMA_VLI_UNKNOWN; ++i) { + // There must be a maximum of four filters plus + // the array terminator. + if (i == LZMA_FILTERS_MAX) { + ret = LZMA_OPTIONS_ERROR; + goto error; + } + + dest[i].id = src[i].id; + + if (src[i].options == NULL) { + dest[i].options = NULL; + } else { + // See if the filter is supported only when the + // options is not NULL. This might be convenient + // sometimes if the app is actually copying only + // a partial filter chain with a place holder ID. + // + // When options is not NULL, the Filter ID must be + // supported by us, because otherwise we don't know + // how big the options are. + size_t j; + for (j = 0; src[i].id != features[j].id; ++j) { + if (features[j].id == LZMA_VLI_UNKNOWN) { + ret = LZMA_OPTIONS_ERROR; + goto error; + } + } + + // Allocate and copy the options. + dest[i].options = lzma_alloc(features[j].options_size, + allocator); + if (dest[i].options == NULL) { + ret = LZMA_MEM_ERROR; + goto error; + } + + memcpy(dest[i].options, src[i].options, + features[j].options_size); + } + } + + // Terminate the filter array. + assert(i <= LZMA_FILTERS_MAX + 1); + dest[i].id = LZMA_VLI_UNKNOWN; + dest[i].options = NULL; + + return LZMA_OK; + +error: + // Free the options which we have already allocated. + while (i-- > 0) { + lzma_free(dest[i].options, allocator); + dest[i].options = NULL; + } + + return ret; +} + + +static lzma_ret +validate_chain(const lzma_filter *filters, size_t *count) +{ + // There must be at least one filter. + if (filters == NULL || filters[0].id == LZMA_VLI_UNKNOWN) + return LZMA_PROG_ERROR; + + // Number of non-last filters that may change the size of the data + // significantly (that is, more than 1-2 % or so). + size_t changes_size_count = 0; + + // True if it is OK to add a new filter after the current filter. + bool non_last_ok = true; + + // True if the last filter in the given chain is actually usable as + // the last filter. Only filters that support embedding End of Payload + // Marker can be used as the last filter in the chain. + bool last_ok = false; + + size_t i = 0; + do { + size_t j; + for (j = 0; filters[i].id != features[j].id; ++j) + if (features[j].id == LZMA_VLI_UNKNOWN) + return LZMA_OPTIONS_ERROR; + + // If the previous filter in the chain cannot be a non-last + // filter, the chain is invalid. + if (!non_last_ok) + return LZMA_OPTIONS_ERROR; + + non_last_ok = features[j].non_last_ok; + last_ok = features[j].last_ok; + changes_size_count += features[j].changes_size; + + } while (filters[++i].id != LZMA_VLI_UNKNOWN); + + // There must be 1-4 filters. The last filter must be usable as + // the last filter in the chain. A maximum of three filters are + // allowed to change the size of the data. + if (i > LZMA_FILTERS_MAX || !last_ok || changes_size_count > 3) + return LZMA_OPTIONS_ERROR; + + *count = i; + return LZMA_OK; +} + + +extern lzma_ret +lzma_raw_coder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options, + lzma_filter_find coder_find, bool is_encoder) +{ + // Do some basic validation and get the number of filters. + size_t count; + return_if_error(validate_chain(options, &count)); + + // Set the filter functions and copy the options pointer. + lzma_filter_info filters[LZMA_FILTERS_MAX + 1]; + if (is_encoder) { + for (size_t i = 0; i < count; ++i) { + // The order of the filters is reversed in the + // encoder. It allows more efficient handling + // of the uncompressed data. + const size_t j = count - i - 1; + + const lzma_filter_coder *const fc + = coder_find(options[i].id); + if (fc == NULL || fc->init == NULL) + return LZMA_OPTIONS_ERROR; + + filters[j].id = options[i].id; + filters[j].init = fc->init; + filters[j].options = options[i].options; + } + } else { + for (size_t i = 0; i < count; ++i) { + const lzma_filter_coder *const fc + = coder_find(options[i].id); + if (fc == NULL || fc->init == NULL) + return LZMA_OPTIONS_ERROR; + + filters[i].id = options[i].id; + filters[i].init = fc->init; + filters[i].options = options[i].options; + } + } + + // Terminate the array. + filters[count].id = LZMA_VLI_UNKNOWN; + filters[count].init = NULL; + + // Initialize the filters. + const lzma_ret ret = lzma_next_filter_init(next, allocator, filters); + if (ret != LZMA_OK) + lzma_next_end(next, allocator); + + return ret; +} + + +extern uint64_t +lzma_raw_coder_memusage(lzma_filter_find coder_find, + const lzma_filter *filters) +{ + // The chain has to have at least one filter. + { + size_t tmp; + if (validate_chain(filters, &tmp) != LZMA_OK) + return UINT64_MAX; + } + + uint64_t total = 0; + size_t i = 0; + + do { + const lzma_filter_coder *const fc + = coder_find(filters[i].id); + if (fc == NULL) + return UINT64_MAX; // Unsupported Filter ID + + if (fc->memusage == NULL) { + // This filter doesn't have a function to calculate + // the memory usage and validate the options. Such + // filters need only little memory, so we use 1 KiB + // as a good estimate. They also accept all possible + // options, so there's no need to worry about lack + // of validation. + total += 1024; + } else { + // Call the filter-specific memory usage calculation + // function. + const uint64_t usage + = fc->memusage(filters[i].options); + if (usage == UINT64_MAX) + return UINT64_MAX; // Invalid options + + total += usage; + } + } while (filters[++i].id != LZMA_VLI_UNKNOWN); + + // Add some fixed amount of extra. It's to compensate memory usage + // of Stream, Block etc. coders, malloc() overhead, stack etc. + return total + LZMA_MEMUSAGE_BASE; +} diff --git a/contrib/xz/src/liblzma/common/filter_common.h b/contrib/xz/src/liblzma/common/filter_common.h new file mode 100644 index 0000000..cd61fc0 --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_common.h @@ -0,0 +1,48 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_common.c +/// \brief Filter-specific stuff common for both encoder and decoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_FILTER_COMMON_H +#define LZMA_FILTER_COMMON_H + +#include "common.h" + + +/// Both lzma_filter_encoder and lzma_filter_decoder begin with these members. +typedef struct { + /// Filter ID + lzma_vli id; + + /// Initializes the filter encoder and calls lzma_next_filter_init() + /// for filters + 1. + lzma_init_function init; + + /// Calculates memory usage of the encoder. If the options are + /// invalid, UINT64_MAX is returned. + uint64_t (*memusage)(const void *options); + +} lzma_filter_coder; + + +typedef const lzma_filter_coder *(*lzma_filter_find)(lzma_vli id); + + +extern lzma_ret lzma_raw_coder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *filters, + lzma_filter_find coder_find, bool is_encoder); + + +extern uint64_t lzma_raw_coder_memusage(lzma_filter_find coder_find, + const lzma_filter *filters); + + +#endif diff --git a/contrib/xz/src/liblzma/common/filter_decoder.c b/contrib/xz/src/liblzma/common/filter_decoder.c new file mode 100644 index 0000000..95f77b7 --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_decoder.c @@ -0,0 +1,199 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_decoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_decoder.h" +#include "filter_common.h" +#include "lzma_decoder.h" +#include "lzma2_decoder.h" +#include "subblock_decoder.h" +#include "subblock_decoder_helper.h" +#include "simple_decoder.h" +#include "delta_decoder.h" + + +typedef struct { + /// Filter ID + lzma_vli id; + + /// Initializes the filter encoder and calls lzma_next_filter_init() + /// for filters + 1. + lzma_init_function init; + + /// Calculates memory usage of the encoder. If the options are + /// invalid, UINT64_MAX is returned. + uint64_t (*memusage)(const void *options); + + /// Decodes Filter Properties. + /// + /// \return - LZMA_OK: Properties decoded successfully. + /// - LZMA_OPTIONS_ERROR: Unsupported properties + /// - LZMA_MEM_ERROR: Memory allocation failed. + lzma_ret (*props_decode)(void **options, lzma_allocator *allocator, + const uint8_t *props, size_t props_size); + +} lzma_filter_decoder; + + +static const lzma_filter_decoder decoders[] = { +#ifdef HAVE_DECODER_LZMA1 + { + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_decoder_init, + .memusage = &lzma_lzma_decoder_memusage, + .props_decode = &lzma_lzma_props_decode, + }, +#endif +#ifdef HAVE_DECODER_LZMA2 + { + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_decoder_init, + .memusage = &lzma_lzma2_decoder_memusage, + .props_decode = &lzma_lzma2_props_decode, + }, +#endif +#ifdef HAVE_DECODER_SUBBLOCK + { + .id = LZMA_FILTER_SUBBLOCK, + .init = &lzma_subblock_decoder_init, +// .memusage = &lzma_subblock_decoder_memusage, + .props_decode = NULL, + }, + { + .id = LZMA_FILTER_SUBBLOCK_HELPER, + .init = &lzma_subblock_decoder_helper_init, + .memusage = NULL, + .props_decode = NULL, + }, +#endif +#ifdef HAVE_DECODER_X86 + { + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_POWERPC + { + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_IA64 + { + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_ARM + { + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_ARMTHUMB + { + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_SPARC + { + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif +#ifdef HAVE_DECODER_DELTA + { + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_decoder_init, + .memusage = &lzma_delta_coder_memusage, + .props_decode = &lzma_delta_props_decode, + }, +#endif +}; + + +static const lzma_filter_decoder * +decoder_find(lzma_vli id) +{ + for (size_t i = 0; i < ARRAY_SIZE(decoders); ++i) + if (decoders[i].id == id) + return decoders + i; + + return NULL; +} + + +extern LZMA_API(lzma_bool) +lzma_filter_decoder_is_supported(lzma_vli id) +{ + return decoder_find(id) != NULL; +} + + +extern lzma_ret +lzma_raw_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options) +{ + return lzma_raw_coder_init(next, allocator, + options, (lzma_filter_find)(&decoder_find), false); +} + + +extern LZMA_API(lzma_ret) +lzma_raw_decoder(lzma_stream *strm, const lzma_filter *options) +{ + lzma_next_strm_init(lzma_raw_decoder_init, strm, options); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} + + +extern LZMA_API(uint64_t) +lzma_raw_decoder_memusage(const lzma_filter *filters) +{ + return lzma_raw_coder_memusage( + (lzma_filter_find)(&decoder_find), filters); +} + + +extern LZMA_API(lzma_ret) +lzma_properties_decode(lzma_filter *filter, lzma_allocator *allocator, + const uint8_t *props, size_t props_size) +{ + // Make it always NULL so that the caller can always safely free() it. + filter->options = NULL; + + const lzma_filter_decoder *const fd = decoder_find(filter->id); + if (fd == NULL) + return LZMA_OPTIONS_ERROR; + + if (fd->props_decode == NULL) + return props_size == 0 ? LZMA_OK : LZMA_OPTIONS_ERROR; + + return fd->props_decode( + &filter->options, allocator, props, props_size); +} diff --git a/contrib/xz/src/liblzma/common/filter_decoder.h b/contrib/xz/src/liblzma/common/filter_decoder.h new file mode 100644 index 0000000..d5c68bd --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_decoder.h @@ -0,0 +1,23 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_decoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_FILTER_DECODER_H +#define LZMA_FILTER_DECODER_H + +#include "common.h" + + +extern lzma_ret lzma_raw_decoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options); + +#endif diff --git a/contrib/xz/src/liblzma/common/filter_encoder.c b/contrib/xz/src/liblzma/common/filter_encoder.c new file mode 100644 index 0000000..ab3d3af --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_encoder.c @@ -0,0 +1,298 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_decoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_encoder.h" +#include "filter_common.h" +#include "lzma_encoder.h" +#include "lzma2_encoder.h" +#include "subblock_encoder.h" +#include "simple_encoder.h" +#include "delta_encoder.h" + + +typedef struct { + /// Filter ID + lzma_vli id; + + /// Initializes the filter encoder and calls lzma_next_filter_init() + /// for filters + 1. + lzma_init_function init; + + /// Calculates memory usage of the encoder. If the options are + /// invalid, UINT64_MAX is returned. + uint64_t (*memusage)(const void *options); + + /// Calculates the minimum sane size for Blocks (or other types of + /// chunks) to which the input data can be split to make + /// multithreaded encoding possible. If this is NULL, it is assumed + /// that the encoder is fast enough with single thread. + lzma_vli (*chunk_size)(const void *options); + + /// Tells the size of the Filter Properties field. If options are + /// invalid, UINT32_MAX is returned. If this is NULL, props_size_fixed + /// is used. + lzma_ret (*props_size_get)(uint32_t *size, const void *options); + uint32_t props_size_fixed; + + /// Encodes Filter Properties. + /// + /// \return - LZMA_OK: Properties encoded successfully. + /// - LZMA_OPTIONS_ERROR: Unsupported options + /// - LZMA_PROG_ERROR: Invalid options or not enough + /// output space + lzma_ret (*props_encode)(const void *options, uint8_t *out); + +} lzma_filter_encoder; + + +static const lzma_filter_encoder encoders[] = { +#ifdef HAVE_ENCODER_LZMA1 + { + .id = LZMA_FILTER_LZMA1, + .init = &lzma_lzma_encoder_init, + .memusage = &lzma_lzma_encoder_memusage, + .chunk_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 5, + .props_encode = &lzma_lzma_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_LZMA2 + { + .id = LZMA_FILTER_LZMA2, + .init = &lzma_lzma2_encoder_init, + .memusage = &lzma_lzma2_encoder_memusage, + .chunk_size = NULL, // FIXME + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_lzma2_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_SUBBLOCK + { + .id = LZMA_FILTER_SUBBLOCK, + .init = &lzma_subblock_encoder_init, +// .memusage = &lzma_subblock_encoder_memusage, + .chunk_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 0, + .props_encode = NULL, + }, +#endif +#ifdef HAVE_ENCODER_X86 + { + .id = LZMA_FILTER_X86, + .init = &lzma_simple_x86_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_POWERPC + { + .id = LZMA_FILTER_POWERPC, + .init = &lzma_simple_powerpc_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_IA64 + { + .id = LZMA_FILTER_IA64, + .init = &lzma_simple_ia64_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_ARM + { + .id = LZMA_FILTER_ARM, + .init = &lzma_simple_arm_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_ARMTHUMB + { + .id = LZMA_FILTER_ARMTHUMB, + .init = &lzma_simple_armthumb_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_SPARC + { + .id = LZMA_FILTER_SPARC, + .init = &lzma_simple_sparc_encoder_init, + .memusage = NULL, + .chunk_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif +#ifdef HAVE_ENCODER_DELTA + { + .id = LZMA_FILTER_DELTA, + .init = &lzma_delta_encoder_init, + .memusage = &lzma_delta_coder_memusage, + .chunk_size = NULL, + .props_size_get = NULL, + .props_size_fixed = 1, + .props_encode = &lzma_delta_props_encode, + }, +#endif +}; + + +static const lzma_filter_encoder * +encoder_find(lzma_vli id) +{ + for (size_t i = 0; i < ARRAY_SIZE(encoders); ++i) + if (encoders[i].id == id) + return encoders + i; + + return NULL; +} + + +extern LZMA_API(lzma_bool) +lzma_filter_encoder_is_supported(lzma_vli id) +{ + return encoder_find(id) != NULL; +} + + +extern LZMA_API(lzma_ret) +lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) +{ + if (strm->internal->next.update == NULL) + return LZMA_PROG_ERROR; + + // Validate the filter chain. + if (lzma_raw_encoder_memusage(filters) == UINT64_MAX) + return LZMA_OPTIONS_ERROR; + + // The actual filter chain in the encoder is reversed. Some things + // still want the normal order chain, so we provide both. + size_t count = 1; + while (filters[count].id != LZMA_VLI_UNKNOWN) + ++count; + + lzma_filter reversed_filters[LZMA_FILTERS_MAX + 1]; + for (size_t i = 0; i < count; ++i) + reversed_filters[count - i - 1] = filters[i]; + + reversed_filters[count].id = LZMA_VLI_UNKNOWN; + + return strm->internal->next.update(strm->internal->next.coder, + strm->allocator, filters, reversed_filters); +} + + +extern lzma_ret +lzma_raw_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *options) +{ + return lzma_raw_coder_init(next, allocator, + options, (lzma_filter_find)(&encoder_find), true); +} + + +extern LZMA_API(lzma_ret) +lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) +{ + lzma_next_strm_init(lzma_raw_coder_init, strm, options, + (lzma_filter_find)(&encoder_find), true); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} + + +extern LZMA_API(uint64_t) +lzma_raw_encoder_memusage(const lzma_filter *filters) +{ + return lzma_raw_coder_memusage( + (lzma_filter_find)(&encoder_find), filters); +} + + +extern LZMA_API(lzma_vli) +lzma_chunk_size(const lzma_filter *filters) +{ + lzma_vli max = 0; + + for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { + const lzma_filter_encoder *const fe + = encoder_find(filters[i].id); + if (fe->chunk_size != NULL) { + const lzma_vli size + = fe->chunk_size(filters[i].options); + if (size == LZMA_VLI_UNKNOWN) + return LZMA_VLI_UNKNOWN; + + if (size > max) + max = size; + } + } + + return max; +} + + +extern LZMA_API(lzma_ret) +lzma_properties_size(uint32_t *size, const lzma_filter *filter) +{ + const lzma_filter_encoder *const fe = encoder_find(filter->id); + if (fe == NULL) { + // Unknown filter - if the Filter ID is a proper VLI, + // return LZMA_OPTIONS_ERROR instead of LZMA_PROG_ERROR, + // because it's possible that we just don't have support + // compiled in for the requested filter. + return filter->id <= LZMA_VLI_MAX + ? LZMA_OPTIONS_ERROR : LZMA_PROG_ERROR; + } + + if (fe->props_size_get == NULL) { + // No props_size_get() function, use props_size_fixed. + *size = fe->props_size_fixed; + return LZMA_OK; + } + + return fe->props_size_get(size, filter->options); +} + + +extern LZMA_API(lzma_ret) +lzma_properties_encode(const lzma_filter *filter, uint8_t *props) +{ + const lzma_filter_encoder *const fe = encoder_find(filter->id); + if (fe == NULL) + return LZMA_PROG_ERROR; + + if (fe->props_encode == NULL) + return LZMA_OK; + + return fe->props_encode(filter->options, props); +} diff --git a/contrib/xz/src/liblzma/common/filter_encoder.h b/contrib/xz/src/liblzma/common/filter_encoder.h new file mode 100644 index 0000000..a978932 --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_encoder.h @@ -0,0 +1,27 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_encoder.c +/// \brief Filter ID mapping to filter-specific functions +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_FILTER_ENCODER_H +#define LZMA_FILTER_ENCODER_H + +#include "common.h" + + +// FIXME !!! Public API +extern lzma_vli lzma_chunk_size(const lzma_filter *filters); + + +extern lzma_ret lzma_raw_encoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *filters); + +#endif diff --git a/contrib/xz/src/liblzma/common/filter_flags_decoder.c b/contrib/xz/src/liblzma/common/filter_flags_decoder.c new file mode 100644 index 0000000..caae10c --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_flags_decoder.c @@ -0,0 +1,46 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_flags_decoder.c +/// \brief Decodes a Filter Flags field +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_decoder.h" + + +extern LZMA_API(lzma_ret) +lzma_filter_flags_decode( + lzma_filter *filter, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size) +{ + // Set the pointer to NULL so the caller can always safely free it. + filter->options = NULL; + + // Filter ID + return_if_error(lzma_vli_decode(&filter->id, NULL, + in, in_pos, in_size)); + + if (filter->id >= LZMA_FILTER_RESERVED_START) + return LZMA_DATA_ERROR; + + // Size of Properties + lzma_vli props_size; + return_if_error(lzma_vli_decode(&props_size, NULL, + in, in_pos, in_size)); + + // Filter Properties + if (in_size - *in_pos < props_size) + return LZMA_DATA_ERROR; + + const lzma_ret ret = lzma_properties_decode( + filter, allocator, in + *in_pos, props_size); + + *in_pos += props_size; + + return ret; +} diff --git a/contrib/xz/src/liblzma/common/filter_flags_encoder.c b/contrib/xz/src/liblzma/common/filter_flags_encoder.c new file mode 100644 index 0000000..d110566 --- /dev/null +++ b/contrib/xz/src/liblzma/common/filter_flags_encoder.c @@ -0,0 +1,56 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file filter_flags_encoder.c +/// \brief Decodes a Filter Flags field +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "filter_encoder.h" + + +extern LZMA_API(lzma_ret) +lzma_filter_flags_size(uint32_t *size, const lzma_filter *filter) +{ + if (filter->id >= LZMA_FILTER_RESERVED_START) + return LZMA_PROG_ERROR; + + return_if_error(lzma_properties_size(size, filter)); + + *size += lzma_vli_size(filter->id) + lzma_vli_size(*size); + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_filter_flags_encode(const lzma_filter *filter, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // Filter ID + if (filter->id >= LZMA_FILTER_RESERVED_START) + return LZMA_PROG_ERROR; + + return_if_error(lzma_vli_encode(filter->id, NULL, + out, out_pos, out_size)); + + // Size of Properties + uint32_t props_size; + return_if_error(lzma_properties_size(&props_size, filter)); + return_if_error(lzma_vli_encode(props_size, NULL, + out, out_pos, out_size)); + + // Filter Properties + if (out_size - *out_pos < props_size) + return LZMA_PROG_ERROR; + + return_if_error(lzma_properties_encode(filter, out + *out_pos)); + + *out_pos += props_size; + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/hardware_physmem.c b/contrib/xz/src/liblzma/common/hardware_physmem.c new file mode 100644 index 0000000..7405b65 --- /dev/null +++ b/contrib/xz/src/liblzma/common/hardware_physmem.c @@ -0,0 +1,25 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file hardware_physmem.c +/// \brief Get the total amount of physical memory (RAM) +// +// Author: Jonathan Nieder +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + +#include "tuklib_physmem.h" + + +extern LZMA_API(uint64_t) +lzma_physmem(void) +{ + // It is simpler to make lzma_physmem() a wrapper for + // tuklib_physmem() than to hack appropriate symbol visiblity + // support for the tuklib modules. + return tuklib_physmem(); +} diff --git a/contrib/xz/src/liblzma/common/index.c b/contrib/xz/src/liblzma/common/index.c new file mode 100644 index 0000000..3941e28 --- /dev/null +++ b/contrib/xz/src/liblzma/common/index.c @@ -0,0 +1,1241 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file index.c +/// \brief Handling of .xz Indexes and some other Stream information +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "index.h" +#include "stream_flags_common.h" + + +/// \brief How many Records to allocate at once +/// +/// This should be big enough to avoid making lots of tiny allocations +/// but small enough to avoid too much unused memory at once. +#define INDEX_GROUP_SIZE 500 + + +/// \brief How many Records can be allocated at once at maximum +#define PREALLOC_MAX ((SIZE_MAX - sizeof(index_group)) / sizeof(index_record)) + + +/// \brief Base structure for index_stream and index_group structures +typedef struct index_tree_node_s index_tree_node; +struct index_tree_node_s { + /// Uncompressed start offset of this Stream (relative to the + /// beginning of the file) or Block (relative to the beginning + /// of the Stream) + lzma_vli uncompressed_base; + + /// Compressed start offset of this Stream or Block + lzma_vli compressed_base; + + index_tree_node *parent; + index_tree_node *left; + index_tree_node *right; +}; + + +/// \brief AVL tree to hold index_stream or index_group structures +typedef struct { + /// Root node + index_tree_node *root; + + /// Leftmost node. Since the tree will be filled sequentially, + /// this won't change after the first node has been added to + /// the tree. + index_tree_node *leftmost; + + /// The rightmost node in the tree. Since the tree is filled + /// sequentially, this is always the node where to add the new data. + index_tree_node *rightmost; + + /// Number of nodes in the tree + uint32_t count; + +} index_tree; + + +typedef struct { + lzma_vli uncompressed_sum; + lzma_vli unpadded_sum; +} index_record; + + +typedef struct { + /// Every Record group is part of index_stream.groups tree. + index_tree_node node; + + /// Number of Blocks in this Stream before this group. + lzma_vli number_base; + + /// Number of Records that can be put in records[]. + size_t allocated; + + /// Index of the last Record in use. + size_t last; + + /// The sizes in this array are stored as cumulative sums relative + /// to the beginning of the Stream. This makes it possible to + /// use binary search in lzma_index_locate(). + /// + /// Note that the cumulative summing is done specially for + /// unpadded_sum: The previous value is rounded up to the next + /// multiple of four before adding the Unpadded Size of the new + /// Block. The total encoded size of the Blocks in the Stream + /// is records[last].unpadded_sum in the last Record group of + /// the Stream. + /// + /// For example, if the Unpadded Sizes are 39, 57, and 81, the + /// stored values are 39, 97 (40 + 57), and 181 (100 + 181). + /// The total encoded size of these Blocks is 184. + /// + /// This is a flexible array, because it makes easy to optimize + /// memory usage in case someone concatenates many Streams that + /// have only one or few Blocks. + index_record records[]; + +} index_group; + + +typedef struct { + /// Every index_stream is a node in the tree of Sreams. + index_tree_node node; + + /// Number of this Stream (first one is 1) + uint32_t number; + + /// Total number of Blocks before this Stream + lzma_vli block_number_base; + + /// Record groups of this Stream are stored in a tree. + /// It's a T-tree with AVL-tree balancing. There are + /// INDEX_GROUP_SIZE Records per node by default. + /// This keeps the number of memory allocations reasonable + /// and finding a Record is fast. + index_tree groups; + + /// Number of Records in this Stream + lzma_vli record_count; + + /// Size of the List of Records field in this Stream. This is used + /// together with record_count to calculate the size of the Index + /// field and thus the total size of the Stream. + lzma_vli index_list_size; + + /// Stream Flags of this Stream. This is meaningful only if + /// the Stream Flags have been told us with lzma_index_stream_flags(). + /// Initially stream_flags.version is set to UINT32_MAX to indicate + /// that the Stream Flags are unknown. + lzma_stream_flags stream_flags; + + /// Amount of Stream Padding after this Stream. This defaults to + /// zero and can be set with lzma_index_stream_padding(). + lzma_vli stream_padding; + +} index_stream; + + +struct lzma_index_s { + /// AVL-tree containing the Stream(s). Often there is just one + /// Stream, but using a tree keeps lookups fast even when there + /// are many concatenated Streams. + index_tree streams; + + /// Uncompressed size of all the Blocks in the Stream(s) + lzma_vli uncompressed_size; + + /// Total size of all the Blocks in the Stream(s) + lzma_vli total_size; + + /// Total number of Records in all Streams in this lzma_index + lzma_vli record_count; + + /// Size of the List of Records field if all the Streams in this + /// lzma_index were packed into a single Stream (makes it simpler to + /// take many .xz files and combine them into a single Stream). + /// + /// This value together with record_count is needed to calculate + /// Backward Size that is stored into Stream Footer. + lzma_vli index_list_size; + + /// How many Records to allocate at once in lzma_index_append(). + /// This defaults to INDEX_GROUP_SIZE but can be overriden with + /// lzma_index_prealloc(). + size_t prealloc; + + /// Bitmask indicating what integrity check types have been used + /// as set by lzma_index_stream_flags(). The bit of the last Stream + /// is not included here, since it is possible to change it by + /// calling lzma_index_stream_flags() again. + uint32_t checks; +}; + + +static void +index_tree_init(index_tree *tree) +{ + tree->root = NULL; + tree->leftmost = NULL; + tree->rightmost = NULL; + tree->count = 0; + return; +} + + +/// Helper for index_tree_end() +static void +index_tree_node_end(index_tree_node *node, lzma_allocator *allocator, + void (*free_func)(void *node, lzma_allocator *allocator)) +{ + // The tree won't ever be very huge, so recursion should be fine. + // 20 levels in the tree is likely quite a lot already in practice. + if (node->left != NULL) + index_tree_node_end(node->left, allocator, free_func); + + if (node->right != NULL) + index_tree_node_end(node->right, allocator, free_func); + + if (free_func != NULL) + free_func(node, allocator); + + lzma_free(node, allocator); + return; +} + + +/// Free the meory allocated for a tree. If free_func is not NULL, +/// it is called on each node before freeing the node. This is used +/// to free the Record groups from each index_stream before freeing +/// the index_stream itself. +static void +index_tree_end(index_tree *tree, lzma_allocator *allocator, + void (*free_func)(void *node, lzma_allocator *allocator)) +{ + if (tree->root != NULL) + index_tree_node_end(tree->root, allocator, free_func); + + return; +} + + +/// Add a new node to the tree. node->uncompressed_base and +/// node->compressed_base must have been set by the caller already. +static void +index_tree_append(index_tree *tree, index_tree_node *node) +{ + node->parent = tree->rightmost; + node->left = NULL; + node->right = NULL; + + ++tree->count; + + // Handle the special case of adding the first node. + if (tree->root == NULL) { + tree->root = node; + tree->leftmost = node; + tree->rightmost = node; + return; + } + + // The tree is always filled sequentially. + assert(tree->rightmost->uncompressed_base <= node->uncompressed_base); + assert(tree->rightmost->compressed_base < node->compressed_base); + + // Add the new node after the rightmost node. It's the correct + // place due to the reason above. + tree->rightmost->right = node; + tree->rightmost = node; + + // Balance the AVL-tree if needed. We don't need to keep the balance + // factors in nodes, because we always fill the tree sequentially, + // and thus know the state of the tree just by looking at the node + // count. From the node count we can calculate how many steps to go + // up in the tree to find the rotation root. + uint32_t up = tree->count ^ (UINT32_C(1) << bsr32(tree->count)); + if (up != 0) { + // Locate the root node for the rotation. + up = ctz32(tree->count) + 2; + do { + node = node->parent; + } while (--up > 0); + + // Rotate left using node as the rotation root. + index_tree_node *pivot = node->right; + + if (node->parent == NULL) { + tree->root = pivot; + } else { + assert(node->parent->right == node); + node->parent->right = pivot; + } + + pivot->parent = node->parent; + + node->right = pivot->left; + if (node->right != NULL) + node->right->parent = node; + + pivot->left = node; + node->parent = pivot; + } + + return; +} + + +/// Get the next node in the tree. Return NULL if there are no more nodes. +static void * +index_tree_next(const index_tree_node *node) +{ + if (node->right != NULL) { + node = node->right; + while (node->left != NULL) + node = node->left; + + return (void *)(node); + } + + while (node->parent != NULL && node->parent->right == node) + node = node->parent; + + return (void *)(node->parent); +} + + +/// Locate a node that contains the given uncompressed offset. It is +/// caller's job to check that target is not bigger than the uncompressed +/// size of the tree (the last node would be returned in that case still). +static void * +index_tree_locate(const index_tree *tree, lzma_vli target) +{ + const index_tree_node *result = NULL; + const index_tree_node *node = tree->root; + + assert(tree->leftmost == NULL + || tree->leftmost->uncompressed_base == 0); + + // Consecutive nodes may have the same uncompressed_base. + // We must pick the rightmost one. + while (node != NULL) { + if (node->uncompressed_base > target) { + node = node->left; + } else { + result = node; + node = node->right; + } + } + + return (void *)(result); +} + + +/// Allocate and initialize a new Stream using the given base offsets. +static index_stream * +index_stream_init(lzma_vli compressed_base, lzma_vli uncompressed_base, + lzma_vli stream_number, lzma_vli block_number_base, + lzma_allocator *allocator) +{ + index_stream *s = lzma_alloc(sizeof(index_stream), allocator); + if (s == NULL) + return NULL; + + s->node.uncompressed_base = uncompressed_base; + s->node.compressed_base = compressed_base; + s->node.parent = NULL; + s->node.left = NULL; + s->node.right = NULL; + + s->number = stream_number; + s->block_number_base = block_number_base; + + index_tree_init(&s->groups); + + s->record_count = 0; + s->index_list_size = 0; + s->stream_flags.version = UINT32_MAX; + s->stream_padding = 0; + + return s; +} + + +/// Free the memory allocated for a Stream and its Record groups. +static void +index_stream_end(void *node, lzma_allocator *allocator) +{ + index_stream *s = node; + index_tree_end(&s->groups, allocator, NULL); + return; +} + + +static lzma_index * +index_init_plain(lzma_allocator *allocator) +{ + lzma_index *i = lzma_alloc(sizeof(lzma_index), allocator); + if (i != NULL) { + index_tree_init(&i->streams); + i->uncompressed_size = 0; + i->total_size = 0; + i->record_count = 0; + i->index_list_size = 0; + i->prealloc = INDEX_GROUP_SIZE; + i->checks = 0; + } + + return i; +} + + +extern LZMA_API(lzma_index *) +lzma_index_init(lzma_allocator *allocator) +{ + lzma_index *i = index_init_plain(allocator); + index_stream *s = index_stream_init(0, 0, 1, 0, allocator); + if (i == NULL || s == NULL) { + index_stream_end(s, allocator); + lzma_free(i, allocator); + } + + index_tree_append(&i->streams, &s->node); + + return i; +} + + +extern LZMA_API(void) +lzma_index_end(lzma_index *i, lzma_allocator *allocator) +{ + // NOTE: If you modify this function, check also the bottom + // of lzma_index_cat(). + if (i != NULL) { + index_tree_end(&i->streams, allocator, &index_stream_end); + lzma_free(i, allocator); + } + + return; +} + + +extern void +lzma_index_prealloc(lzma_index *i, lzma_vli records) +{ + if (records > PREALLOC_MAX) + records = PREALLOC_MAX; + + i->prealloc = (size_t)(records); + return; +} + + +extern LZMA_API(uint64_t) +lzma_index_memusage(lzma_vli streams, lzma_vli blocks) +{ + // This calculates an upper bound that is only a little bit + // bigger than the exact maximum memory usage with the given + // parameters. + + // Typical malloc() overhead is 2 * sizeof(void *) but we take + // a little bit extra just in case. Using LZMA_MEMUSAGE_BASE + // instead would give too inaccurate estimate. + const size_t alloc_overhead = 4 * sizeof(void *); + + // Amount of memory needed for each Stream base structures. + // We assume that every Stream has at least one Block and + // thus at least one group. + const size_t stream_base = sizeof(index_stream) + + sizeof(index_group) + 2 * alloc_overhead; + + // Amount of memory needed per group. + const size_t group_base = sizeof(index_group) + + INDEX_GROUP_SIZE * sizeof(index_record) + + alloc_overhead; + + // Number of groups. There may actually be more, but that overhead + // has been taken into account in stream_base already. + const lzma_vli groups + = (blocks + INDEX_GROUP_SIZE - 1) / INDEX_GROUP_SIZE; + + // Memory used by index_stream and index_group structures. + const uint64_t streams_mem = streams * stream_base; + const uint64_t groups_mem = groups * group_base; + + // Memory used by the base structure. + const uint64_t index_base = sizeof(lzma_index) + alloc_overhead; + + // Validate the arguments and catch integer overflows. + // Maximum number of Streams is "only" UINT32_MAX, because + // that limit is used by the tree containing the Streams. + const uint64_t limit = UINT64_MAX - index_base; + if (streams == 0 || streams > UINT32_MAX || blocks > LZMA_VLI_MAX + || streams > limit / stream_base + || groups > limit / group_base + || limit - streams_mem < groups_mem) + return UINT64_MAX; + + return index_base + streams_mem + groups_mem; +} + + +extern LZMA_API(uint64_t) +lzma_index_memused(const lzma_index *i) +{ + return lzma_index_memusage(i->streams.count, i->record_count); +} + + +extern LZMA_API(lzma_vli) +lzma_index_block_count(const lzma_index *i) +{ + return i->record_count; +} + + +extern LZMA_API(lzma_vli) +lzma_index_stream_count(const lzma_index *i) +{ + return i->streams.count; +} + + +extern LZMA_API(lzma_vli) +lzma_index_size(const lzma_index *i) +{ + return index_size(i->record_count, i->index_list_size); +} + + +extern LZMA_API(lzma_vli) +lzma_index_total_size(const lzma_index *i) +{ + return i->total_size; +} + + +extern LZMA_API(lzma_vli) +lzma_index_stream_size(const lzma_index *i) +{ + // Stream Header + Blocks + Index + Stream Footer + return LZMA_STREAM_HEADER_SIZE + i->total_size + + index_size(i->record_count, i->index_list_size) + + LZMA_STREAM_HEADER_SIZE; +} + + +static lzma_vli +index_file_size(lzma_vli compressed_base, lzma_vli unpadded_sum, + lzma_vli record_count, lzma_vli index_list_size, + lzma_vli stream_padding) +{ + // Earlier Streams and Stream Paddings + Stream Header + // + Blocks + Index + Stream Footer + Stream Padding + // + // This might go over LZMA_VLI_MAX due to too big unpadded_sum + // when this function is used in lzma_index_append(). + lzma_vli file_size = compressed_base + 2 * LZMA_STREAM_HEADER_SIZE + + stream_padding + vli_ceil4(unpadded_sum); + if (file_size > LZMA_VLI_MAX) + return LZMA_VLI_UNKNOWN; + + // The same applies here. + file_size += index_size(record_count, index_list_size); + if (file_size > LZMA_VLI_MAX) + return LZMA_VLI_UNKNOWN; + + return file_size; +} + + +extern LZMA_API(lzma_vli) +lzma_index_file_size(const lzma_index *i) +{ + const index_stream *s = (const index_stream *)(i->streams.rightmost); + const index_group *g = (const index_group *)(s->groups.rightmost); + return index_file_size(s->node.compressed_base, + g == NULL ? 0 : g->records[g->last].unpadded_sum, + s->record_count, s->index_list_size, + s->stream_padding); +} + + +extern LZMA_API(lzma_vli) +lzma_index_uncompressed_size(const lzma_index *i) +{ + return i->uncompressed_size; +} + + +extern LZMA_API(uint32_t) +lzma_index_checks(const lzma_index *i) +{ + uint32_t checks = i->checks; + + // Get the type of the Check of the last Stream too. + const index_stream *s = (const index_stream *)(i->streams.rightmost); + if (s->stream_flags.version != UINT32_MAX) + checks |= UINT32_C(1) << s->stream_flags.check; + + return checks; +} + + +extern uint32_t +lzma_index_padding_size(const lzma_index *i) +{ + return (LZMA_VLI_C(4) - index_size_unpadded( + i->record_count, i->index_list_size)) & 3; +} + + +extern LZMA_API(lzma_ret) +lzma_index_stream_flags(lzma_index *i, const lzma_stream_flags *stream_flags) +{ + if (i == NULL || stream_flags == NULL) + return LZMA_PROG_ERROR; + + // Validate the Stream Flags. + return_if_error(lzma_stream_flags_compare( + stream_flags, stream_flags)); + + index_stream *s = (index_stream *)(i->streams.rightmost); + s->stream_flags = *stream_flags; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_index_stream_padding(lzma_index *i, lzma_vli stream_padding) +{ + if (i == NULL || stream_padding > LZMA_VLI_MAX + || (stream_padding & 3) != 0) + return LZMA_PROG_ERROR; + + index_stream *s = (index_stream *)(i->streams.rightmost); + + // Check that the new value won't make the file grow too big. + const lzma_vli old_stream_padding = s->stream_padding; + s->stream_padding = 0; + if (lzma_index_file_size(i) + stream_padding > LZMA_VLI_MAX) { + s->stream_padding = old_stream_padding; + return LZMA_DATA_ERROR; + } + + s->stream_padding = stream_padding; + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_index_append(lzma_index *i, lzma_allocator *allocator, + lzma_vli unpadded_size, lzma_vli uncompressed_size) +{ + // Validate. + if (i == NULL || unpadded_size < UNPADDED_SIZE_MIN + || unpadded_size > UNPADDED_SIZE_MAX + || uncompressed_size > LZMA_VLI_MAX) + return LZMA_PROG_ERROR; + + index_stream *s = (index_stream *)(i->streams.rightmost); + index_group *g = (index_group *)(s->groups.rightmost); + + const lzma_vli compressed_base = g == NULL ? 0 + : vli_ceil4(g->records[g->last].unpadded_sum); + const lzma_vli uncompressed_base = g == NULL ? 0 + : g->records[g->last].uncompressed_sum; + const uint32_t index_list_size_add = lzma_vli_size(unpadded_size) + + lzma_vli_size(uncompressed_size); + + // Check that the file size will stay within limits. + if (index_file_size(s->node.compressed_base, + compressed_base + unpadded_size, s->record_count + 1, + s->index_list_size + index_list_size_add, + s->stream_padding) == LZMA_VLI_UNKNOWN) + return LZMA_DATA_ERROR; + + // The size of the Index field must not exceed the maximum value + // that can be stored in the Backward Size field. + if (index_size(i->record_count + 1, + i->index_list_size + index_list_size_add) + > LZMA_BACKWARD_SIZE_MAX) + return LZMA_DATA_ERROR; + + if (g != NULL && g->last + 1 < g->allocated) { + // There is space in the last group at least for one Record. + ++g->last; + } else { + // We need to allocate a new group. + g = lzma_alloc(sizeof(index_group) + + i->prealloc * sizeof(index_record), + allocator); + if (g == NULL) + return LZMA_MEM_ERROR; + + g->last = 0; + g->allocated = i->prealloc; + + // Reset prealloc so that if the application happens to + // add new Records, the allocation size will be sane. + i->prealloc = INDEX_GROUP_SIZE; + + // Set the start offsets of this group. + g->node.uncompressed_base = uncompressed_base; + g->node.compressed_base = compressed_base; + g->number_base = s->record_count + 1; + + // Add the new group to the Stream. + index_tree_append(&s->groups, &g->node); + } + + // Add the new Record to the group. + g->records[g->last].uncompressed_sum + = uncompressed_base + uncompressed_size; + g->records[g->last].unpadded_sum + = compressed_base + unpadded_size; + + // Update the totals. + ++s->record_count; + s->index_list_size += index_list_size_add; + + i->total_size += vli_ceil4(unpadded_size); + i->uncompressed_size += uncompressed_size; + ++i->record_count; + i->index_list_size += index_list_size_add; + + return LZMA_OK; +} + + +/// Structure to pass info to index_cat_helper() +typedef struct { + /// Uncompressed size of the destination + lzma_vli uncompressed_size; + + /// Compressed file size of the destination + lzma_vli file_size; + + /// Same as above but for Block numbers + lzma_vli block_number_add; + + /// Number of Streams that were in the destination index before we + /// started appending new Streams from the source index. This is + /// used to fix the Stream numbering. + uint32_t stream_number_add; + + /// Destination index' Stream tree + index_tree *streams; + +} index_cat_info; + + +/// Add the Stream nodes from the source index to dest using recursion. +/// Simplest iterative traversal of the source tree wouldn't work, because +/// we update the pointers in nodes when moving them to the destination tree. +static void +index_cat_helper(const index_cat_info *info, index_stream *this) +{ + index_stream *left = (index_stream *)(this->node.left); + index_stream *right = (index_stream *)(this->node.right); + + if (left != NULL) + index_cat_helper(info, left); + + this->node.uncompressed_base += info->uncompressed_size; + this->node.compressed_base += info->file_size; + this->number += info->stream_number_add; + this->block_number_base += info->block_number_add; + index_tree_append(info->streams, &this->node); + + if (right != NULL) + index_cat_helper(info, right); + + return; +} + + +extern LZMA_API(lzma_ret) +lzma_index_cat(lzma_index *restrict dest, lzma_index *restrict src, + lzma_allocator *allocator) +{ + const lzma_vli dest_file_size = lzma_index_file_size(dest); + + // Check that we don't exceed the file size limits. + if (dest_file_size + lzma_index_file_size(src) > LZMA_VLI_MAX + || dest->uncompressed_size + src->uncompressed_size + > LZMA_VLI_MAX) + return LZMA_DATA_ERROR; + + // Check that the encoded size of the combined lzma_indexes stays + // within limits. In theory, this should be done only if we know + // that the user plans to actually combine the Streams and thus + // construct a single Index (probably rare). However, exceeding + // this limit is quite theoretical, so we do this check always + // to simplify things elsewhere. + { + const lzma_vli dest_size = index_size_unpadded( + dest->record_count, dest->index_list_size); + const lzma_vli src_size = index_size_unpadded( + src->record_count, src->index_list_size); + if (vli_ceil4(dest_size + src_size) > LZMA_BACKWARD_SIZE_MAX) + return LZMA_DATA_ERROR; + } + + // Optimize the last group to minimize memory usage. Allocation has + // to be done before modifying dest or src. + { + index_stream *s = (index_stream *)(dest->streams.rightmost); + index_group *g = (index_group *)(s->groups.rightmost); + if (g != NULL && g->last + 1 < g->allocated) { + assert(g->node.left == NULL); + assert(g->node.right == NULL); + + index_group *newg = lzma_alloc(sizeof(index_group) + + (g->last + 1) + * sizeof(index_record), + allocator); + if (newg == NULL) + return LZMA_MEM_ERROR; + + newg->node = g->node; + newg->allocated = g->last + 1; + newg->last = g->last; + newg->number_base = g->number_base; + + memcpy(newg->records, g->records, newg->allocated + * sizeof(index_record)); + + if (g->node.parent != NULL) { + assert(g->node.parent->right == &g->node); + g->node.parent->right = &newg->node; + } + + if (s->groups.leftmost == &g->node) { + assert(s->groups.root == &g->node); + s->groups.leftmost = &newg->node; + s->groups.root = &newg->node; + } + + if (s->groups.rightmost == &g->node) + s->groups.rightmost = &newg->node; + + lzma_free(g, allocator); + } + } + + // Add all the Streams from src to dest. Update the base offsets + // of each Stream from src. + const index_cat_info info = { + .uncompressed_size = dest->uncompressed_size, + .file_size = dest_file_size, + .stream_number_add = dest->streams.count, + .block_number_add = dest->record_count, + .streams = &dest->streams, + }; + index_cat_helper(&info, (index_stream *)(src->streams.root)); + + // Update info about all the combined Streams. + dest->uncompressed_size += src->uncompressed_size; + dest->total_size += src->total_size; + dest->record_count += src->record_count; + dest->index_list_size += src->index_list_size; + dest->checks = lzma_index_checks(dest) | src->checks; + + // There's nothing else left in src than the base structure. + lzma_free(src, allocator); + + return LZMA_OK; +} + + +/// Duplicate an index_stream. +static index_stream * +index_dup_stream(const index_stream *src, lzma_allocator *allocator) +{ + // Catch a somewhat theoretical integer overflow. + if (src->record_count > PREALLOC_MAX) + return NULL; + + // Allocate and initialize a new Stream. + index_stream *dest = index_stream_init(src->node.compressed_base, + src->node.uncompressed_base, src->number, + src->block_number_base, allocator); + + // Return immediately if allocation failed or if there are + // no groups to duplicate. + if (dest == NULL || src->groups.leftmost == NULL) + return dest; + + // Copy the overall information. + dest->record_count = src->record_count; + dest->index_list_size = src->index_list_size; + dest->stream_flags = src->stream_flags; + dest->stream_padding = src->stream_padding; + + // Allocate memory for the Records. We put all the Records into + // a single group. It's simplest and also tends to make + // lzma_index_locate() a little bit faster with very big Indexes. + index_group *destg = lzma_alloc(sizeof(index_group) + + src->record_count * sizeof(index_record), + allocator); + if (destg == NULL) { + index_stream_end(dest, allocator); + return NULL; + } + + // Initialize destg. + destg->node.uncompressed_base = 0; + destg->node.compressed_base = 0; + destg->number_base = 1; + destg->allocated = src->record_count; + destg->last = src->record_count - 1; + + // Go through all the groups in src and copy the Records into destg. + const index_group *srcg = (const index_group *)(src->groups.leftmost); + size_t i = 0; + do { + memcpy(destg->records + i, srcg->records, + (srcg->last + 1) * sizeof(index_record)); + i += srcg->last + 1; + srcg = index_tree_next(&srcg->node); + } while (srcg != NULL); + + assert(i == destg->allocated); + + // Add the group to the new Stream. + index_tree_append(&dest->groups, &destg->node); + + return dest; +} + + +extern LZMA_API(lzma_index *) +lzma_index_dup(const lzma_index *src, lzma_allocator *allocator) +{ + // Allocate the base structure (no initial Stream). + lzma_index *dest = index_init_plain(allocator); + if (dest == NULL) + return NULL; + + // Copy the totals. + dest->uncompressed_size = src->uncompressed_size; + dest->total_size = src->total_size; + dest->record_count = src->record_count; + dest->index_list_size = src->index_list_size; + + // Copy the Streams and the groups in them. + const index_stream *srcstream + = (const index_stream *)(src->streams.leftmost); + do { + index_stream *deststream = index_dup_stream( + srcstream, allocator); + if (deststream == NULL) { + lzma_index_end(dest, allocator); + return NULL; + } + + index_tree_append(&dest->streams, &deststream->node); + + srcstream = index_tree_next(&srcstream->node); + } while (srcstream != NULL); + + return dest; +} + + +/// Indexing for lzma_index_iter.internal[] +enum { + ITER_INDEX, + ITER_STREAM, + ITER_GROUP, + ITER_RECORD, + ITER_METHOD, +}; + + +/// Values for lzma_index_iter.internal[ITER_METHOD].s +enum { + ITER_METHOD_NORMAL, + ITER_METHOD_NEXT, + ITER_METHOD_LEFTMOST, +}; + + +static void +iter_set_info(lzma_index_iter *iter) +{ + const lzma_index *i = iter->internal[ITER_INDEX].p; + const index_stream *stream = iter->internal[ITER_STREAM].p; + const index_group *group = iter->internal[ITER_GROUP].p; + const size_t record = iter->internal[ITER_RECORD].s; + + // lzma_index_iter.internal must not contain a pointer to the last + // group in the index, because that may be reallocated by + // lzma_index_cat(). + if (group == NULL) { + // There are no groups. + assert(stream->groups.root == NULL); + iter->internal[ITER_METHOD].s = ITER_METHOD_LEFTMOST; + + } else if (i->streams.rightmost != &stream->node + || stream->groups.rightmost != &group->node) { + // The group is not not the last group in the index. + iter->internal[ITER_METHOD].s = ITER_METHOD_NORMAL; + + } else if (stream->groups.leftmost != &group->node) { + // The group isn't the only group in the Stream, thus we + // know that it must have a parent group i.e. it's not + // the root node. + assert(stream->groups.root != &group->node); + assert(group->node.parent->right == &group->node); + iter->internal[ITER_METHOD].s = ITER_METHOD_NEXT; + iter->internal[ITER_GROUP].p = group->node.parent; + + } else { + // The Stream has only one group. + assert(stream->groups.root == &group->node); + assert(group->node.parent == NULL); + iter->internal[ITER_METHOD].s = ITER_METHOD_LEFTMOST; + iter->internal[ITER_GROUP].p = NULL; + } + + iter->stream.number = stream->number; + iter->stream.block_count = stream->record_count; + iter->stream.compressed_offset = stream->node.compressed_base; + iter->stream.uncompressed_offset = stream->node.uncompressed_base; + + // iter->stream.flags will be NULL if the Stream Flags haven't been + // set with lzma_index_stream_flags(). + iter->stream.flags = stream->stream_flags.version == UINT32_MAX + ? NULL : &stream->stream_flags; + iter->stream.padding = stream->stream_padding; + + if (stream->groups.rightmost == NULL) { + // Stream has no Blocks. + iter->stream.compressed_size = index_size(0, 0) + + 2 * LZMA_STREAM_HEADER_SIZE; + iter->stream.uncompressed_size = 0; + } else { + const index_group *g = (const index_group *)( + stream->groups.rightmost); + + // Stream Header + Stream Footer + Index + Blocks + iter->stream.compressed_size = 2 * LZMA_STREAM_HEADER_SIZE + + index_size(stream->record_count, + stream->index_list_size) + + vli_ceil4(g->records[g->last].unpadded_sum); + iter->stream.uncompressed_size + = g->records[g->last].uncompressed_sum; + } + + if (group != NULL) { + iter->block.number_in_stream = group->number_base + record; + iter->block.number_in_file = iter->block.number_in_stream + + stream->block_number_base; + + iter->block.compressed_stream_offset + = record == 0 ? group->node.compressed_base + : vli_ceil4(group->records[ + record - 1].unpadded_sum); + iter->block.uncompressed_stream_offset + = record == 0 ? group->node.uncompressed_base + : group->records[record - 1].uncompressed_sum; + + iter->block.uncompressed_size + = group->records[record].uncompressed_sum + - iter->block.uncompressed_stream_offset; + iter->block.unpadded_size + = group->records[record].unpadded_sum + - iter->block.compressed_stream_offset; + iter->block.total_size = vli_ceil4(iter->block.unpadded_size); + + iter->block.compressed_stream_offset + += LZMA_STREAM_HEADER_SIZE; + + iter->block.compressed_file_offset + = iter->block.compressed_stream_offset + + iter->stream.compressed_offset; + iter->block.uncompressed_file_offset + = iter->block.uncompressed_stream_offset + + iter->stream.uncompressed_offset; + } + + return; +} + + +extern LZMA_API(void) +lzma_index_iter_init(lzma_index_iter *iter, const lzma_index *i) +{ + iter->internal[ITER_INDEX].p = i; + lzma_index_iter_rewind(iter); + return; +} + + +extern LZMA_API(void) +lzma_index_iter_rewind(lzma_index_iter *iter) +{ + iter->internal[ITER_STREAM].p = NULL; + iter->internal[ITER_GROUP].p = NULL; + iter->internal[ITER_RECORD].s = 0; + iter->internal[ITER_METHOD].s = ITER_METHOD_NORMAL; + return; +} + + +extern LZMA_API(lzma_bool) +lzma_index_iter_next(lzma_index_iter *iter, lzma_index_iter_mode mode) +{ + // Catch unsupported mode values. + if ((unsigned int)(mode) > LZMA_INDEX_ITER_NONEMPTY_BLOCK) + return true; + + const lzma_index *i = iter->internal[ITER_INDEX].p; + const index_stream *stream = iter->internal[ITER_STREAM].p; + const index_group *group = NULL; + size_t record = iter->internal[ITER_RECORD].s; + + // If we are being asked for the next Stream, leave group to NULL + // so that the rest of the this function thinks that this Stream + // has no groups and will thus go to the next Stream. + if (mode != LZMA_INDEX_ITER_STREAM) { + // Get the pointer to the current group. See iter_set_inf() + // for explanation. + switch (iter->internal[ITER_METHOD].s) { + case ITER_METHOD_NORMAL: + group = iter->internal[ITER_GROUP].p; + break; + + case ITER_METHOD_NEXT: + group = index_tree_next(iter->internal[ITER_GROUP].p); + break; + + case ITER_METHOD_LEFTMOST: + group = (const index_group *)( + stream->groups.leftmost); + break; + } + } + +again: + if (stream == NULL) { + // We at the beginning of the lzma_index. + // Locate the first Stream. + stream = (const index_stream *)(i->streams.leftmost); + if (mode >= LZMA_INDEX_ITER_BLOCK) { + // Since we are being asked to return information + // about the first a Block, skip Streams that have + // no Blocks. + while (stream->groups.leftmost == NULL) { + stream = index_tree_next(&stream->node); + if (stream == NULL) + return true; + } + } + + // Start from the first Record in the Stream. + group = (const index_group *)(stream->groups.leftmost); + record = 0; + + } else if (group != NULL && record < group->last) { + // The next Record is in the same group. + ++record; + + } else { + // This group has no more Records or this Stream has + // no Blocks at all. + record = 0; + + // If group is not NULL, this Stream has at least one Block + // and thus at least one group. Find the next group. + if (group != NULL) + group = index_tree_next(&group->node); + + if (group == NULL) { + // This Stream has no more Records. Find the next + // Stream. If we are being asked to return information + // about a Block, we skip empty Streams. + do { + stream = index_tree_next(&stream->node); + if (stream == NULL) + return true; + } while (mode >= LZMA_INDEX_ITER_BLOCK + && stream->groups.leftmost == NULL); + + group = (const index_group *)( + stream->groups.leftmost); + } + } + + if (mode == LZMA_INDEX_ITER_NONEMPTY_BLOCK) { + // We need to look for the next Block again if this Block + // is empty. + if (record == 0) { + if (group->node.uncompressed_base + == group->records[0].uncompressed_sum) + goto again; + } else if (group->records[record - 1].uncompressed_sum + == group->records[record].uncompressed_sum) { + goto again; + } + } + + iter->internal[ITER_STREAM].p = stream; + iter->internal[ITER_GROUP].p = group; + iter->internal[ITER_RECORD].s = record; + + iter_set_info(iter); + + return false; +} + + +extern LZMA_API(lzma_bool) +lzma_index_iter_locate(lzma_index_iter *iter, lzma_vli target) +{ + const lzma_index *i = iter->internal[ITER_INDEX].p; + + // If the target is past the end of the file, return immediately. + if (i->uncompressed_size <= target) + return true; + + // Locate the Stream containing the target offset. + const index_stream *stream = index_tree_locate(&i->streams, target); + assert(stream != NULL); + target -= stream->node.uncompressed_base; + + // Locate the group containing the target offset. + const index_group *group = index_tree_locate(&stream->groups, target); + assert(group != NULL); + + // Use binary search to locate the exact Record. It is the first + // Record whose uncompressed_sum is greater than target. + // This is because we want the rightmost Record that fullfills the + // search criterion. It is possible that there are empty Blocks; + // we don't want to return them. + size_t left = 0; + size_t right = group->last; + + while (left < right) { + const size_t pos = left + (right - left) / 2; + if (group->records[pos].uncompressed_sum <= target) + left = pos + 1; + else + right = pos; + } + + iter->internal[ITER_STREAM].p = stream; + iter->internal[ITER_GROUP].p = group; + iter->internal[ITER_RECORD].s = left; + + iter_set_info(iter); + + return false; +} diff --git a/contrib/xz/src/liblzma/common/index.h b/contrib/xz/src/liblzma/common/index.h new file mode 100644 index 0000000..64e9724 --- /dev/null +++ b/contrib/xz/src/liblzma/common/index.h @@ -0,0 +1,73 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file index.h +/// \brief Handling of Index +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_INDEX_H +#define LZMA_INDEX_H + +#include "common.h" + + +/// Minimum Unpadded Size +#define UNPADDED_SIZE_MIN LZMA_VLI_C(5) + +/// Maximum Unpadded Size +#define UNPADDED_SIZE_MAX (LZMA_VLI_MAX & ~LZMA_VLI_C(3)) + + +/// Get the size of the Index Padding field. This is needed by Index encoder +/// and decoder, but applications should have no use for this. +extern uint32_t lzma_index_padding_size(const lzma_index *i); + + +/// Set for how many Records to allocate memory the next time +/// lzma_index_append() needs to allocate space for a new Record. +/// This is used only by the Index decoder. +extern void lzma_index_prealloc(lzma_index *i, lzma_vli records); + + +/// Round the variable-length integer to the next multiple of four. +static inline lzma_vli +vli_ceil4(lzma_vli vli) +{ + assert(vli <= LZMA_VLI_MAX); + return (vli + 3) & ~LZMA_VLI_C(3); +} + + +/// Calculate the size of the Index field excluding Index Padding +static inline lzma_vli +index_size_unpadded(lzma_vli count, lzma_vli index_list_size) +{ + // Index Indicator + Number of Records + List of Records + CRC32 + return 1 + lzma_vli_size(count) + index_list_size + 4; +} + + +/// Calculate the size of the Index field including Index Padding +static inline lzma_vli +index_size(lzma_vli count, lzma_vli index_list_size) +{ + return vli_ceil4(index_size_unpadded(count, index_list_size)); +} + + +/// Calculate the total size of the Stream +static inline lzma_vli +index_stream_size(lzma_vli blocks_size, + lzma_vli count, lzma_vli index_list_size) +{ + return LZMA_STREAM_HEADER_SIZE + blocks_size + + index_size(count, index_list_size) + + LZMA_STREAM_HEADER_SIZE; +} + +#endif diff --git a/contrib/xz/src/liblzma/common/index_decoder.c b/contrib/xz/src/liblzma/common/index_decoder.c new file mode 100644 index 0000000..86a2297 --- /dev/null +++ b/contrib/xz/src/liblzma/common/index_decoder.c @@ -0,0 +1,343 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file index_decoder.c +/// \brief Decodes the Index field +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "index.h" +#include "check.h" + + +struct lzma_coder_s { + enum { + SEQ_INDICATOR, + SEQ_COUNT, + SEQ_MEMUSAGE, + SEQ_UNPADDED, + SEQ_UNCOMPRESSED, + SEQ_PADDING_INIT, + SEQ_PADDING, + SEQ_CRC32, + } sequence; + + /// Memory usage limit + uint64_t memlimit; + + /// Target Index + lzma_index *index; + + /// Pointer give by the application, which is set after + /// successful decoding. + lzma_index **index_ptr; + + /// Number of Records left to decode. + lzma_vli count; + + /// The most recent Unpadded Size field + lzma_vli unpadded_size; + + /// The most recent Uncompressed Size field + lzma_vli uncompressed_size; + + /// Position in integers + size_t pos; + + /// CRC32 of the List of Records field + uint32_t crc32; +}; + + +static lzma_ret +index_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out lzma_attribute((unused)), + size_t *restrict out_pos lzma_attribute((unused)), + size_t out_size lzma_attribute((unused)), + lzma_action action lzma_attribute((unused))) +{ + // Similar optimization as in index_encoder.c + const size_t in_start = *in_pos; + lzma_ret ret = LZMA_OK; + + while (*in_pos < in_size) + switch (coder->sequence) { + case SEQ_INDICATOR: + // Return LZMA_DATA_ERROR instead of e.g. LZMA_PROG_ERROR or + // LZMA_FORMAT_ERROR, because a typical usage case for Index + // decoder is when parsing the Stream backwards. If seeking + // backward from the Stream Footer gives us something that + // doesn't begin with Index Indicator, the file is considered + // corrupt, not "programming error" or "unrecognized file + // format". One could argue that the application should + // verify the Index Indicator before trying to decode the + // Index, but well, I suppose it is simpler this way. + if (in[(*in_pos)++] != 0x00) + return LZMA_DATA_ERROR; + + coder->sequence = SEQ_COUNT; + break; + + case SEQ_COUNT: + ret = lzma_vli_decode(&coder->count, &coder->pos, + in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + goto out; + + coder->pos = 0; + coder->sequence = SEQ_MEMUSAGE; + + // Fall through + + case SEQ_MEMUSAGE: + if (lzma_index_memusage(1, coder->count) > coder->memlimit) { + ret = LZMA_MEMLIMIT_ERROR; + goto out; + } + + // Tell the Index handling code how many Records this + // Index has to allow it to allocate memory more efficiently. + lzma_index_prealloc(coder->index, coder->count); + + ret = LZMA_OK; + coder->sequence = coder->count == 0 + ? SEQ_PADDING_INIT : SEQ_UNPADDED; + break; + + case SEQ_UNPADDED: + case SEQ_UNCOMPRESSED: { + lzma_vli *size = coder->sequence == SEQ_UNPADDED + ? &coder->unpadded_size + : &coder->uncompressed_size; + + ret = lzma_vli_decode(size, &coder->pos, + in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + goto out; + + ret = LZMA_OK; + coder->pos = 0; + + if (coder->sequence == SEQ_UNPADDED) { + // Validate that encoded Unpadded Size isn't too small + // or too big. + if (coder->unpadded_size < UNPADDED_SIZE_MIN + || coder->unpadded_size + > UNPADDED_SIZE_MAX) + return LZMA_DATA_ERROR; + + coder->sequence = SEQ_UNCOMPRESSED; + } else { + // Add the decoded Record to the Index. + return_if_error(lzma_index_append( + coder->index, allocator, + coder->unpadded_size, + coder->uncompressed_size)); + + // Check if this was the last Record. + coder->sequence = --coder->count == 0 + ? SEQ_PADDING_INIT + : SEQ_UNPADDED; + } + + break; + } + + case SEQ_PADDING_INIT: + coder->pos = lzma_index_padding_size(coder->index); + coder->sequence = SEQ_PADDING; + + // Fall through + + case SEQ_PADDING: + if (coder->pos > 0) { + --coder->pos; + if (in[(*in_pos)++] != 0x00) + return LZMA_DATA_ERROR; + + break; + } + + // Finish the CRC32 calculation. + coder->crc32 = lzma_crc32(in + in_start, + *in_pos - in_start, coder->crc32); + + coder->sequence = SEQ_CRC32; + + // Fall through + + case SEQ_CRC32: + do { + if (*in_pos == in_size) + return LZMA_OK; + + if (((coder->crc32 >> (coder->pos * 8)) & 0xFF) + != in[(*in_pos)++]) + return LZMA_DATA_ERROR; + + } while (++coder->pos < 4); + + // Decoding was successful, now we can let the application + // see the decoded Index. + *coder->index_ptr = coder->index; + + // Make index NULL so we don't free it unintentionally. + coder->index = NULL; + + return LZMA_STREAM_END; + + default: + assert(0); + return LZMA_PROG_ERROR; + } + +out: + // Update the CRC32, + coder->crc32 = lzma_crc32(in + in_start, + *in_pos - in_start, coder->crc32); + + return ret; +} + + +static void +index_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_index_end(coder->index, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +index_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + *memusage = lzma_index_memusage(1, coder->count); + *old_memlimit = coder->memlimit; + + if (new_memlimit != 0) { + if (new_memlimit < *memusage) + return LZMA_MEMLIMIT_ERROR; + + coder->memlimit = new_memlimit; + } + + return LZMA_OK; +} + + +static lzma_ret +index_decoder_reset(lzma_coder *coder, lzma_allocator *allocator, + lzma_index **i, uint64_t memlimit) +{ + // Remember the pointer given by the application. We will set it + // to point to the decoded Index only if decoding is successful. + // Before that, keep it NULL so that applications can always safely + // pass it to lzma_index_end() no matter did decoding succeed or not. + coder->index_ptr = i; + *i = NULL; + + // We always allocate a new lzma_index. + coder->index = lzma_index_init(allocator); + if (coder->index == NULL) + return LZMA_MEM_ERROR; + + // Initialize the rest. + coder->sequence = SEQ_INDICATOR; + coder->memlimit = memlimit; + coder->count = 0; // Needs to be initialized due to _memconfig(). + coder->pos = 0; + coder->crc32 = 0; + + return LZMA_OK; +} + + +static lzma_ret +index_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + lzma_index **i, uint64_t memlimit) +{ + lzma_next_coder_init(&index_decoder_init, next, allocator); + + if (i == NULL || memlimit == 0) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &index_decode; + next->end = &index_decoder_end; + next->memconfig = &index_decoder_memconfig; + next->coder->index = NULL; + } else { + lzma_index_end(next->coder->index, allocator); + } + + return index_decoder_reset(next->coder, allocator, i, memlimit); +} + + +extern LZMA_API(lzma_ret) +lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) +{ + lzma_next_strm_init(index_decoder_init, strm, i, memlimit); + + strm->internal->supported_actions[LZMA_RUN] = true; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_index_buffer_decode( + lzma_index **i, uint64_t *memlimit, lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size) +{ + // Sanity checks + if (i == NULL || memlimit == NULL + || in == NULL || in_pos == NULL || *in_pos > in_size) + return LZMA_PROG_ERROR; + + // Initialize the decoder. + lzma_coder coder; + return_if_error(index_decoder_reset(&coder, allocator, i, *memlimit)); + + // Store the input start position so that we can restore it in case + // of an error. + const size_t in_start = *in_pos; + + // Do the actual decoding. + lzma_ret ret = index_decode(&coder, allocator, in, in_pos, in_size, + NULL, NULL, 0, LZMA_RUN); + + if (ret == LZMA_STREAM_END) { + ret = LZMA_OK; + } else { + // Something went wrong, free the Index structure and restore + // the input position. + lzma_index_end(coder.index, allocator); + *in_pos = in_start; + + if (ret == LZMA_OK) { + // The input is truncated or otherwise corrupt. + // Use LZMA_DATA_ERROR instead of LZMA_BUF_ERROR + // like lzma_vli_decode() does in single-call mode. + ret = LZMA_DATA_ERROR; + + } else if (ret == LZMA_MEMLIMIT_ERROR) { + // Tell the caller how much memory would have + // been needed. + *memlimit = lzma_index_memusage(1, coder.count); + } + } + + return ret; +} diff --git a/contrib/xz/src/liblzma/common/index_encoder.c b/contrib/xz/src/liblzma/common/index_encoder.c new file mode 100644 index 0000000..706f1fd --- /dev/null +++ b/contrib/xz/src/liblzma/common/index_encoder.c @@ -0,0 +1,252 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file index_encoder.c +/// \brief Encodes the Index field +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "index_encoder.h" +#include "index.h" +#include "check.h" + + +struct lzma_coder_s { + enum { + SEQ_INDICATOR, + SEQ_COUNT, + SEQ_UNPADDED, + SEQ_UNCOMPRESSED, + SEQ_NEXT, + SEQ_PADDING, + SEQ_CRC32, + } sequence; + + /// Index being encoded + const lzma_index *index; + + /// Iterator for the Index being encoded + lzma_index_iter iter; + + /// Position in integers + size_t pos; + + /// CRC32 of the List of Records field + uint32_t crc32; +}; + + +static lzma_ret +index_encode(lzma_coder *coder, + lzma_allocator *allocator lzma_attribute((unused)), + const uint8_t *restrict in lzma_attribute((unused)), + size_t *restrict in_pos lzma_attribute((unused)), + size_t in_size lzma_attribute((unused)), + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size, lzma_action action lzma_attribute((unused))) +{ + // Position where to start calculating CRC32. The idea is that we + // need to call lzma_crc32() only once per call to index_encode(). + const size_t out_start = *out_pos; + + // Return value to use if we return at the end of this function. + // We use "goto out" to jump out of the while-switch construct + // instead of returning directly, because that way we don't need + // to copypaste the lzma_crc32() call to many places. + lzma_ret ret = LZMA_OK; + + while (*out_pos < out_size) + switch (coder->sequence) { + case SEQ_INDICATOR: + out[*out_pos] = 0x00; + ++*out_pos; + coder->sequence = SEQ_COUNT; + break; + + case SEQ_COUNT: { + const lzma_vli count = lzma_index_block_count(coder->index); + ret = lzma_vli_encode(count, &coder->pos, + out, out_pos, out_size); + if (ret != LZMA_STREAM_END) + goto out; + + ret = LZMA_OK; + coder->pos = 0; + coder->sequence = SEQ_NEXT; + break; + } + + case SEQ_NEXT: + if (lzma_index_iter_next( + &coder->iter, LZMA_INDEX_ITER_BLOCK)) { + // Get the size of the Index Padding field. + coder->pos = lzma_index_padding_size(coder->index); + assert(coder->pos <= 3); + coder->sequence = SEQ_PADDING; + break; + } + + coder->sequence = SEQ_UNPADDED; + + // Fall through + + case SEQ_UNPADDED: + case SEQ_UNCOMPRESSED: { + const lzma_vli size = coder->sequence == SEQ_UNPADDED + ? coder->iter.block.unpadded_size + : coder->iter.block.uncompressed_size; + + ret = lzma_vli_encode(size, &coder->pos, + out, out_pos, out_size); + if (ret != LZMA_STREAM_END) + goto out; + + ret = LZMA_OK; + coder->pos = 0; + + // Advance to SEQ_UNCOMPRESSED or SEQ_NEXT. + ++coder->sequence; + break; + } + + case SEQ_PADDING: + if (coder->pos > 0) { + --coder->pos; + out[(*out_pos)++] = 0x00; + break; + } + + // Finish the CRC32 calculation. + coder->crc32 = lzma_crc32(out + out_start, + *out_pos - out_start, coder->crc32); + + coder->sequence = SEQ_CRC32; + + // Fall through + + case SEQ_CRC32: + // We don't use the main loop, because we don't want + // coder->crc32 to be touched anymore. + do { + if (*out_pos == out_size) + return LZMA_OK; + + out[*out_pos] = (coder->crc32 >> (coder->pos * 8)) + & 0xFF; + ++*out_pos; + + } while (++coder->pos < 4); + + return LZMA_STREAM_END; + + default: + assert(0); + return LZMA_PROG_ERROR; + } + +out: + // Update the CRC32. + coder->crc32 = lzma_crc32(out + out_start, + *out_pos - out_start, coder->crc32); + + return ret; +} + + +static void +index_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_free(coder, allocator); + return; +} + + +static void +index_encoder_reset(lzma_coder *coder, const lzma_index *i) +{ + lzma_index_iter_init(&coder->iter, i); + + coder->sequence = SEQ_INDICATOR; + coder->index = i; + coder->pos = 0; + coder->crc32 = 0; + + return; +} + + +extern lzma_ret +lzma_index_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_index *i) +{ + lzma_next_coder_init(&lzma_index_encoder_init, next, allocator); + + if (i == NULL) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &index_encode; + next->end = &index_encoder_end; + } + + index_encoder_reset(next->coder, i); + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_index_encoder(lzma_stream *strm, const lzma_index *i) +{ + lzma_next_strm_init(lzma_index_encoder_init, strm, i); + + strm->internal->supported_actions[LZMA_RUN] = true; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_index_buffer_encode(const lzma_index *i, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // Validate the arguments. + if (i == NULL || out == NULL || out_pos == NULL || *out_pos > out_size) + return LZMA_PROG_ERROR; + + // Don't try to encode if there's not enough output space. + if (out_size - *out_pos < lzma_index_size(i)) + return LZMA_BUF_ERROR; + + // The Index encoder needs just one small data structure so we can + // allocate it on stack. + lzma_coder coder; + index_encoder_reset(&coder, i); + + // Do the actual encoding. This should never fail, but store + // the original *out_pos just in case. + const size_t out_start = *out_pos; + lzma_ret ret = index_encode(&coder, NULL, NULL, NULL, 0, + out, out_pos, out_size, LZMA_RUN); + + if (ret == LZMA_STREAM_END) { + ret = LZMA_OK; + } else { + // We should never get here, but just in case, restore the + // output position and set the error accordingly if something + // goes wrong and debugging isn't enabled. + assert(0); + *out_pos = out_start; + ret = LZMA_PROG_ERROR; + } + + return ret; +} diff --git a/contrib/xz/src/liblzma/common/index_encoder.h b/contrib/xz/src/liblzma/common/index_encoder.h new file mode 100644 index 0000000..a13c94d --- /dev/null +++ b/contrib/xz/src/liblzma/common/index_encoder.h @@ -0,0 +1,23 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file index_encoder.h +/// \brief Encodes the Index field +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_INDEX_ENCODER_H +#define LZMA_INDEX_ENCODER_H + +#include "common.h" + + +extern lzma_ret lzma_index_encoder_init(lzma_next_coder *next, + lzma_allocator *allocator, const lzma_index *i); + + +#endif diff --git a/contrib/xz/src/liblzma/common/index_hash.c b/contrib/xz/src/liblzma/common/index_hash.c new file mode 100644 index 0000000..e3e9386 --- /dev/null +++ b/contrib/xz/src/liblzma/common/index_hash.c @@ -0,0 +1,332 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file index_hash.c +/// \brief Validates Index by using a hash function +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" +#include "index.h" +#include "check.h" + + +typedef struct { + /// Sum of the Block sizes (including Block Padding) + lzma_vli blocks_size; + + /// Sum of the Uncompressed Size fields + lzma_vli uncompressed_size; + + /// Number of Records + lzma_vli count; + + /// Size of the List of Index Records as bytes + lzma_vli index_list_size; + + /// Check calculated from Unpadded Sizes and Uncompressed Sizes. + lzma_check_state check; + +} lzma_index_hash_info; + + +struct lzma_index_hash_s { + enum { + SEQ_BLOCK, + SEQ_COUNT, + SEQ_UNPADDED, + SEQ_UNCOMPRESSED, + SEQ_PADDING_INIT, + SEQ_PADDING, + SEQ_CRC32, + } sequence; + + /// Information collected while decoding the actual Blocks. + lzma_index_hash_info blocks; + + /// Information collected from the Index field. + lzma_index_hash_info records; + + /// Number of Records not fully decoded + lzma_vli remaining; + + /// Unpadded Size currently being read from an Index Record. + lzma_vli unpadded_size; + + /// Uncompressed Size currently being read from an Index Record. + lzma_vli uncompressed_size; + + /// Position in variable-length integers when decoding them from + /// the List of Records. + size_t pos; + + /// CRC32 of the Index + uint32_t crc32; +}; + + +extern LZMA_API(lzma_index_hash *) +lzma_index_hash_init(lzma_index_hash *index_hash, lzma_allocator *allocator) +{ + if (index_hash == NULL) { + index_hash = lzma_alloc(sizeof(lzma_index_hash), allocator); + if (index_hash == NULL) + return NULL; + } + + index_hash->sequence = SEQ_BLOCK; + index_hash->blocks.blocks_size = 0; + index_hash->blocks.uncompressed_size = 0; + index_hash->blocks.count = 0; + index_hash->blocks.index_list_size = 0; + index_hash->records.blocks_size = 0; + index_hash->records.uncompressed_size = 0; + index_hash->records.count = 0; + index_hash->records.index_list_size = 0; + index_hash->unpadded_size = 0; + index_hash->uncompressed_size = 0; + index_hash->pos = 0; + index_hash->crc32 = 0; + + // These cannot fail because LZMA_CHECK_BEST is known to be supported. + (void)lzma_check_init(&index_hash->blocks.check, LZMA_CHECK_BEST); + (void)lzma_check_init(&index_hash->records.check, LZMA_CHECK_BEST); + + return index_hash; +} + + +extern LZMA_API(void) +lzma_index_hash_end(lzma_index_hash *index_hash, lzma_allocator *allocator) +{ + lzma_free(index_hash, allocator); + return; +} + + +extern LZMA_API(lzma_vli) +lzma_index_hash_size(const lzma_index_hash *index_hash) +{ + // Get the size of the Index from ->blocks instead of ->records for + // cases where application wants to know the Index Size before + // decoding the Index. + return index_size(index_hash->blocks.count, + index_hash->blocks.index_list_size); +} + + +/// Updates the sizes and the hash without any validation. +static lzma_ret +hash_append(lzma_index_hash_info *info, lzma_vli unpadded_size, + lzma_vli uncompressed_size) +{ + info->blocks_size += vli_ceil4(unpadded_size); + info->uncompressed_size += uncompressed_size; + info->index_list_size += lzma_vli_size(unpadded_size) + + lzma_vli_size(uncompressed_size); + ++info->count; + + const lzma_vli sizes[2] = { unpadded_size, uncompressed_size }; + lzma_check_update(&info->check, LZMA_CHECK_BEST, + (const uint8_t *)(sizes), sizeof(sizes)); + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_index_hash_append(lzma_index_hash *index_hash, lzma_vli unpadded_size, + lzma_vli uncompressed_size) +{ + // Validate the arguments. + if (index_hash->sequence != SEQ_BLOCK + || unpadded_size < UNPADDED_SIZE_MIN + || unpadded_size > UNPADDED_SIZE_MAX + || uncompressed_size > LZMA_VLI_MAX) + return LZMA_PROG_ERROR; + + // Update the hash. + return_if_error(hash_append(&index_hash->blocks, + unpadded_size, uncompressed_size)); + + // Validate the properties of *info are still in allowed limits. + if (index_hash->blocks.blocks_size > LZMA_VLI_MAX + || index_hash->blocks.uncompressed_size > LZMA_VLI_MAX + || index_size(index_hash->blocks.count, + index_hash->blocks.index_list_size) + > LZMA_BACKWARD_SIZE_MAX + || index_stream_size(index_hash->blocks.blocks_size, + index_hash->blocks.count, + index_hash->blocks.index_list_size) + > LZMA_VLI_MAX) + return LZMA_DATA_ERROR; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_index_hash_decode(lzma_index_hash *index_hash, const uint8_t *in, + size_t *in_pos, size_t in_size) +{ + // Catch zero input buffer here, because in contrast to Index encoder + // and decoder functions, applications call this function directly + // instead of via lzma_code(), which does the buffer checking. + if (*in_pos >= in_size) + return LZMA_BUF_ERROR; + + // NOTE: This function has many similarities to index_encode() and + // index_decode() functions found from index_encoder.c and + // index_decoder.c. See the comments especially in index_encoder.c. + const size_t in_start = *in_pos; + lzma_ret ret = LZMA_OK; + + while (*in_pos < in_size) + switch (index_hash->sequence) { + case SEQ_BLOCK: + // Check the Index Indicator is present. + if (in[(*in_pos)++] != 0x00) + return LZMA_DATA_ERROR; + + index_hash->sequence = SEQ_COUNT; + break; + + case SEQ_COUNT: { + ret = lzma_vli_decode(&index_hash->remaining, + &index_hash->pos, in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + goto out; + + // The count must match the count of the Blocks decoded. + if (index_hash->remaining != index_hash->blocks.count) + return LZMA_DATA_ERROR; + + ret = LZMA_OK; + index_hash->pos = 0; + + // Handle the special case when there are no Blocks. + index_hash->sequence = index_hash->remaining == 0 + ? SEQ_PADDING_INIT : SEQ_UNPADDED; + break; + } + + case SEQ_UNPADDED: + case SEQ_UNCOMPRESSED: { + lzma_vli *size = index_hash->sequence == SEQ_UNPADDED + ? &index_hash->unpadded_size + : &index_hash->uncompressed_size; + + ret = lzma_vli_decode(size, &index_hash->pos, + in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + goto out; + + ret = LZMA_OK; + index_hash->pos = 0; + + if (index_hash->sequence == SEQ_UNPADDED) { + if (index_hash->unpadded_size < UNPADDED_SIZE_MIN + || index_hash->unpadded_size + > UNPADDED_SIZE_MAX) + return LZMA_DATA_ERROR; + + index_hash->sequence = SEQ_UNCOMPRESSED; + } else { + // Update the hash. + return_if_error(hash_append(&index_hash->records, + index_hash->unpadded_size, + index_hash->uncompressed_size)); + + // Verify that we don't go over the known sizes. Note + // that this validation is simpler than the one used + // in lzma_index_hash_append(), because here we know + // that values in index_hash->blocks are already + // validated and we are fine as long as we don't + // exceed them in index_hash->records. + if (index_hash->blocks.blocks_size + < index_hash->records.blocks_size + || index_hash->blocks.uncompressed_size + < index_hash->records.uncompressed_size + || index_hash->blocks.index_list_size + < index_hash->records.index_list_size) + return LZMA_DATA_ERROR; + + // Check if this was the last Record. + index_hash->sequence = --index_hash->remaining == 0 + ? SEQ_PADDING_INIT : SEQ_UNPADDED; + } + + break; + } + + case SEQ_PADDING_INIT: + index_hash->pos = (LZMA_VLI_C(4) - index_size_unpadded( + index_hash->records.count, + index_hash->records.index_list_size)) & 3; + index_hash->sequence = SEQ_PADDING; + + // Fall through + + case SEQ_PADDING: + if (index_hash->pos > 0) { + --index_hash->pos; + if (in[(*in_pos)++] != 0x00) + return LZMA_DATA_ERROR; + + break; + } + + // Compare the sizes. + if (index_hash->blocks.blocks_size + != index_hash->records.blocks_size + || index_hash->blocks.uncompressed_size + != index_hash->records.uncompressed_size + || index_hash->blocks.index_list_size + != index_hash->records.index_list_size) + return LZMA_DATA_ERROR; + + // Finish the hashes and compare them. + lzma_check_finish(&index_hash->blocks.check, LZMA_CHECK_BEST); + lzma_check_finish(&index_hash->records.check, LZMA_CHECK_BEST); + if (memcmp(index_hash->blocks.check.buffer.u8, + index_hash->records.check.buffer.u8, + lzma_check_size(LZMA_CHECK_BEST)) != 0) + return LZMA_DATA_ERROR; + + // Finish the CRC32 calculation. + index_hash->crc32 = lzma_crc32(in + in_start, + *in_pos - in_start, index_hash->crc32); + + index_hash->sequence = SEQ_CRC32; + + // Fall through + + case SEQ_CRC32: + do { + if (*in_pos == in_size) + return LZMA_OK; + + if (((index_hash->crc32 >> (index_hash->pos * 8)) + & 0xFF) != in[(*in_pos)++]) + return LZMA_DATA_ERROR; + + } while (++index_hash->pos < 4); + + return LZMA_STREAM_END; + + default: + assert(0); + return LZMA_PROG_ERROR; + } + +out: + // Update the CRC32, + index_hash->crc32 = lzma_crc32(in + in_start, + *in_pos - in_start, index_hash->crc32); + + return ret; +} diff --git a/contrib/xz/src/liblzma/common/stream_buffer_decoder.c b/contrib/xz/src/liblzma/common/stream_buffer_decoder.c new file mode 100644 index 0000000..ae75315 --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_buffer_decoder.c @@ -0,0 +1,91 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_buffer_decoder.c +/// \brief Single-call .xz Stream decoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_decoder.h" + + +extern LZMA_API(lzma_ret) +lzma_stream_buffer_decode(uint64_t *memlimit, uint32_t flags, + lzma_allocator *allocator, + const uint8_t *in, size_t *in_pos, size_t in_size, + uint8_t *out, size_t *out_pos, size_t out_size) +{ + // Sanity checks + if (in_pos == NULL || (in == NULL && *in_pos != in_size) + || *in_pos > in_size || out_pos == NULL + || (out == NULL && *out_pos != out_size) + || *out_pos > out_size) + return LZMA_PROG_ERROR; + + // Catch flags that are not allowed in buffer-to-buffer decoding. + if (flags & LZMA_TELL_ANY_CHECK) + return LZMA_PROG_ERROR; + + // Initialize the Stream decoder. + // TODO: We need something to tell the decoder that it can use the + // output buffer as workspace, and thus save significant amount of RAM. + lzma_next_coder stream_decoder = LZMA_NEXT_CODER_INIT; + lzma_ret ret = lzma_stream_decoder_init( + &stream_decoder, allocator, *memlimit, flags); + + if (ret == LZMA_OK) { + // Save the positions so that we can restore them in case + // an error occurs. + const size_t in_start = *in_pos; + const size_t out_start = *out_pos; + + // Do the actual decoding. + ret = stream_decoder.code(stream_decoder.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, + LZMA_FINISH); + + if (ret == LZMA_STREAM_END) { + ret = LZMA_OK; + } else { + // Something went wrong, restore the positions. + *in_pos = in_start; + *out_pos = out_start; + + if (ret == LZMA_OK) { + // Either the input was truncated or the + // output buffer was too small. + assert(*in_pos == in_size + || *out_pos == out_size); + + // If all the input was consumed, then the + // input is truncated, even if the output + // buffer is also full. This is because + // processing the last byte of the Stream + // never produces output. + if (*in_pos == in_size) + ret = LZMA_DATA_ERROR; + else + ret = LZMA_BUF_ERROR; + + } else if (ret == LZMA_MEMLIMIT_ERROR) { + // Let the caller know how much memory would + // have been needed. + uint64_t memusage; + (void)stream_decoder.memconfig( + stream_decoder.coder, + memlimit, &memusage, 0); + } + } + } + + // Free the decoder memory. This needs to be done even if + // initialization fails, because the internal API doesn't + // require the initialization function to free its memory on error. + lzma_next_end(&stream_decoder, allocator); + + return ret; +} diff --git a/contrib/xz/src/liblzma/common/stream_buffer_encoder.c b/contrib/xz/src/liblzma/common/stream_buffer_encoder.c new file mode 100644 index 0000000..bbafaa6 --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_buffer_encoder.c @@ -0,0 +1,131 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_buffer_encoder.c +/// \brief Single-call .xz Stream encoder +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "index.h" + + +/// Maximum size of Index that has exactly one Record. +/// Index Indicator + Number of Records + Record + CRC32 rounded up to +/// the next multiple of four. +#define INDEX_BOUND ((1 + 1 + 2 * LZMA_VLI_BYTES_MAX + 4 + 3) & ~3) + +/// Stream Header, Stream Footer, and Index +#define HEADERS_BOUND (2 * LZMA_STREAM_HEADER_SIZE + INDEX_BOUND) + + +extern LZMA_API(size_t) +lzma_stream_buffer_bound(size_t uncompressed_size) +{ + // Get the maximum possible size of a Block. + const size_t block_bound = lzma_block_buffer_bound(uncompressed_size); + if (block_bound == 0) + return 0; + + // Catch the possible integer overflow and also prevent the size of + // the Stream exceeding LZMA_VLI_MAX (theoretically possible on + // 64-bit systems). + if (MIN(SIZE_MAX, LZMA_VLI_MAX) - block_bound < HEADERS_BOUND) + return 0; + + return block_bound + HEADERS_BOUND; +} + + +extern LZMA_API(lzma_ret) +lzma_stream_buffer_encode(lzma_filter *filters, lzma_check check, + lzma_allocator *allocator, const uint8_t *in, size_t in_size, + uint8_t *out, size_t *out_pos_ptr, size_t out_size) +{ + // Sanity checks + if (filters == NULL || (unsigned int)(check) > LZMA_CHECK_ID_MAX + || (in == NULL && in_size != 0) || out == NULL + || out_pos_ptr == NULL || *out_pos_ptr > out_size) + return LZMA_PROG_ERROR; + + // Note for the paranoids: Index encoder prevents the Stream from + // getting too big and still being accepted with LZMA_OK, and Block + // encoder catches if the input is too big. So we don't need to + // separately check if the buffers are too big. + + // Use a local copy. We update *out_pos_ptr only if everything + // succeeds. + size_t out_pos = *out_pos_ptr; + + // Check that there's enough space for both Stream Header and + // Stream Footer. + if (out_size - out_pos <= 2 * LZMA_STREAM_HEADER_SIZE) + return LZMA_BUF_ERROR; + + // Reserve space for Stream Footer so we don't need to check for + // available space again before encoding Stream Footer. + out_size -= LZMA_STREAM_HEADER_SIZE; + + // Encode the Stream Header. + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; + + if (lzma_stream_header_encode(&stream_flags, out + out_pos) + != LZMA_OK) + return LZMA_PROG_ERROR; + + out_pos += LZMA_STREAM_HEADER_SIZE; + + // Block + lzma_block block = { + .version = 0, + .check = check, + .filters = filters, + }; + + return_if_error(lzma_block_buffer_encode(&block, allocator, + in, in_size, out, &out_pos, out_size)); + + // Index + { + // Create an Index with one Record. + lzma_index *i = lzma_index_init(allocator); + if (i == NULL) + return LZMA_MEM_ERROR; + + lzma_ret ret = lzma_index_append(i, allocator, + lzma_block_unpadded_size(&block), + block.uncompressed_size); + + // If adding the Record was successful, encode the Index + // and get its size which will be stored into Stream Footer. + if (ret == LZMA_OK) { + ret = lzma_index_buffer_encode( + i, out, &out_pos, out_size); + + stream_flags.backward_size = lzma_index_size(i); + } + + lzma_index_end(i, allocator); + + if (ret != LZMA_OK) + return ret; + } + + // Stream Footer. We have already reserved space for this. + if (lzma_stream_footer_encode(&stream_flags, out + out_pos) + != LZMA_OK) + return LZMA_PROG_ERROR; + + out_pos += LZMA_STREAM_HEADER_SIZE; + + // Everything went fine, make the new output position available + // to the application. + *out_pos_ptr = out_pos; + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/stream_decoder.c b/contrib/xz/src/liblzma/common/stream_decoder.c new file mode 100644 index 0000000..37ea71e --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_decoder.c @@ -0,0 +1,451 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_decoder.c +/// \brief Decodes .xz Streams +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_decoder.h" +#include "block_decoder.h" + + +struct lzma_coder_s { + enum { + SEQ_STREAM_HEADER, + SEQ_BLOCK_HEADER, + SEQ_BLOCK, + SEQ_INDEX, + SEQ_STREAM_FOOTER, + SEQ_STREAM_PADDING, + } sequence; + + /// Block or Metadata decoder. This takes little memory and the same + /// data structure can be used to decode every Block Header, so it's + /// a good idea to have a separate lzma_next_coder structure for it. + lzma_next_coder block_decoder; + + /// Block options decoded by the Block Header decoder and used by + /// the Block decoder. + lzma_block block_options; + + /// Stream Flags from Stream Header + lzma_stream_flags stream_flags; + + /// Index is hashed so that it can be compared to the sizes of Blocks + /// with O(1) memory usage. + lzma_index_hash *index_hash; + + /// Memory usage limit + uint64_t memlimit; + + /// Amount of memory actually needed (only an estimate) + uint64_t memusage; + + /// If true, LZMA_NO_CHECK is returned if the Stream has + /// no integrity check. + bool tell_no_check; + + /// If true, LZMA_UNSUPPORTED_CHECK is returned if the Stream has + /// an integrity check that isn't supported by this liblzma build. + bool tell_unsupported_check; + + /// If true, LZMA_GET_CHECK is returned after decoding Stream Header. + bool tell_any_check; + + /// If true, we will decode concatenated Streams that possibly have + /// Stream Padding between or after them. LZMA_STREAM_END is returned + /// once the application isn't giving us any new input, and we aren't + /// in the middle of a Stream, and possible Stream Padding is a + /// multiple of four bytes. + bool concatenated; + + /// When decoding concatenated Streams, this is true as long as we + /// are decoding the first Stream. This is needed to avoid misleading + /// LZMA_FORMAT_ERROR in case the later Streams don't have valid magic + /// bytes. + bool first_stream; + + /// Write position in buffer[] and position in Stream Padding + size_t pos; + + /// Buffer to hold Stream Header, Block Header, and Stream Footer. + /// Block Header has biggest maximum size. + uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; +}; + + +static lzma_ret +stream_decoder_reset(lzma_coder *coder, lzma_allocator *allocator) +{ + // Initialize the Index hash used to verify the Index. + coder->index_hash = lzma_index_hash_init(coder->index_hash, allocator); + if (coder->index_hash == NULL) + return LZMA_MEM_ERROR; + + // Reset the rest of the variables. + coder->sequence = SEQ_STREAM_HEADER; + coder->pos = 0; + + return LZMA_OK; +} + + +static lzma_ret +stream_decode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + // When decoding the actual Block, it may be able to produce more + // output even if we don't give it any new input. + while (true) + switch (coder->sequence) { + case SEQ_STREAM_HEADER: { + // Copy the Stream Header to the internal buffer. + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, + LZMA_STREAM_HEADER_SIZE); + + // Return if we didn't get the whole Stream Header yet. + if (coder->pos < LZMA_STREAM_HEADER_SIZE) + return LZMA_OK; + + coder->pos = 0; + + // Decode the Stream Header. + const lzma_ret ret = lzma_stream_header_decode( + &coder->stream_flags, coder->buffer); + if (ret != LZMA_OK) + return ret == LZMA_FORMAT_ERROR && !coder->first_stream + ? LZMA_DATA_ERROR : ret; + + // If we are decoding concatenated Streams, and the later + // Streams have invalid Header Magic Bytes, we give + // LZMA_DATA_ERROR instead of LZMA_FORMAT_ERROR. + coder->first_stream = false; + + // Copy the type of the Check so that Block Header and Block + // decoders see it. + coder->block_options.check = coder->stream_flags.check; + + // Even if we return LZMA_*_CHECK below, we want + // to continue from Block Header decoding. + coder->sequence = SEQ_BLOCK_HEADER; + + // Detect if there's no integrity check or if it is + // unsupported if those were requested by the application. + if (coder->tell_no_check && coder->stream_flags.check + == LZMA_CHECK_NONE) + return LZMA_NO_CHECK; + + if (coder->tell_unsupported_check + && !lzma_check_is_supported( + coder->stream_flags.check)) + return LZMA_UNSUPPORTED_CHECK; + + if (coder->tell_any_check) + return LZMA_GET_CHECK; + } + + // Fall through + + case SEQ_BLOCK_HEADER: { + if (*in_pos >= in_size) + return LZMA_OK; + + if (coder->pos == 0) { + // Detect if it's Index. + if (in[*in_pos] == 0x00) { + coder->sequence = SEQ_INDEX; + break; + } + + // Calculate the size of the Block Header. Note that + // Block Header decoder wants to see this byte too + // so don't advance *in_pos. + coder->block_options.header_size + = lzma_block_header_size_decode( + in[*in_pos]); + } + + // Copy the Block Header to the internal buffer. + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, + coder->block_options.header_size); + + // Return if we didn't get the whole Block Header yet. + if (coder->pos < coder->block_options.header_size) + return LZMA_OK; + + coder->pos = 0; + + // Version 0 is currently the only possible version. + coder->block_options.version = 0; + + // Set up a buffer to hold the filter chain. Block Header + // decoder will initialize all members of this array so + // we don't need to do it here. + lzma_filter filters[LZMA_FILTERS_MAX + 1]; + coder->block_options.filters = filters; + + // Decode the Block Header. + return_if_error(lzma_block_header_decode(&coder->block_options, + allocator, coder->buffer)); + + // Check the memory usage limit. + const uint64_t memusage = lzma_raw_decoder_memusage(filters); + lzma_ret ret; + + if (memusage == UINT64_MAX) { + // One or more unknown Filter IDs. + ret = LZMA_OPTIONS_ERROR; + } else { + // Now we can set coder->memusage since we know that + // the filter chain is valid. We don't want + // lzma_memusage() to return UINT64_MAX in case of + // invalid filter chain. + coder->memusage = memusage; + + if (memusage > coder->memlimit) { + // The chain would need too much memory. + ret = LZMA_MEMLIMIT_ERROR; + } else { + // Memory usage is OK. + // Initialize the Block decoder. + ret = lzma_block_decoder_init( + &coder->block_decoder, + allocator, + &coder->block_options); + } + } + + // Free the allocated filter options since they are needed + // only to initialize the Block decoder. + for (size_t i = 0; i < LZMA_FILTERS_MAX; ++i) + lzma_free(filters[i].options, allocator); + + coder->block_options.filters = NULL; + + // Check if memory usage calculation and Block enocoder + // initialization succeeded. + if (ret != LZMA_OK) + return ret; + + coder->sequence = SEQ_BLOCK; + } + + // Fall through + + case SEQ_BLOCK: { + const lzma_ret ret = coder->block_decoder.code( + coder->block_decoder.coder, allocator, + in, in_pos, in_size, out, out_pos, out_size, + action); + + if (ret != LZMA_STREAM_END) + return ret; + + // Block decoded successfully. Add the new size pair to + // the Index hash. + return_if_error(lzma_index_hash_append(coder->index_hash, + lzma_block_unpadded_size( + &coder->block_options), + coder->block_options.uncompressed_size)); + + coder->sequence = SEQ_BLOCK_HEADER; + break; + } + + case SEQ_INDEX: { + // If we don't have any input, don't call + // lzma_index_hash_decode() since it would return + // LZMA_BUF_ERROR, which we must not do here. + if (*in_pos >= in_size) + return LZMA_OK; + + // Decode the Index and compare it to the hash calculated + // from the sizes of the Blocks (if any). + const lzma_ret ret = lzma_index_hash_decode(coder->index_hash, + in, in_pos, in_size); + if (ret != LZMA_STREAM_END) + return ret; + + coder->sequence = SEQ_STREAM_FOOTER; + } + + // Fall through + + case SEQ_STREAM_FOOTER: { + // Copy the Stream Footer to the internal buffer. + lzma_bufcpy(in, in_pos, in_size, coder->buffer, &coder->pos, + LZMA_STREAM_HEADER_SIZE); + + // Return if we didn't get the whole Stream Footer yet. + if (coder->pos < LZMA_STREAM_HEADER_SIZE) + return LZMA_OK; + + coder->pos = 0; + + // Decode the Stream Footer. The decoder gives + // LZMA_FORMAT_ERROR if the magic bytes don't match, + // so convert that return code to LZMA_DATA_ERROR. + lzma_stream_flags footer_flags; + const lzma_ret ret = lzma_stream_footer_decode( + &footer_flags, coder->buffer); + if (ret != LZMA_OK) + return ret == LZMA_FORMAT_ERROR + ? LZMA_DATA_ERROR : ret; + + // Check that Index Size stored in the Stream Footer matches + // the real size of the Index field. + if (lzma_index_hash_size(coder->index_hash) + != footer_flags.backward_size) + return LZMA_DATA_ERROR; + + // Compare that the Stream Flags fields are identical in + // both Stream Header and Stream Footer. + return_if_error(lzma_stream_flags_compare( + &coder->stream_flags, &footer_flags)); + + if (!coder->concatenated) + return LZMA_STREAM_END; + + coder->sequence = SEQ_STREAM_PADDING; + } + + // Fall through + + case SEQ_STREAM_PADDING: + assert(coder->concatenated); + + // Skip over possible Stream Padding. + while (true) { + if (*in_pos >= in_size) { + // Unless LZMA_FINISH was used, we cannot + // know if there's more input coming later. + if (action != LZMA_FINISH) + return LZMA_OK; + + // Stream Padding must be a multiple of + // four bytes. + return coder->pos == 0 + ? LZMA_STREAM_END + : LZMA_DATA_ERROR; + } + + // If the byte is not zero, it probably indicates + // beginning of a new Stream (or the file is corrupt). + if (in[*in_pos] != 0x00) + break; + + ++*in_pos; + coder->pos = (coder->pos + 1) & 3; + } + + // Stream Padding must be a multiple of four bytes (empty + // Stream Padding is OK). + if (coder->pos != 0) { + ++*in_pos; + return LZMA_DATA_ERROR; + } + + // Prepare to decode the next Stream. + return_if_error(stream_decoder_reset(coder, allocator)); + break; + + default: + assert(0); + return LZMA_PROG_ERROR; + } + + // Never reached +} + + +static void +stream_decoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_end(&coder->block_decoder, allocator); + lzma_index_hash_end(coder->index_hash, allocator); + lzma_free(coder, allocator); + return; +} + + +static lzma_check +stream_decoder_get_check(const lzma_coder *coder) +{ + return coder->stream_flags.check; +} + + +static lzma_ret +stream_decoder_memconfig(lzma_coder *coder, uint64_t *memusage, + uint64_t *old_memlimit, uint64_t new_memlimit) +{ + *memusage = coder->memusage; + *old_memlimit = coder->memlimit; + + if (new_memlimit != 0) { + if (new_memlimit < coder->memusage) + return LZMA_MEMLIMIT_ERROR; + + coder->memlimit = new_memlimit; + } + + return LZMA_OK; +} + + +extern lzma_ret +lzma_stream_decoder_init(lzma_next_coder *next, lzma_allocator *allocator, + uint64_t memlimit, uint32_t flags) +{ + lzma_next_coder_init(&lzma_stream_decoder_init, next, allocator); + + if (memlimit == 0) + return LZMA_PROG_ERROR; + + if (flags & ~LZMA_SUPPORTED_FLAGS) + return LZMA_OPTIONS_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &stream_decode; + next->end = &stream_decoder_end; + next->get_check = &stream_decoder_get_check; + next->memconfig = &stream_decoder_memconfig; + + next->coder->block_decoder = LZMA_NEXT_CODER_INIT; + next->coder->index_hash = NULL; + } + + next->coder->memlimit = memlimit; + next->coder->memusage = LZMA_MEMUSAGE_BASE; + next->coder->tell_no_check = (flags & LZMA_TELL_NO_CHECK) != 0; + next->coder->tell_unsupported_check + = (flags & LZMA_TELL_UNSUPPORTED_CHECK) != 0; + next->coder->tell_any_check = (flags & LZMA_TELL_ANY_CHECK) != 0; + next->coder->concatenated = (flags & LZMA_CONCATENATED) != 0; + next->coder->first_stream = true; + + return stream_decoder_reset(next->coder, allocator); +} + + +extern LZMA_API(lzma_ret) +lzma_stream_decoder(lzma_stream *strm, uint64_t memlimit, uint32_t flags) +{ + lzma_next_strm_init(lzma_stream_decoder_init, strm, memlimit, flags); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/stream_decoder.h b/contrib/xz/src/liblzma/common/stream_decoder.h new file mode 100644 index 0000000..e54ac28 --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_decoder.h @@ -0,0 +1,21 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_decoder.h +/// \brief Decodes .xz Streams +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_STREAM_DECODER_H +#define LZMA_STREAM_DECODER_H + +#include "common.h" + +extern lzma_ret lzma_stream_decoder_init(lzma_next_coder *next, + lzma_allocator *allocator, uint64_t memlimit, uint32_t flags); + +#endif diff --git a/contrib/xz/src/liblzma/common/stream_encoder.c b/contrib/xz/src/liblzma/common/stream_encoder.c new file mode 100644 index 0000000..48d91da --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_encoder.c @@ -0,0 +1,331 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_encoder.c +/// \brief Encodes .xz Streams +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_encoder.h" +#include "block_encoder.h" +#include "index_encoder.h" + + +struct lzma_coder_s { + enum { + SEQ_STREAM_HEADER, + SEQ_BLOCK_INIT, + SEQ_BLOCK_HEADER, + SEQ_BLOCK_ENCODE, + SEQ_INDEX_ENCODE, + SEQ_STREAM_FOOTER, + } sequence; + + /// True if Block encoder has been initialized by + /// lzma_stream_encoder_init() or stream_encoder_update() + /// and thus doesn't need to be initialized in stream_encode(). + bool block_encoder_is_initialized; + + /// Block + lzma_next_coder block_encoder; + + /// Options for the Block encoder + lzma_block block_options; + + /// The filter chain currently in use + lzma_filter filters[LZMA_FILTERS_MAX + 1]; + + /// Index encoder. This is separate from Block encoder, because this + /// doesn't take much memory, and when encoding multiple Streams + /// with the same encoding options we avoid reallocating memory. + lzma_next_coder index_encoder; + + /// Index to hold sizes of the Blocks + lzma_index *index; + + /// Read position in buffer[] + size_t buffer_pos; + + /// Total number of bytes in buffer[] + size_t buffer_size; + + /// Buffer to hold Stream Header, Block Header, and Stream Footer. + /// Block Header has biggest maximum size. + uint8_t buffer[LZMA_BLOCK_HEADER_SIZE_MAX]; +}; + + +static lzma_ret +block_encoder_init(lzma_coder *coder, lzma_allocator *allocator) +{ + // Prepare the Block options. Even though Block encoder doesn't need + // compressed_size, uncompressed_size, and header_size to be + // initialized, it is a good idea to do it here, because this way + // we catch if someone gave us Filter ID that cannot be used in + // Blocks/Streams. + coder->block_options.compressed_size = LZMA_VLI_UNKNOWN; + coder->block_options.uncompressed_size = LZMA_VLI_UNKNOWN; + + return_if_error(lzma_block_header_size(&coder->block_options)); + + // Initialize the actual Block encoder. + return lzma_block_encoder_init(&coder->block_encoder, allocator, + &coder->block_options); +} + + +static lzma_ret +stream_encode(lzma_coder *coder, lzma_allocator *allocator, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size, uint8_t *restrict out, + size_t *restrict out_pos, size_t out_size, lzma_action action) +{ + // Main loop + while (*out_pos < out_size) + switch (coder->sequence) { + case SEQ_STREAM_HEADER: + case SEQ_BLOCK_HEADER: + case SEQ_STREAM_FOOTER: + lzma_bufcpy(coder->buffer, &coder->buffer_pos, + coder->buffer_size, out, out_pos, out_size); + if (coder->buffer_pos < coder->buffer_size) + return LZMA_OK; + + if (coder->sequence == SEQ_STREAM_FOOTER) + return LZMA_STREAM_END; + + coder->buffer_pos = 0; + ++coder->sequence; + break; + + case SEQ_BLOCK_INIT: { + if (*in_pos == in_size) { + // If we are requested to flush or finish the current + // Block, return LZMA_STREAM_END immediately since + // there's nothing to do. + if (action != LZMA_FINISH) + return action == LZMA_RUN + ? LZMA_OK : LZMA_STREAM_END; + + // The application had used LZMA_FULL_FLUSH to finish + // the previous Block, but now wants to finish without + // encoding new data, or it is simply creating an + // empty Stream with no Blocks. + // + // Initialize the Index encoder, and continue to + // actually encoding the Index. + return_if_error(lzma_index_encoder_init( + &coder->index_encoder, allocator, + coder->index)); + coder->sequence = SEQ_INDEX_ENCODE; + break; + } + + // Initialize the Block encoder unless it was already + // initialized by lzma_stream_encoder_init() or + // stream_encoder_update(). + if (!coder->block_encoder_is_initialized) + return_if_error(block_encoder_init(coder, allocator)); + + // Make it false so that we don't skip the initialization + // with the next Block. + coder->block_encoder_is_initialized = false; + + // Encode the Block Header. This shouldn't fail since we have + // already initialized the Block encoder. + if (lzma_block_header_encode(&coder->block_options, + coder->buffer) != LZMA_OK) + return LZMA_PROG_ERROR; + + coder->buffer_size = coder->block_options.header_size; + coder->sequence = SEQ_BLOCK_HEADER; + break; + } + + case SEQ_BLOCK_ENCODE: { + static const lzma_action convert[4] = { + LZMA_RUN, + LZMA_SYNC_FLUSH, + LZMA_FINISH, + LZMA_FINISH, + }; + + const lzma_ret ret = coder->block_encoder.code( + coder->block_encoder.coder, allocator, + in, in_pos, in_size, + out, out_pos, out_size, convert[action]); + if (ret != LZMA_STREAM_END || action == LZMA_SYNC_FLUSH) + return ret; + + // Add a new Index Record. + const lzma_vli unpadded_size = lzma_block_unpadded_size( + &coder->block_options); + assert(unpadded_size != 0); + return_if_error(lzma_index_append(coder->index, allocator, + unpadded_size, + coder->block_options.uncompressed_size)); + + coder->sequence = SEQ_BLOCK_INIT; + break; + } + + case SEQ_INDEX_ENCODE: { + // Call the Index encoder. It doesn't take any input, so + // those pointers can be NULL. + const lzma_ret ret = coder->index_encoder.code( + coder->index_encoder.coder, allocator, + NULL, NULL, 0, + out, out_pos, out_size, LZMA_RUN); + if (ret != LZMA_STREAM_END) + return ret; + + // Encode the Stream Footer into coder->buffer. + const lzma_stream_flags stream_flags = { + .version = 0, + .backward_size = lzma_index_size(coder->index), + .check = coder->block_options.check, + }; + + if (lzma_stream_footer_encode(&stream_flags, coder->buffer) + != LZMA_OK) + return LZMA_PROG_ERROR; + + coder->buffer_size = LZMA_STREAM_HEADER_SIZE; + coder->sequence = SEQ_STREAM_FOOTER; + break; + } + + default: + assert(0); + return LZMA_PROG_ERROR; + } + + return LZMA_OK; +} + + +static void +stream_encoder_end(lzma_coder *coder, lzma_allocator *allocator) +{ + lzma_next_end(&coder->block_encoder, allocator); + lzma_next_end(&coder->index_encoder, allocator); + lzma_index_end(coder->index, allocator); + + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + lzma_free(coder, allocator); + return; +} + + +static lzma_ret +stream_encoder_update(lzma_coder *coder, lzma_allocator *allocator, + const lzma_filter *filters, + const lzma_filter *reversed_filters) +{ + if (coder->sequence <= SEQ_BLOCK_INIT) { + // There is no incomplete Block waiting to be finished, + // thus we can change the whole filter chain. Start by + // trying to initialize the Block encoder with the new + // chain. This way we detect if the chain is valid. + coder->block_encoder_is_initialized = false; + coder->block_options.filters = (lzma_filter *)(filters); + const lzma_ret ret = block_encoder_init(coder, allocator); + coder->block_options.filters = coder->filters; + if (ret != LZMA_OK) + return ret; + + coder->block_encoder_is_initialized = true; + + } else if (coder->sequence <= SEQ_BLOCK_ENCODE) { + // We are in the middle of a Block. Try to update only + // the filter-specific options. + return_if_error(coder->block_encoder.update( + coder->block_encoder.coder, allocator, + filters, reversed_filters)); + } else { + // Trying to update the filter chain when we are already + // encoding Index or Stream Footer. + return LZMA_PROG_ERROR; + } + + // Free the copy of the old chain and make a copy of the new chain. + for (size_t i = 0; coder->filters[i].id != LZMA_VLI_UNKNOWN; ++i) + lzma_free(coder->filters[i].options, allocator); + + return lzma_filters_copy(filters, coder->filters, allocator); +} + + +extern lzma_ret +lzma_stream_encoder_init(lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *filters, lzma_check check) +{ + lzma_next_coder_init(&lzma_stream_encoder_init, next, allocator); + + if (filters == NULL) + return LZMA_PROG_ERROR; + + if (next->coder == NULL) { + next->coder = lzma_alloc(sizeof(lzma_coder), allocator); + if (next->coder == NULL) + return LZMA_MEM_ERROR; + + next->code = &stream_encode; + next->end = &stream_encoder_end; + next->update = &stream_encoder_update; + + next->coder->block_encoder = LZMA_NEXT_CODER_INIT; + next->coder->index_encoder = LZMA_NEXT_CODER_INIT; + next->coder->index = NULL; + } + + // Basic initializations + next->coder->sequence = SEQ_STREAM_HEADER; + next->coder->block_options.version = 0; + next->coder->block_options.check = check; + next->coder->filters[0].id = LZMA_VLI_UNKNOWN; + + // Initialize the Index + lzma_index_end(next->coder->index, allocator); + next->coder->index = lzma_index_init(allocator); + if (next->coder->index == NULL) + return LZMA_MEM_ERROR; + + // Encode the Stream Header + lzma_stream_flags stream_flags = { + .version = 0, + .check = check, + }; + return_if_error(lzma_stream_header_encode( + &stream_flags, next->coder->buffer)); + + next->coder->buffer_pos = 0; + next->coder->buffer_size = LZMA_STREAM_HEADER_SIZE; + + // Initialize the Block encoder. This way we detect unsupported + // filter chains when initializing the Stream encoder instead of + // giving an error after Stream Header has already written out. + return stream_encoder_update( + next->coder, allocator, filters, NULL); +} + + +extern LZMA_API(lzma_ret) +lzma_stream_encoder(lzma_stream *strm, + const lzma_filter *filters, lzma_check check) +{ + lzma_next_strm_init(lzma_stream_encoder_init, strm, filters, check); + + strm->internal->supported_actions[LZMA_RUN] = true; + strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; + strm->internal->supported_actions[LZMA_FULL_FLUSH] = true; + strm->internal->supported_actions[LZMA_FINISH] = true; + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/stream_encoder.h b/contrib/xz/src/liblzma/common/stream_encoder.h new file mode 100644 index 0000000..46a7aed --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_encoder.h @@ -0,0 +1,23 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_encoder.h +/// \brief Encodes .xz Streams +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_STREAM_ENCODER_H +#define LZMA_STREAM_ENCODER_H + +#include "common.h" + + +extern lzma_ret lzma_stream_encoder_init( + lzma_next_coder *next, lzma_allocator *allocator, + const lzma_filter *filters, lzma_check check); + +#endif diff --git a/contrib/xz/src/liblzma/common/stream_flags_common.c b/contrib/xz/src/liblzma/common/stream_flags_common.c new file mode 100644 index 0000000..fbe8eb8 --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_flags_common.c @@ -0,0 +1,47 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_flags_common.c +/// \brief Common stuff for Stream flags coders +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_flags_common.h" + + +const uint8_t lzma_header_magic[6] = { 0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00 }; +const uint8_t lzma_footer_magic[2] = { 0x59, 0x5A }; + + +extern LZMA_API(lzma_ret) +lzma_stream_flags_compare( + const lzma_stream_flags *a, const lzma_stream_flags *b) +{ + // We can compare only version 0 structures. + if (a->version != 0 || b->version != 0) + return LZMA_OPTIONS_ERROR; + + // Check type + if ((unsigned int)(a->check) > LZMA_CHECK_ID_MAX + || (unsigned int)(b->check) > LZMA_CHECK_ID_MAX) + return LZMA_PROG_ERROR; + + if (a->check != b->check) + return LZMA_DATA_ERROR; + + // Backward Sizes are compared only if they are known in both. + if (a->backward_size != LZMA_VLI_UNKNOWN + && b->backward_size != LZMA_VLI_UNKNOWN) { + if (!is_backward_size_valid(a) || !is_backward_size_valid(b)) + return LZMA_PROG_ERROR; + + if (a->backward_size != b->backward_size) + return LZMA_DATA_ERROR; + } + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/stream_flags_common.h b/contrib/xz/src/liblzma/common/stream_flags_common.h new file mode 100644 index 0000000..9f3122a3 --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_flags_common.h @@ -0,0 +1,33 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_flags_common.h +/// \brief Common stuff for Stream flags coders +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_STREAM_FLAGS_COMMON_H +#define LZMA_STREAM_FLAGS_COMMON_H + +#include "common.h" + +/// Size of the Stream Flags field +#define LZMA_STREAM_FLAGS_SIZE 2 + +extern const uint8_t lzma_header_magic[6]; +extern const uint8_t lzma_footer_magic[2]; + + +static inline bool +is_backward_size_valid(const lzma_stream_flags *options) +{ + return options->backward_size >= LZMA_BACKWARD_SIZE_MIN + && options->backward_size <= LZMA_BACKWARD_SIZE_MAX + && (options->backward_size & 3) == 0; +} + +#endif diff --git a/contrib/xz/src/liblzma/common/stream_flags_decoder.c b/contrib/xz/src/liblzma/common/stream_flags_decoder.c new file mode 100644 index 0000000..1bc2f97 --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_flags_decoder.c @@ -0,0 +1,82 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_flags_decoder.c +/// \brief Decodes Stream Header and Stream Footer from .xz files +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_flags_common.h" + + +static bool +stream_flags_decode(lzma_stream_flags *options, const uint8_t *in) +{ + // Reserved bits must be unset. + if (in[0] != 0x00 || (in[1] & 0xF0)) + return true; + + options->version = 0; + options->check = in[1] & 0x0F; + + return false; +} + + +extern LZMA_API(lzma_ret) +lzma_stream_header_decode(lzma_stream_flags *options, const uint8_t *in) +{ + // Magic + if (memcmp(in, lzma_header_magic, sizeof(lzma_header_magic)) != 0) + return LZMA_FORMAT_ERROR; + + // Verify the CRC32 so we can distinguish between corrupt + // and unsupported files. + const uint32_t crc = lzma_crc32(in + sizeof(lzma_header_magic), + LZMA_STREAM_FLAGS_SIZE, 0); + if (crc != unaligned_read32le(in + sizeof(lzma_header_magic) + + LZMA_STREAM_FLAGS_SIZE)) + return LZMA_DATA_ERROR; + + // Stream Flags + if (stream_flags_decode(options, in + sizeof(lzma_header_magic))) + return LZMA_OPTIONS_ERROR; + + // Set Backward Size to indicate unknown value. That way + // lzma_stream_flags_compare() can be used to compare Stream Header + // and Stream Footer while keeping it useful also for comparing + // two Stream Footers. + options->backward_size = LZMA_VLI_UNKNOWN; + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_stream_footer_decode(lzma_stream_flags *options, const uint8_t *in) +{ + // Magic + if (memcmp(in + sizeof(uint32_t) * 2 + LZMA_STREAM_FLAGS_SIZE, + lzma_footer_magic, sizeof(lzma_footer_magic)) != 0) + return LZMA_FORMAT_ERROR; + + // CRC32 + const uint32_t crc = lzma_crc32(in + sizeof(uint32_t), + sizeof(uint32_t) + LZMA_STREAM_FLAGS_SIZE, 0); + if (crc != unaligned_read32le(in)) + return LZMA_DATA_ERROR; + + // Stream Flags + if (stream_flags_decode(options, in + sizeof(uint32_t) * 2)) + return LZMA_OPTIONS_ERROR; + + // Backward Size + options->backward_size = unaligned_read32le(in + sizeof(uint32_t)); + options->backward_size = (options->backward_size + 1) * 4; + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/stream_flags_encoder.c b/contrib/xz/src/liblzma/common/stream_flags_encoder.c new file mode 100644 index 0000000..4e71715 --- /dev/null +++ b/contrib/xz/src/liblzma/common/stream_flags_encoder.c @@ -0,0 +1,86 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file stream_flags_encoder.c +/// \brief Encodes Stream Header and Stream Footer for .xz files +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "stream_flags_common.h" + + +static bool +stream_flags_encode(const lzma_stream_flags *options, uint8_t *out) +{ + if ((unsigned int)(options->check) > LZMA_CHECK_ID_MAX) + return true; + + out[0] = 0x00; + out[1] = options->check; + + return false; +} + + +extern LZMA_API(lzma_ret) +lzma_stream_header_encode(const lzma_stream_flags *options, uint8_t *out) +{ + assert(sizeof(lzma_header_magic) + LZMA_STREAM_FLAGS_SIZE + + 4 == LZMA_STREAM_HEADER_SIZE); + + if (options->version != 0) + return LZMA_OPTIONS_ERROR; + + // Magic + memcpy(out, lzma_header_magic, sizeof(lzma_header_magic)); + + // Stream Flags + if (stream_flags_encode(options, out + sizeof(lzma_header_magic))) + return LZMA_PROG_ERROR; + + // CRC32 of the Stream Header + const uint32_t crc = lzma_crc32(out + sizeof(lzma_header_magic), + LZMA_STREAM_FLAGS_SIZE, 0); + + unaligned_write32le(out + sizeof(lzma_header_magic) + + LZMA_STREAM_FLAGS_SIZE, crc); + + return LZMA_OK; +} + + +extern LZMA_API(lzma_ret) +lzma_stream_footer_encode(const lzma_stream_flags *options, uint8_t *out) +{ + assert(2 * 4 + LZMA_STREAM_FLAGS_SIZE + sizeof(lzma_footer_magic) + == LZMA_STREAM_HEADER_SIZE); + + if (options->version != 0) + return LZMA_OPTIONS_ERROR; + + // Backward Size + if (!is_backward_size_valid(options)) + return LZMA_PROG_ERROR; + + unaligned_write32le(out + 4, options->backward_size / 4 - 1); + + // Stream Flags + if (stream_flags_encode(options, out + 2 * 4)) + return LZMA_PROG_ERROR; + + // CRC32 + const uint32_t crc = lzma_crc32( + out + 4, 4 + LZMA_STREAM_FLAGS_SIZE, 0); + + unaligned_write32le(out, crc); + + // Magic + memcpy(out + 2 * 4 + LZMA_STREAM_FLAGS_SIZE, + lzma_footer_magic, sizeof(lzma_footer_magic)); + + return LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/vli_decoder.c b/contrib/xz/src/liblzma/common/vli_decoder.c new file mode 100644 index 0000000..c181828 --- /dev/null +++ b/contrib/xz/src/liblzma/common/vli_decoder.c @@ -0,0 +1,86 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file vli_decoder.c +/// \brief Decodes variable-length integers +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API(lzma_ret) +lzma_vli_decode(lzma_vli *restrict vli, size_t *vli_pos, + const uint8_t *restrict in, size_t *restrict in_pos, + size_t in_size) +{ + // If we haven't been given vli_pos, work in single-call mode. + size_t vli_pos_internal = 0; + if (vli_pos == NULL) { + vli_pos = &vli_pos_internal; + *vli = 0; + + // If there's no input, use LZMA_DATA_ERROR. This way it is + // easy to decode VLIs from buffers that have known size, + // and get the correct error code in case the buffer is + // too short. + if (*in_pos >= in_size) + return LZMA_DATA_ERROR; + + } else { + // Initialize *vli when starting to decode a new integer. + if (*vli_pos == 0) + *vli = 0; + + // Validate the arguments. + if (*vli_pos >= LZMA_VLI_BYTES_MAX + || (*vli >> (*vli_pos * 7)) != 0) + return LZMA_PROG_ERROR;; + + if (*in_pos >= in_size) + return LZMA_BUF_ERROR; + } + + do { + // Read the next byte. Use a temporary variable so that we + // can update *in_pos immediately. + const uint8_t byte = in[*in_pos]; + ++*in_pos; + + // Add the newly read byte to *vli. + *vli += (lzma_vli)(byte & 0x7F) << (*vli_pos * 7); + ++*vli_pos; + + // Check if this is the last byte of a multibyte integer. + if ((byte & 0x80) == 0) { + // We don't allow using variable-length integers as + // padding i.e. the encoding must use the most the + // compact form. + if (byte == 0x00 && *vli_pos > 1) + return LZMA_DATA_ERROR; + + return vli_pos == &vli_pos_internal + ? LZMA_OK : LZMA_STREAM_END; + } + + // There is at least one more byte coming. If we have already + // read maximum number of bytes, the integer is considered + // corrupt. + // + // If we need bigger integers in future, old versions liblzma + // will confusingly indicate the file being corrupt istead of + // unsupported. I suppose it's still better this way, because + // in the foreseeable future (writing this in 2008) the only + // reason why files would appear having over 63-bit integers + // is that the files are simply corrupt. + if (*vli_pos == LZMA_VLI_BYTES_MAX) + return LZMA_DATA_ERROR; + + } while (*in_pos < in_size); + + return vli_pos == &vli_pos_internal ? LZMA_DATA_ERROR : LZMA_OK; +} diff --git a/contrib/xz/src/liblzma/common/vli_encoder.c b/contrib/xz/src/liblzma/common/vli_encoder.c new file mode 100644 index 0000000..f864269 --- /dev/null +++ b/contrib/xz/src/liblzma/common/vli_encoder.c @@ -0,0 +1,69 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file vli_encoder.c +/// \brief Encodes variable-length integers +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API(lzma_ret) +lzma_vli_encode(lzma_vli vli, size_t *vli_pos, + uint8_t *restrict out, size_t *restrict out_pos, + size_t out_size) +{ + // If we haven't been given vli_pos, work in single-call mode. + size_t vli_pos_internal = 0; + if (vli_pos == NULL) { + vli_pos = &vli_pos_internal; + + // In single-call mode, we expect that the caller has + // reserved enough output space. + if (*out_pos >= out_size) + return LZMA_PROG_ERROR; + } else { + // This never happens when we are called by liblzma, but + // may happen if called directly from an application. + if (*out_pos >= out_size) + return LZMA_BUF_ERROR; + } + + // Validate the arguments. + if (*vli_pos >= LZMA_VLI_BYTES_MAX || vli > LZMA_VLI_MAX) + return LZMA_PROG_ERROR; + + // Shift vli so that the next bits to encode are the lowest. In + // single-call mode this never changes vli since *vli_pos is zero. + vli >>= *vli_pos * 7; + + // Write the non-last bytes in a loop. + while (vli >= 0x80) { + // We don't need *vli_pos during this function call anymore, + // but update it here so that it is ready if we need to + // return before the whole integer has been decoded. + ++*vli_pos; + assert(*vli_pos < LZMA_VLI_BYTES_MAX); + + // Write the next byte. + out[*out_pos] = (uint8_t)(vli) | 0x80; + vli >>= 7; + + if (++*out_pos == out_size) + return vli_pos == &vli_pos_internal + ? LZMA_PROG_ERROR : LZMA_OK; + } + + // Write the last byte. + out[*out_pos] = (uint8_t)(vli); + ++*out_pos; + ++*vli_pos; + + return vli_pos == &vli_pos_internal ? LZMA_OK : LZMA_STREAM_END; + +} diff --git a/contrib/xz/src/liblzma/common/vli_size.c b/contrib/xz/src/liblzma/common/vli_size.c new file mode 100644 index 0000000..ec1b4fa --- /dev/null +++ b/contrib/xz/src/liblzma/common/vli_size.c @@ -0,0 +1,30 @@ +/////////////////////////////////////////////////////////////////////////////// +// +/// \file vli_size.c +/// \brief Calculates the encoded size of a variable-length integer +// +// Author: Lasse Collin +// +// This file has been put into the public domain. +// You can do whatever you want with this file. +// +/////////////////////////////////////////////////////////////////////////////// + +#include "common.h" + + +extern LZMA_API(uint32_t) +lzma_vli_size(lzma_vli vli) +{ + if (vli > LZMA_VLI_MAX) + return 0; + + uint32_t i = 0; + do { + vli >>= 7; + ++i; + } while (vli != 0); + + assert(i <= LZMA_VLI_BYTES_MAX); + return i; +} |
