diff options
Diffstat (limited to 'libavcodec/vmdvideo.c')
-rw-r--r-- | libavcodec/vmdvideo.c | 91 |
1 files changed, 48 insertions, 43 deletions
diff --git a/libavcodec/vmdvideo.c b/libavcodec/vmdvideo.c index aaeff43..fa0fbe3 100644 --- a/libavcodec/vmdvideo.c +++ b/libavcodec/vmdvideo.c @@ -1,20 +1,21 @@ /* * Sierra VMD video decoder + * Copyright (c) 2004 The FFmpeg Project * - * This file is part of Libav. + * This file is part of FFmpeg. * - * Libav is free software; you can redistribute it and/or + * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * - * Libav is distributed in the hope that it will be useful, + * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public - * License along with Libav; if not, write to the Free Software + * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ @@ -62,7 +63,7 @@ typedef struct VmdVideoContext { #define QUEUE_SIZE 0x1000 #define QUEUE_MASK 0x0FFF -static void lz_unpack(const unsigned char *src, int src_len, +static int lz_unpack(const unsigned char *src, int src_len, unsigned char *dest, int dest_len) { unsigned char *d; @@ -83,9 +84,9 @@ static void lz_unpack(const unsigned char *src, int src_len, dataleft = bytestream2_get_le32(&gb); memset(queue, 0x20, QUEUE_SIZE); if (bytestream2_get_bytes_left(&gb) < 4) - return; + return AVERROR_INVALIDDATA; if (bytestream2_peek_le32(&gb) == 0x56781234) { - bytestream2_get_le32(&gb); + bytestream2_skipu(&gb, 4); qpos = 0x111; speclen = 0xF + 3; } else { @@ -96,8 +97,8 @@ static void lz_unpack(const unsigned char *src, int src_len, while (dataleft > 0 && bytestream2_get_bytes_left(&gb) > 0) { tag = bytestream2_get_byteu(&gb); if ((tag == 0xFF) && (dataleft > 8)) { - if (d + 8 > d_end || bytestream2_get_bytes_left(&gb) < 8) - return; + if (d_end - d < 8 || bytestream2_get_bytes_left(&gb) < 8) + return AVERROR_INVALIDDATA; for (i = 0; i < 8; i++) { queue[qpos++] = *d++ = bytestream2_get_byteu(&gb); qpos &= QUEUE_MASK; @@ -108,9 +109,9 @@ static void lz_unpack(const unsigned char *src, int src_len, if (dataleft == 0) break; if (tag & 0x01) { - if (d + 1 > d_end || bytestream2_get_bytes_left(&gb) < 1) - return; - queue[qpos++] = *d++ = bytestream2_get_byte(&gb); + if (d_end - d < 1 || bytestream2_get_bytes_left(&gb) < 1) + return AVERROR_INVALIDDATA; + queue[qpos++] = *d++ = bytestream2_get_byteu(&gb); qpos &= QUEUE_MASK; dataleft--; } else { @@ -120,8 +121,8 @@ static void lz_unpack(const unsigned char *src, int src_len, if (chainlen == speclen) { chainlen = bytestream2_get_byte(&gb) + 0xF + 3; } - if (d + chainlen > d_end) - return; + if (d_end - d < chainlen) + return AVERROR_INVALIDDATA; for (j = 0; j < chainlen; j++) { *d = queue[chainofs++ & QUEUE_MASK]; queue[qpos++] = *d++; @@ -133,10 +134,10 @@ static void lz_unpack(const unsigned char *src, int src_len, } } } + return d - dest; } - static int rle_unpack(const unsigned char *src, unsigned char *dest, - int src_count, int src_size, int dest_len) + int src_count, int src_size, int dest_len) { unsigned char *pd; int i, l, used = 0; @@ -159,12 +160,12 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest, l = bytestream2_get_byteu(&gb); if (l & 0x80) { l = (l & 0x7F) * 2; - if (pd + l > dest_end || bytestream2_get_bytes_left(&gb) < l) + if (dest_end - pd < l || bytestream2_get_bytes_left(&gb) < l) return bytestream2_tell(&gb); - bytestream2_get_buffer(&gb, pd, l); + bytestream2_get_bufferu(&gb, pd, l); pd += l; } else { - if (pd + l > dest_end || bytestream2_get_bytes_left(&gb) < 2) + if (dest_end - pd < 2*l || bytestream2_get_bytes_left(&gb) < 2) return bytestream2_tell(&gb); run_val = bytestream2_get_ne16(&gb); for (i = 0; i < l; i++) { @@ -200,6 +201,16 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) frame_y = AV_RL16(&s->buf[8]); frame_width = AV_RL16(&s->buf[10]) - frame_x + 1; frame_height = AV_RL16(&s->buf[12]) - frame_y + 1; + + if ((frame_width == s->avctx->width && frame_height == s->avctx->height) && + (frame_x || frame_y)) { + + s->x_off = frame_x; + s->y_off = frame_y; + } + frame_x -= s->x_off; + frame_y -= s->y_off; + if (frame_x < 0 || frame_width < 0 || frame_x >= s->avctx->width || frame_width > s->avctx->width || @@ -219,15 +230,6 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) return AVERROR_INVALIDDATA; } - if ((frame_width == s->avctx->width && frame_height == s->avctx->height) && - (frame_x || frame_y)) { - - s->x_off = frame_x; - s->y_off = frame_y; - } - frame_x -= s->x_off; - frame_y -= s->y_off; - /* if only a certain region will be updated, copy the entire previous * frame before the decode */ if (s->prev_frame->data[0] && @@ -248,13 +250,13 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) r = bytestream2_get_byteu(&gb) * 4; g = bytestream2_get_byteu(&gb) * 4; b = bytestream2_get_byteu(&gb) * 4; - palette32[i] = (r << 16) | (g << 8) | (b); + palette32[i] = 0xFFU << 24 | (r << 16) | (g << 8) | (b); + palette32[i] |= palette32[i] >> 6 & 0x30303; } } else { av_log(s->avctx, AV_LOG_ERROR, "Incomplete palette\n"); return AVERROR_INVALIDDATA; } - s->size -= PALETTE_COUNT * 3 + 2; } if (!s->size) @@ -265,15 +267,18 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) return AVERROR_INVALIDDATA; meth = bytestream2_get_byteu(&gb); if (meth & 0x80) { + int size; if (!s->unpack_buffer_size) { av_log(s->avctx, AV_LOG_ERROR, "Trying to unpack LZ-compressed frame with no LZ buffer\n"); return AVERROR_INVALIDDATA; } - lz_unpack(gb.buffer, bytestream2_get_bytes_left(&gb), - s->unpack_buffer, s->unpack_buffer_size); + size = lz_unpack(gb.buffer, bytestream2_get_bytes_left(&gb), + s->unpack_buffer, s->unpack_buffer_size); + if (size < 0) + return size; meth &= 0x7F; - bytestream2_init(&gb, s->unpack_buffer, s->unpack_buffer_size); + bytestream2_init(&gb, s->unpack_buffer, size); } dp = &frame->data[0][frame_y * frame->linesize[0] + frame_x]; @@ -289,7 +294,7 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) if (ofs + len > frame_width || bytestream2_get_bytes_left(&gb) < len) return AVERROR_INVALIDDATA; - bytestream2_get_buffer(&gb, &dp[ofs], len); + bytestream2_get_bufferu(&gb, &dp[ofs], len); ofs += len; } else { /* interframe pixel copy */ @@ -301,7 +306,7 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) } while (ofs < frame_width); if (ofs > frame_width) { av_log(s->avctx, AV_LOG_ERROR, - "VMD video: offset > width (%d > %d)\n", + "offset > width (%d > %d)\n", ofs, frame_width); return AVERROR_INVALIDDATA; } @@ -347,7 +352,7 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) } while (ofs < frame_width); if (ofs > frame_width) { av_log(s->avctx, AV_LOG_ERROR, - "VMD video: offset > width (%d > %d)\n", + "offset > width (%d > %d)\n", ofs, frame_width); return AVERROR_INVALIDDATA; } @@ -364,7 +369,8 @@ static av_cold int vmdvideo_decode_end(AVCodecContext *avctx) VmdVideoContext *s = avctx->priv_data; av_frame_free(&s->prev_frame); - av_free(s->unpack_buffer); + av_freep(&s->unpack_buffer); + s->unpack_buffer_size = 0; return 0; } @@ -384,9 +390,9 @@ static av_cold int vmdvideo_decode_init(AVCodecContext *avctx) /* make sure the VMD header made it */ if (s->avctx->extradata_size != VMD_HEADER_SIZE) { - av_log(s->avctx, AV_LOG_ERROR, "VMD video: expected extradata size of %d\n", + av_log(s->avctx, AV_LOG_ERROR, "expected extradata size of %d\n", VMD_HEADER_SIZE); - return -1; + return AVERROR_INVALIDDATA; } vmd_header = (unsigned char *)avctx->extradata; @@ -404,7 +410,8 @@ static av_cold int vmdvideo_decode_init(AVCodecContext *avctx) r = raw_palette[palette_index++] * 4; g = raw_palette[palette_index++] * 4; b = raw_palette[palette_index++] * 4; - palette32[i] = (r << 16) | (g << 8) | (b); + palette32[i] = 0xFFU << 24 | (r << 16) | (g << 8) | (b); + palette32[i] |= palette32[i] >> 6 & 0x30303; } s->prev_frame = av_frame_alloc(); @@ -432,10 +439,8 @@ static int vmdvideo_decode_frame(AVCodecContext *avctx, if (buf_size < 16) return AVERROR_INVALIDDATA; - if ((ret = ff_get_buffer(avctx, frame, AV_GET_BUFFER_FLAG_REF)) < 0) { - av_log(s->avctx, AV_LOG_ERROR, "VMD Video: get_buffer() failed\n"); + if ((ret = ff_get_buffer(avctx, frame, AV_GET_BUFFER_FLAG_REF)) < 0) return ret; - } if ((ret = vmd_decode(s, frame)) < 0) return ret; |