From 91ec1c6cc35e3c7950fa5acf80568672ad538f14 Mon Sep 17 00:00:00 2001 From: Nicolas George Date: Sun, 29 Jul 2012 00:12:27 +0200 Subject: loco: take decode overflow into account. Commit 2bf0982 introduced an overflow check in loco_decode_plane, but the error code is never taken into account, leading to completely idiotic return values. --- libavcodec/loco.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/loco.c b/libavcodec/loco.c index 9f14757..cc8ce21 100644 --- a/libavcodec/loco.c +++ b/libavcodec/loco.c @@ -179,51 +179,56 @@ static int decode_frame(AVCodecContext *avctx, } p->key_frame = 1; +#define ADVANCE_BY_DECODED do { \ + if (decoded < 0) goto stop; \ + buf += decoded; buf_size -= decoded; \ +} while(0) switch(l->mode) { case LOCO_CYUY2: case LOCO_YUY2: case LOCO_UYVY: decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height, p->linesize[0], buf, buf_size, 1); - buf += decoded; buf_size -= decoded; + ADVANCE_BY_DECODED; decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height, p->linesize[1], buf, buf_size, 1); - buf += decoded; buf_size -= decoded; + ADVANCE_BY_DECODED; decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height, p->linesize[2], buf, buf_size, 1); break; case LOCO_CYV12: case LOCO_YV12: decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height, p->linesize[0], buf, buf_size, 1); - buf += decoded; buf_size -= decoded; + ADVANCE_BY_DECODED; decoded = loco_decode_plane(l, p->data[2], avctx->width / 2, avctx->height / 2, p->linesize[2], buf, buf_size, 1); - buf += decoded; buf_size -= decoded; + ADVANCE_BY_DECODED; decoded = loco_decode_plane(l, p->data[1], avctx->width / 2, avctx->height / 2, p->linesize[1], buf, buf_size, 1); break; case LOCO_CRGB: case LOCO_RGB: decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1), avctx->width, avctx->height, -p->linesize[0], buf, buf_size, 3); - buf += decoded; buf_size -= decoded; + ADVANCE_BY_DECODED; decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 1, avctx->width, avctx->height, -p->linesize[0], buf, buf_size, 3); - buf += decoded; buf_size -= decoded; + ADVANCE_BY_DECODED; decoded = loco_decode_plane(l, p->data[0] + p->linesize[0]*(avctx->height-1) + 2, avctx->width, avctx->height, -p->linesize[0], buf, buf_size, 3); break; case LOCO_CRGBA: case LOCO_RGBA: decoded = loco_decode_plane(l, p->data[0], avctx->width, avctx->height, p->linesize[0], buf, buf_size, 4); - buf += decoded; buf_size -= decoded; + ADVANCE_BY_DECODED; decoded = loco_decode_plane(l, p->data[0] + 1, avctx->width, avctx->height, p->linesize[0], buf, buf_size, 4); - buf += decoded; buf_size -= decoded; + ADVANCE_BY_DECODED; decoded = loco_decode_plane(l, p->data[0] + 2, avctx->width, avctx->height, p->linesize[0], buf, buf_size, 4); - buf += decoded; buf_size -= decoded; + ADVANCE_BY_DECODED; decoded = loco_decode_plane(l, p->data[0] + 3, avctx->width, avctx->height, p->linesize[0], buf, buf_size, 4); break; } +stop: *data_size = sizeof(AVFrame); *(AVFrame*)data = l->pic; -- cgit v1.1