From bf39d3b59d85e5734babe48b61b8d92d18188185 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 29 Mar 2012 09:29:03 -0700 Subject: truemotion2: handle out-of-frame motion vectors through edge extension. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libavcodec/truemotion2.c | 117 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 27 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/truemotion2.c b/libavcodec/truemotion2.c index a2170fc..646c222 100644 --- a/libavcodec/truemotion2.c +++ b/libavcodec/truemotion2.c @@ -57,7 +57,9 @@ typedef struct TM2Context{ int *clast; /* data for current and previous frame */ + int *Y1_base, *U1_base, *V1_base, *Y2_base, *U2_base, *V2_base; int *Y1, *U1, *V1, *Y2, *U2, *V2; + int y_stride, uv_stride; int cur; } TM2Context; @@ -348,9 +350,9 @@ static inline int GET_TOK(TM2Context *ctx,int type) { int *Y, *U, *V;\ int Ystride, Ustride, Vstride;\ \ - Ystride = ctx->avctx->width;\ - Vstride = (ctx->avctx->width + 1) >> 1;\ - Ustride = (ctx->avctx->width + 1) >> 1;\ + Ystride = ctx->y_stride;\ + Vstride = ctx->uv_stride;\ + Ustride = ctx->uv_stride;\ Y = (ctx->cur?ctx->Y2:ctx->Y1) + by * 4 * Ystride + bx * 4;\ V = (ctx->cur?ctx->V2:ctx->V1) + by * 2 * Vstride + bx * 2;\ U = (ctx->cur?ctx->U2:ctx->U1) + by * 2 * Ustride + bx * 2;\ @@ -638,6 +640,8 @@ static inline void tm2_motion_block(TM2Context *ctx, AVFrame *pic, int bx, int b mx = GET_TOK(ctx, TM2_MOT); my = GET_TOK(ctx, TM2_MOT); + mx = av_clip(mx, -(bx * 4 + 4), ctx->avctx->width - bx * 4); + my = av_clip(my, -(by * 4 + 4), ctx->avctx->height - by * 4); Yo += my * oYstride + mx; Uo += (my >> 1) * oUstride + (mx >> 1); @@ -678,15 +682,12 @@ static inline void tm2_motion_block(TM2Context *ctx, AVFrame *pic, int bx, int b static int tm2_decode_blocks(TM2Context *ctx, AVFrame *p) { int i, j; - int bw, bh; + int w = ctx->avctx->width, h = ctx->avctx->height, bw = w >> 2, bh = h >> 2, cw = w >> 1; int type; int keyframe = 1; int *Y, *U, *V; uint8_t *dst; - bw = ctx->avctx->width >> 2; - bh = ctx->avctx->height >> 2; - for(i = 0; i < TM2_NUM_STREAMS; i++) ctx->tok_ptrs[i] = 0; @@ -739,17 +740,54 @@ static int tm2_decode_blocks(TM2Context *ctx, AVFrame *p) U = (ctx->cur?ctx->U2:ctx->U1); V = (ctx->cur?ctx->V2:ctx->V1); dst = p->data[0]; - for(j = 0; j < ctx->avctx->height; j++){ - for(i = 0; i < ctx->avctx->width; i++){ + for(j = 0; j < h; j++){ + for(i = 0; i < w; i++){ int y = Y[i], u = U[i >> 1], v = V[i >> 1]; dst[3*i+0] = av_clip_uint8(y + v); dst[3*i+1] = av_clip_uint8(y); dst[3*i+2] = av_clip_uint8(y + u); } - Y += ctx->avctx->width; + + /* horizontal edge extension */ + Y[-4] = Y[-3] = Y[-2] = Y[-1] = Y[0]; + Y[w + 3] = Y[w + 2] = Y[w + 1] = Y[w] = Y[w - 1]; + + /* vertical edge extension */ + if (j == 0) { + memcpy(Y - 4 - 1 * ctx->y_stride, Y - 4, ctx->y_stride); + memcpy(Y - 4 - 2 * ctx->y_stride, Y - 4, ctx->y_stride); + memcpy(Y - 4 - 3 * ctx->y_stride, Y - 4, ctx->y_stride); + memcpy(Y - 4 - 4 * ctx->y_stride, Y - 4, ctx->y_stride); + } else if (j == h - 1) { + memcpy(Y - 4 + 1 * ctx->y_stride, Y - 4, ctx->y_stride); + memcpy(Y - 4 + 2 * ctx->y_stride, Y - 4, ctx->y_stride); + memcpy(Y - 4 + 3 * ctx->y_stride, Y - 4, ctx->y_stride); + memcpy(Y - 4 + 4 * ctx->y_stride, Y - 4, ctx->y_stride); + } + + Y += ctx->y_stride; if (j & 1) { - U += ctx->avctx->width >> 1; - V += ctx->avctx->width >> 1; + /* horizontal edge extension */ + U[-2] = U[-1] = U[0]; + V[-2] = V[-1] = V[0]; + U[cw + 1] = U[cw] = U[cw - 1]; + V[cw + 1] = V[cw] = V[cw - 1]; + + /* vertical edge extension */ + if (j == 1) { + memcpy(U - 2 - 1 * ctx->uv_stride, U - 2, ctx->uv_stride); + memcpy(V - 2 - 1 * ctx->uv_stride, V - 2, ctx->uv_stride); + memcpy(U - 2 - 2 * ctx->uv_stride, U - 2, ctx->uv_stride); + memcpy(V - 2 - 2 * ctx->uv_stride, V - 2, ctx->uv_stride); + } else if (j == h - 1) { + memcpy(U - 2 + 1 * ctx->uv_stride, U - 2, ctx->uv_stride); + memcpy(V - 2 + 1 * ctx->uv_stride, V - 2, ctx->uv_stride); + memcpy(U - 2 + 2 * ctx->uv_stride, U - 2, ctx->uv_stride); + memcpy(V - 2 + 2 * ctx->uv_stride, V - 2, ctx->uv_stride); + } + + U += ctx->uv_stride; + V += ctx->uv_stride; } dst += p->linesize[0]; } @@ -821,7 +859,7 @@ static int decode_frame(AVCodecContext *avctx, static av_cold int decode_init(AVCodecContext *avctx){ TM2Context * const l = avctx->priv_data; - int i; + int i, w = avctx->width, h = avctx->height; if((avctx->width & 3) || (avctx->height & 3)){ av_log(avctx, AV_LOG_ERROR, "Width and height must be multiple of 4\n"); @@ -834,21 +872,46 @@ static av_cold int decode_init(AVCodecContext *avctx){ ff_dsputil_init(&l->dsp, avctx); - l->last = av_malloc(4 * sizeof(int) * (avctx->width >> 2)); - l->clast = av_malloc(4 * sizeof(int) * (avctx->width >> 2)); + l->last = av_malloc(4 * sizeof(*l->last) * (w >> 2)); + l->clast = av_malloc(4 * sizeof(*l->clast) * (w >> 2)); for(i = 0; i < TM2_NUM_STREAMS; i++) { l->tokens[i] = NULL; l->tok_lens[i] = 0; } - l->Y1 = av_malloc(sizeof(int) * avctx->width * avctx->height); - l->U1 = av_malloc(sizeof(int) * ((avctx->width + 1) >> 1) * ((avctx->height + 1) >> 1)); - l->V1 = av_malloc(sizeof(int) * ((avctx->width + 1) >> 1) * ((avctx->height + 1) >> 1)); - l->Y2 = av_malloc(sizeof(int) * avctx->width * avctx->height); - l->U2 = av_malloc(sizeof(int) * ((avctx->width + 1) >> 1) * ((avctx->height + 1) >> 1)); - l->V2 = av_malloc(sizeof(int) * ((avctx->width + 1) >> 1) * ((avctx->height + 1) >> 1)); + w += 8; + h += 8; + l->Y1_base = av_malloc(sizeof(*l->Y1_base) * w * h); + l->Y2_base = av_malloc(sizeof(*l->Y2_base) * w * h); + l->y_stride = w; + w = (w + 1) >> 1; + h = (h + 1) >> 1; + l->U1_base = av_malloc(sizeof(*l->U1_base) * w * h); + l->V1_base = av_malloc(sizeof(*l->V1_base) * w * h); + l->U2_base = av_malloc(sizeof(*l->U2_base) * w * h); + l->V2_base = av_malloc(sizeof(*l->V1_base) * w * h); + l->uv_stride = w; l->cur = 0; + if (!l->Y1_base || !l->Y2_base || !l->U1_base || + !l->V1_base || !l->U2_base || !l->V2_base || + !l->last || !l->clast) { + av_freep(l->Y1_base); + av_freep(l->Y2_base); + av_freep(l->U1_base); + av_freep(l->U2_base); + av_freep(l->V1_base); + av_freep(l->V2_base); + av_freep(l->last); + av_freep(l->clast); + return AVERROR(ENOMEM); + } + l->Y1 = l->Y1_base + l->y_stride * 4 + 4; + l->Y2 = l->Y2_base + l->y_stride * 4 + 4; + l->U1 = l->U1_base + l->uv_stride * 2 + 2; + l->U2 = l->U2_base + l->uv_stride * 2 + 2; + l->V1 = l->V1_base + l->uv_stride * 2 + 2; + l->V2 = l->V2_base + l->uv_stride * 2 + 2; return 0; } @@ -863,12 +926,12 @@ static av_cold int decode_end(AVCodecContext *avctx){ for(i = 0; i < TM2_NUM_STREAMS; i++) av_free(l->tokens[i]); if(l->Y1){ - av_free(l->Y1); - av_free(l->U1); - av_free(l->V1); - av_free(l->Y2); - av_free(l->U2); - av_free(l->V2); + av_free(l->Y1_base); + av_free(l->U1_base); + av_free(l->V1_base); + av_free(l->Y2_base); + av_free(l->U2_base); + av_free(l->V2_base); } if (pic->data[0]) -- cgit v1.1 From b7b1509d06d3696d3b944791227fe198ded0654b Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Thu, 29 Mar 2012 10:25:04 -0700 Subject: truemotion: forbid invalid VLC bitsizes and token values. SHOW_UBITS() is only defined up to n_bits is 25, therefore forbid values larger than this in get_vlc2() (max_bits). tokens[][] can be used as an index in deltas[], which has a size of 64, so ensure the values are smaller than that. This prevents crashes on corrupt bitstreams. Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind CC: libav-stable@libav.org --- libavcodec/truemotion2.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/truemotion2.c b/libavcodec/truemotion2.c index 646c222..67976c8 100644 --- a/libavcodec/truemotion2.c +++ b/libavcodec/truemotion2.c @@ -130,7 +130,7 @@ static int tm2_build_huff_table(TM2Context *ctx, TM2Codes *code) /* check for correct codes parameters */ if((huff.val_bits < 1) || (huff.val_bits > 32) || - (huff.max_bits < 0) || (huff.max_bits > 32)) { + (huff.max_bits < 0) || (huff.max_bits > 25)) { av_log(ctx->avctx, AV_LOG_ERROR, "Incorrect tree parameters - literal length: %i, max code length: %i\n", huff.val_bits, huff.max_bits); return -1; @@ -322,10 +322,21 @@ static int tm2_read_stream(TM2Context *ctx, const uint8_t *buf, int stream_id, i return -1; } ctx->tokens[stream_id][i] = tm2_get_token(&ctx->gb, &codes); + if (stream_id <= TM2_MOT && ctx->tokens[stream_id][i] >= TM2_DELTAS) { + av_log(ctx->avctx, AV_LOG_ERROR, "Invalid delta token index %d for type %d, n=%d\n", + ctx->tokens[stream_id][i], stream_id, i); + return AVERROR_INVALIDDATA; + } } } else { - for(i = 0; i < toks; i++) + for(i = 0; i < toks; i++) { ctx->tokens[stream_id][i] = codes.recode[0]; + if (stream_id <= TM2_MOT && ctx->tokens[stream_id][i] >= TM2_DELTAS) { + av_log(ctx->avctx, AV_LOG_ERROR, "Invalid delta token index %d for type %d, n=%d\n", + ctx->tokens[stream_id][i], stream_id, i); + return AVERROR_INVALIDDATA; + } + } } tm2_free_codes(&codes); @@ -837,9 +848,9 @@ static int decode_frame(AVCodecContext *avctx, return AVERROR_INVALIDDATA; } t = tm2_read_stream(l, swbuf + skip, tm2_stream_order[i], buf_size - skip); - if(t == -1){ + if(t < 0){ av_free(swbuf); - return -1; + return t; } skip += t; } -- cgit v1.1 From 0399fe0fd23ff272282dd093cc2720deb6f87ea8 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 28 Mar 2012 20:00:38 -0700 Subject: lzw: fix potential integer overflow. --- libavcodec/lzw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libavcodec') diff --git a/libavcodec/lzw.c b/libavcodec/lzw.c index b674d4e..8761510 100644 --- a/libavcodec/lzw.c +++ b/libavcodec/lzw.c @@ -102,7 +102,7 @@ void ff_lzw_decode_tail(LZWState *p) if(s->mode == FF_LZW_GIF) { while (s->bs > 0) { - if (s->pbuf + s->bs >= s->ebuf) { + if (s->bs >= s->ebuf - s->pbuf) { s->pbuf = s->ebuf; break; } else { -- cgit v1.1 From 9ad80ef3db41a70a4c272f17216b6e09011368ea Mon Sep 17 00:00:00 2001 From: Diego Biurrun Date: Thu, 29 Mar 2012 13:52:21 +0200 Subject: h264: Make ff_h264_decode_end() static, it is not used externally. Also drop the now unnecessary ff_ prefix from its name. --- libavcodec/h264.c | 6 +++--- libavcodec/h264.h | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/h264.c b/libavcodec/h264.c index d49857e..520a97d 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -4559,7 +4559,7 @@ av_cold void ff_h264_free_context(H264Context *h) av_freep(h->pps_buffers + i); } -av_cold int ff_h264_decode_end(AVCodecContext *avctx) +static av_cold int h264_decode_end(AVCodecContext *avctx) { H264Context *h = avctx->priv_data; MpegEncContext *s = &h->s; @@ -4596,7 +4596,7 @@ AVCodec ff_h264_decoder = { .id = CODEC_ID_H264, .priv_data_size = sizeof(H264Context), .init = ff_h264_decode_init, - .close = ff_h264_decode_end, + .close = h264_decode_end, .decode = decode_frame, .capabilities = /*CODEC_CAP_DRAW_HORIZ_BAND |*/ CODEC_CAP_DR1 | CODEC_CAP_DELAY | CODEC_CAP_SLICE_THREADS | @@ -4615,7 +4615,7 @@ AVCodec ff_h264_vdpau_decoder = { .id = CODEC_ID_H264, .priv_data_size = sizeof(H264Context), .init = ff_h264_decode_init, - .close = ff_h264_decode_end, + .close = h264_decode_end, .decode = decode_frame, .capabilities = CODEC_CAP_DR1 | CODEC_CAP_DELAY | CODEC_CAP_HWACCEL_VDPAU, .flush = flush_dpb, diff --git a/libavcodec/h264.h b/libavcodec/h264.h index 8680f5f..99c46cc 100644 --- a/libavcodec/h264.h +++ b/libavcodec/h264.h @@ -663,7 +663,6 @@ void ff_h264_hl_decode_mb(H264Context *h); int ff_h264_frame_start(H264Context *h); int ff_h264_decode_extradata(H264Context *h); av_cold int ff_h264_decode_init(AVCodecContext *avctx); -av_cold int ff_h264_decode_end(AVCodecContext *avctx); av_cold void ff_h264_decode_init_vlc(void); /** -- cgit v1.1 From b691fd7a4dfca766075c022922a75cdbca4e6d92 Mon Sep 17 00:00:00 2001 From: Diego Biurrun Date: Thu, 29 Mar 2012 14:15:16 +0200 Subject: h264: drop ff_h264_ prefix from static function ff_h264_decode_rbsp_trailing() --- libavcodec/h264.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libavcodec') diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 520a97d..3b1fdb2 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -262,7 +262,7 @@ nsc: * Identify the exact end of the bitstream * @return the length of the trailing, or 0 if damaged */ -static int ff_h264_decode_rbsp_trailing(H264Context *h, const uint8_t *src) +static int decode_rbsp_trailing(H264Context *h, const uint8_t *src) { int v = *src; int r; @@ -4225,7 +4225,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size) dst_length--; bit_length = !dst_length ? 0 : (8 * dst_length - - ff_h264_decode_rbsp_trailing(h, ptr + dst_length - 1)); + decode_rbsp_trailing(h, ptr + dst_length - 1)); if (s->avctx->debug & FF_DEBUG_STARTCODE) av_log(h->s.avctx, AV_LOG_DEBUG, -- cgit v1.1