bink: Split read_dct_coeffs()

This works around type aliasing violations and related warnings.
Also add some missing error checking.

1 files changed, 28 insertions, 14 deletions

diff --git a/libavcodec/bink.c b/libavcodec/bink.c
index 011d2d8..98fc46e 100644
--- a/libavcodec/bink.c
+++ b/libavcodec/bink.c
@@ -595,17 +595,15 @@ static inline int binkb_get_value(BinkContext *c, int bundle_num)
  * @return 0 for success, negative value in other cases
  */
 static int read_dct_coeffs(BitstreamContext *bc, int32_t block[64],
-                           const uint8_t *scan,
-                           const int32_t quant_matrices[16][64], int q)
+                           const uint8_t *scan, int *coef_count_,
+                           int coef_idx[64], int q)
 {
     int coef_list[128];
     int mode_list[128];
     int i, t, bits, ccoef, mode;
     int list_start = 64, list_end = 64, list_pos;
     int coef_count = 0;
-    int coef_idx[64];
     int quant_idx;
-    const int32_t *quant;
 
     coef_list[list_end] = 4;  mode_list[list_end++] = 0;
     coef_list[list_end] = 24; mode_list[list_end++] = 0;
@@ -681,15 +679,21 @@ static int read_dct_coeffs(BitstreamContext *bc, int32_t block[64],
     if (quant_idx >= 16)
         return AVERROR_INVALIDDATA;
 
-    quant = quant_matrices[quant_idx];
+    *coef_count_ = coef_count;
 
+    return quant_idx;
+}
+
+static void unquantize_dct_coeffs(int32_t block[64], const int32_t quant[64],
+                                  int coef_count, int coef_idx[64],
+                                  const uint8_t *scan)
+{
+    int i;
     block[0] = (block[0] * quant[0]) >> 11;
     for (i = 0; i < coef_count; i++) {
         int idx = coef_idx[i];
         block[scan[idx]] = (block[scan[idx]] * quant[idx]) >> 11;
     }
-
-    return 0;
 }
 
 /**
@@ -806,7 +810,7 @@ static int binkb_decode_plane(BinkContext *c, AVFrame *frame, BitstreamContext *
     LOCAL_ALIGNED_16(int32_t, dctblock, [64]);
     int coordmap[64];
     int ybias = is_key ? -15 : 0;
-    int qp;
+    int qp, quant_idx, coef_count, coef_idx[64];
 
     const int stride = frame->linesize[plane_idx];
     int bw = is_chroma ? (c->avctx->width  + 15) >> 4 : (c->avctx->width  + 7) >> 3;
@@ -859,7 +863,9 @@ static int binkb_decode_plane(BinkContext *c, AVFrame *frame, BitstreamContext *
                 memset(dctblock, 0, sizeof(*dctblock) * 64);
                 dctblock[0] = binkb_get_value(c, BINKB_SRC_INTRA_DC);
                 qp = binkb_get_value(c, BINKB_SRC_INTRA_Q);
-                read_dct_coeffs(bc, dctblock, bink_scan, binkb_intra_quant, qp);
+                if ((quant_idx = read_dct_coeffs(bc, dctblock, bink_scan, &coef_count, coef_idx, qp)) < 0)
+                    return quant_idx;
+                unquantize_dct_coeffs(dctblock, binkb_intra_quant[quant_idx], coef_count, coef_idx, bink_scan);
                 c->binkdsp.idct_put(dst, stride, dctblock);
                 break;
             case 3:
@@ -892,7 +898,9 @@ static int binkb_decode_plane(BinkContext *c, AVFrame *frame, BitstreamContext *
                 memset(dctblock, 0, sizeof(*dctblock) * 64);
                 dctblock[0] = binkb_get_value(c, BINKB_SRC_INTER_DC);
                 qp = binkb_get_value(c, BINKB_SRC_INTER_Q);
-                read_dct_coeffs(bc, dctblock, bink_scan, binkb_inter_quant, qp);
+                if ((quant_idx = read_dct_coeffs(bc, dctblock, bink_scan, &coef_count, coef_idx, qp)) < 0)
+                    return quant_idx;
+                unquantize_dct_coeffs(dctblock, binkb_inter_quant[quant_idx], coef_count, coef_idx, bink_scan);
                 c->binkdsp.idct_add(dst, stride, dctblock);
                 break;
             case 5:
@@ -966,7 +974,7 @@ static int bink_decode_plane(BinkContext *c, AVFrame *frame, BitstreamContext *b
     LOCAL_ALIGNED_16(int16_t, block, [64]);
     LOCAL_ALIGNED_16(uint8_t, ublock, [64]);
     LOCAL_ALIGNED_16(int32_t, dctblock, [64]);
-    int coordmap[64];
+    int coordmap[64], quant_idx, coef_count, coef_idx[64];
 
     const int stride = frame->linesize[plane_idx];
     int bw = is_chroma ? (c->avctx->width  + 15) >> 4 : (c->avctx->width  + 7) >> 3;
@@ -1052,7 +1060,9 @@ static int bink_decode_plane(BinkContext *c, AVFrame *frame, BitstreamContext *b
                 case INTRA_BLOCK:
                     memset(dctblock, 0, sizeof(*dctblock) * 64);
                     dctblock[0] = get_value(c, BINK_SRC_INTRA_DC);
-                    read_dct_coeffs(bc, dctblock, bink_scan, bink_intra_quant, -1);
+                    if ((quant_idx = read_dct_coeffs(bc, dctblock, bink_scan, &coef_count, coef_idx, -1)) < 0)
+                        return quant_idx;
+                    unquantize_dct_coeffs(dctblock, bink_intra_quant[quant_idx], coef_count, coef_idx, bink_scan);
                     c->binkdsp.idct_put(ublock, 8, dctblock);
                     break;
                 case FILL_BLOCK:
@@ -1125,7 +1135,9 @@ static int bink_decode_plane(BinkContext *c, AVFrame *frame, BitstreamContext *b
             case INTRA_BLOCK:
                 memset(dctblock, 0, sizeof(*dctblock) * 64);
                 dctblock[0] = get_value(c, BINK_SRC_INTRA_DC);
-                read_dct_coeffs(bc, dctblock, bink_scan, bink_intra_quant, -1);
+                if ((quant_idx = read_dct_coeffs(bc, dctblock, bink_scan, &coef_count, coef_idx, -1)) < 0)
+                    return quant_idx;
+                unquantize_dct_coeffs(dctblock, bink_intra_quant[quant_idx], coef_count, coef_idx, bink_scan);
                 c->binkdsp.idct_put(dst, stride, dctblock);
                 break;
             case FILL_BLOCK:
@@ -1139,7 +1151,9 @@ static int bink_decode_plane(BinkContext *c, AVFrame *frame, BitstreamContext *b
                     return ret;
                 memset(dctblock, 0, sizeof(*dctblock) * 64);
                 dctblock[0] = get_value(c, BINK_SRC_INTER_DC);
-                read_dct_coeffs(bc, dctblock, bink_scan, bink_inter_quant, -1);
+                if ((quant_idx = read_dct_coeffs(bc, dctblock, bink_scan, &coef_count, coef_idx, -1)) < 0)
+                    return quant_idx;
+                unquantize_dct_coeffs(dctblock, bink_inter_quant[quant_idx], coef_count, coef_idx, bink_scan);
                 c->binkdsp.idct_add(dst, stride, dctblock);
                 break;
             case PATTERN_BLOCK: