avutil/pixdesc: Add av_write_image_line2(), av_read_image_line2()

This is needed because of 32bit float formats (which are difficult to
store in 16bits)

This also fixes undefined behavior found by fate

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

3 files changed, 68 insertions, 17 deletions

diff --git a/libavutil/pixdesc.c b/libavutil/pixdesc.c
index 970a832..1c36577 100644
--- a/libavutil/pixdesc.c
+++ b/libavutil/pixdesc.c
@@ -31,19 +31,22 @@
 #include "intreadwrite.h"
 #include "version.h"
 
-void av_read_image_line(uint16_t *dst,
+void av_read_image_line2(void *dst,
                         const uint8_t *data[4], const int linesize[4],
                         const AVPixFmtDescriptor *desc,
                         int x, int y, int c, int w,
-                        int read_pal_component)
+                        int read_pal_component,
+                        int dst_element_size)
 {
     AVComponentDescriptor comp = desc->comp[c];
     int plane = comp.plane;
     int depth = comp.depth;
-    int mask  = (1 << depth) - 1;
+    unsigned mask  = (1ULL << depth) - 1;
     int shift = comp.shift;
     int step  = comp.step;
     int flags = desc->flags;
+    uint16_t *dst16 = dst;
+    uint32_t *dst32 = dst;
 
     if (flags & AV_PIX_FMT_FLAG_BITSTREAM) {
         int skip = x * step + comp.offset;
@@ -57,38 +60,56 @@ void av_read_image_line(uint16_t *dst,
             shift -= step;
             p -= shift >> 3;
             shift &= 7;
-            *dst++ = val;
+            if (dst_element_size == 4) *dst32++ = val;
+            else                       *dst16++ = val;
         }
     } else {
         const uint8_t *p = data[plane] + y * linesize[plane] +
                            x * step + comp.offset;
         int is_8bit = shift + depth <= 8;
+        int is_16bit= shift + depth <=16;
 
         if (is_8bit)
             p += !!(flags & AV_PIX_FMT_FLAG_BE);
 
         while (w--) {
-            int val = is_8bit ? *p :
-                flags & AV_PIX_FMT_FLAG_BE ? AV_RB16(p) : AV_RL16(p);
+            unsigned val;
+            if     (is_8bit)  val = *p;
+            else if(is_16bit) val = flags & AV_PIX_FMT_FLAG_BE ? AV_RB16(p) : AV_RL16(p);
+            else              val = flags & AV_PIX_FMT_FLAG_BE ? AV_RB32(p) : AV_RL32(p);
             val = (val >> shift) & mask;
             if (read_pal_component)
                 val = data[1][4 * val + c];
             p += step;
-            *dst++ = val;
+            if (dst_element_size == 4) *dst32++ = val;
+            else                       *dst16++ = val;
         }
     }
 }
 
-void av_write_image_line(const uint16_t *src,
+void av_read_image_line(uint16_t *dst,
+                        const uint8_t *data[4], const int linesize[4],
+                        const AVPixFmtDescriptor *desc,
+                        int x, int y, int c, int w,
+                        int read_pal_component)
+{
+    av_read_image_line2(dst, data, linesize, desc,x, y, c, w,
+                        read_pal_component,
+                        2);
+}
+
+void av_write_image_line2(const void *src,
                          uint8_t *data[4], const int linesize[4],
                          const AVPixFmtDescriptor *desc,
-                         int x, int y, int c, int w)
+                         int x, int y, int c, int w, int src_element_size)
 {
     AVComponentDescriptor comp = desc->comp[c];
     int plane = comp.plane;
     int depth = comp.depth;
     int step  = comp.step;
     int flags = desc->flags;
+    const uint32_t *src32 = src;
+    const uint16_t *src16 = src;
 
     if (flags & AV_PIX_FMT_FLAG_BITSTREAM) {
         int skip = x * step + comp.offset;
@@ -96,7 +117,7 @@ void av_write_image_line(const uint16_t *src,
         int shift = 8 - depth - (skip & 7);
 
         while (w--) {
-            *p |= *src++ << shift;
+            *p |= (src_element_size == 4 ? *src32++ : *src16++) << shift;
             shift -= step;
             p -= shift >> 3;
             shift &= 7;
@@ -109,17 +130,28 @@ void av_write_image_line(const uint16_t *src,
         if (shift + depth <= 8) {
             p += !!(flags & AV_PIX_FMT_FLAG_BE);
             while (w--) {
-                *p |= (*src++ << shift);
+                *p |= ((src_element_size == 4 ? *src32++ : *src16++) << shift);
                 p += step;
             }
         } else {
             while (w--) {
-                if (flags & AV_PIX_FMT_FLAG_BE) {
-                    uint16_t val = AV_RB16(p) | (*src++ << shift);
-                    AV_WB16(p, val);
+                unsigned s = (src_element_size == 4 ? *src32++ : *src16++);
+                if (shift + depth <= 16) {
+                    if (flags & AV_PIX_FMT_FLAG_BE) {
+                        uint16_t val = AV_RB16(p) | (s << shift);
+                        AV_WB16(p, val);
+                    } else {
+                        uint16_t val = AV_RL16(p) | (s << shift);
+                        AV_WL16(p, val);
+                    }
                 } else {
-                    uint16_t val = AV_RL16(p) | (*src++ << shift);
-                    AV_WL16(p, val);
+                    if (flags & AV_PIX_FMT_FLAG_BE) {
+                        uint32_t val = AV_RB32(p) | (s << shift);
+                        AV_WB32(p, val);
+                    } else {
+                        uint32_t val = AV_RL32(p) | (s << shift);
+                        AV_WL32(p, val);
+                    }
                 }
                 p += step;
             }
@@ -127,6 +159,14 @@ void av_write_image_line(const uint16_t *src,
     }
 }
 
+void av_write_image_line(const uint16_t *src,
+                         uint8_t *data[4], const int linesize[4],
+                         const AVPixFmtDescriptor *desc,
+                         int x, int y, int c, int w)
+{
+    av_write_image_line2(src, data, linesize, desc, x, y, c, w, 2);
+}
+
 #if FF_API_PLUS1_MINUS1
 FF_DISABLE_DEPRECATION_WARNINGS
 #endif
diff --git a/libavutil/pixdesc.h b/libavutil/pixdesc.h
index 4f9c5a2..c055810 100644
--- a/libavutil/pixdesc.h
+++ b/libavutil/pixdesc.h
@@ -343,7 +343,13 @@ char *av_get_pix_fmt_string(char *buf, int buf_size,
  * format writes the values corresponding to the palette
  * component c in data[1] to dst, rather than the palette indexes in
  * data[0]. The behavior is undefined if the format is not paletted.
+ * @param dst_element_size size of elements in dst array (2 or 4 byte)
  */
+void av_read_image_line2(void *dst, const uint8_t *data[4],
+                        const int linesize[4], const AVPixFmtDescriptor *desc,
+                        int x, int y, int c, int w, int read_pal_component,
+                        int dst_element_size);
+
 void av_read_image_line(uint16_t *dst, const uint8_t *data[4],
                         const int linesize[4], const AVPixFmtDescriptor *desc,
                         int x, int y, int c, int w, int read_pal_component);
@@ -361,7 +367,12 @@ void av_read_image_line(uint16_t *dst, const uint8_t *data[4],
  * @param y the vertical coordinate of the first pixel to write
  * @param w the width of the line to write, that is the number of
  * values to write to the image line
+ * @param src_element_size size of elements in src array (2 or 4 byte)
  */
+void av_write_image_line2(const void *src, uint8_t *data[4],
+                         const int linesize[4], const AVPixFmtDescriptor *desc,
+                         int x, int y, int c, int w, int src_element_size);
+
 void av_write_image_line(const uint16_t *src, uint8_t *data[4],
                          const int linesize[4], const AVPixFmtDescriptor *desc,
                          int x, int y, int c, int w);
diff --git a/libavutil/version.h b/libavutil/version.h
index 377714a..8e27e6b 100644
--- a/libavutil/version.h
+++ b/libavutil/version.h
@@ -79,7 +79,7 @@
  */
 
 #define LIBAVUTIL_VERSION_MAJOR  56
-#define LIBAVUTIL_VERSION_MINOR  20
+#define LIBAVUTIL_VERSION_MINOR  21
 #define LIBAVUTIL_VERSION_MICRO 100
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \