avutil/pixelutils: sad_32x32 sse2/avx2 optimizations.

add ff_pixelutils_sad_32x32_sse2, ff_pixelutils_sad_{a,u}_32x32_sse2,
ff_pixelutils_sad_32x32_avx22, ff_pixelutils_sad_{a,u}_32x32_avx2

use perf record/report profiling, get instructions:u for avx2 sad_32x32:

  72.05%  pixelutils  pixelutils     [.] block_sad_32x32_c
  18.50%  pixelutils  pixelutils     [.] block_sad_16x16_c
   4.78%  pixelutils  pixelutils     [.] block_sad_8x8_c
   2.69%  pixelutils  pixelutils     [.] block_sad_4x4_c
   0.89%  pixelutils  pixelutils     [.] block_sad_2x2_c
   0.16%  pixelutils  pixelutils     [.] ff_pixelutils_sad_32x32_avx2
   0.16%  pixelutils  pixelutils     [.] ff_pixelutils_sad_u_32x32_avx2
   0.12%  pixelutils  pixelutils     [.] ff_pixelutils_sad_a_32x32_avx2

sse2 sad_32x32 instructions:u like:

  71.86%  pixelutils  pixelutils     [.] block_sad_32x32_c
  18.42%  pixelutils  pixelutils     [.] block_sad_16x16_c
   4.81%  pixelutils  pixelutils     [.] block_sad_8x8_c
   2.68%  pixelutils  pixelutils     [.] block_sad_4x4_c
   0.88%  pixelutils  pixelutils     [.] block_sad_2x2_c
   0.29%  pixelutils  pixelutils     [.] ff_pixelutils_sad_32x32_sse2
   0.26%  pixelutils  pixelutils     [.] ff_pixelutils_sad_u_32x32_sse2
   0.23%  pixelutils  pixelutils     [.] ff_pixelutils_sad_a_32x32_sse2

Signed-off-by: Jun Zhao <mypopydev@gmail.com>

2 files changed, 249 insertions, 0 deletions

diff --git a/libavutil/x86/pixelutils.asm b/libavutil/x86/pixelutils.asm
index 171a3d1..bbecf0e 100644
--- a/libavutil/x86/pixelutils.asm
+++ b/libavutil/x86/pixelutils.asm
@@ -163,3 +163,222 @@ cglobal pixelutils_sad_%1_16x16, 4,4,3, src1, stride1, src2, stride2
 
 SAD_XMM_16x16 a
 SAD_XMM_16x16 u
+
+
+%macro PROCESS_SAD_32x4_U 0
+    movu    m1,  [r2]
+    movu    m2,  [r2 + 16]
+    movu    m3,  [r0]
+    movu    m4,  [r0 + 16]
+    psadbw  m1,  m3
+    psadbw  m2,  m4
+    paddd   m1,  m2
+    paddd   m0,  m1
+    lea     r2,  [r2 + r3]
+    lea     r0,  [r0 + r1]
+
+    movu    m1,  [r2]
+    movu    m2,  [r2 + 16]
+    movu    m3,  [r0]
+    movu    m4,  [r0 + 16]
+    psadbw  m1,  m3
+    psadbw  m2,  m4
+    paddd   m1,  m2
+    paddd   m0,  m1
+    lea     r2,  [r2 + r3]
+    lea     r0,  [r0 + r1]
+
+    movu    m1,  [r2]
+    movu    m2,  [r2 + 16]
+    movu    m3,  [r0]
+    movu    m4,  [r0 + 16]
+    psadbw  m1,  m3
+    psadbw  m2,  m4
+    paddd   m1,  m2
+    paddd   m0,  m1
+    lea     r2,  [r2 + r3]
+    lea     r0,  [r0 + r1]
+
+    movu    m1,  [r2]
+    movu    m2,  [r2 + 16]
+    movu    m3,  [r0]
+    movu    m4,  [r0 + 16]
+    psadbw  m1,  m3
+    psadbw  m2,  m4
+    paddd   m1,  m2
+    paddd   m0,  m1
+    lea     r2,  [r2 + r3]
+    lea     r0,  [r0 + r1]
+%endmacro
+
+%macro PROCESS_SAD_32x4 1
+    mov%1   m1,  [r2]
+    mov%1   m2,  [r2 + 16]
+    psadbw  m1,  [r0]
+    psadbw  m2,  [r0 + 16]
+    paddd   m1,  m2
+    paddd   m0,  m1
+    lea     r2,  [r2 + r3]
+    lea     r0,  [r0 + r1]
+
+    mov%1   m1,  [r2]
+    mov%1   m2,  [r2 + 16]
+    psadbw  m1,  [r0]
+    psadbw  m2,  [r0 + 16]
+    paddd   m1,  m2
+    paddd   m0,  m1
+    lea     r2,  [r2 + r3]
+    lea     r0,  [r0 + r1]
+
+    mov%1   m1,  [r2]
+    mov%1   m2,  [r2 + 16]
+    psadbw  m1,  [r0]
+    psadbw  m2,  [r0 + 16]
+    paddd   m1,  m2
+    paddd   m0,  m1
+    lea     r2,  [r2 + r3]
+    lea     r0,  [r0 + r1]
+
+    mov%1   m1,  [r2]
+    mov%1   m2,  [r2 + 16]
+    psadbw  m1,  [r0]
+    psadbw  m2,  [r0 + 16]
+    paddd   m1,  m2
+    paddd   m0,  m1
+    lea     r2,  [r2 + r3]
+    lea     r0,  [r0 + r1]
+%endmacro
+
+;-----------------------------------------------------------------------------
+; int ff_pixelutils_sad_32x32_sse2(const uint8_t *src1, ptrdiff_t stride1,
+;                                  const uint8_t *src2, ptrdiff_t stride2);
+;-----------------------------------------------------------------------------
+INIT_XMM sse2
+cglobal pixelutils_sad_32x32, 4,5,5, src1, stride1, src2, stride2
+    pxor  m0,  m0
+    mov   r4d, 4
+.loop:
+    PROCESS_SAD_32x4_U
+    PROCESS_SAD_32x4_U
+    dec r4d
+    jnz .loop
+
+    movhlps m1,  m0
+    paddd   m0,  m1
+    movd    eax, m0
+    RET
+
+;-------------------------------------------------------------------------------
+; int ff_pixelutils_sad_[au]_32x32_sse2(const uint8_t *src1, ptrdiff_t stride1,
+;                                       const uint8_t *src2, ptrdiff_t stride2);
+;-------------------------------------------------------------------------------
+%macro SAD_XMM_32x32 1
+INIT_XMM sse2
+cglobal pixelutils_sad_%1_32x32, 4,5,3, src1, stride1, src2, stride2
+    pxor  m0,  m0
+    mov   r4d, 4
+.loop:
+    PROCESS_SAD_32x4 %1
+    PROCESS_SAD_32x4 %1
+    dec r4d
+    jnz .loop
+
+    movhlps m1,  m0
+    paddd   m0,  m1
+    movd    eax, m0
+    RET
+%endmacro
+
+SAD_XMM_32x32 a
+SAD_XMM_32x32 u
+
+;-------------------------------------------------------------------------------
+; int ff_pixelutils_sad_32x32_avx2(const uint8_t *src1, ptrdiff_t stride1,
+;                                  const uint8_t *src2, ptrdiff_t stride2);
+;-------------------------------------------------------------------------------
+INIT_YMM avx2
+cglobal pixelutils_sad_32x32, 4,7,5, src1, stride1, src2, stride2
+    pxor            m0, m0
+    mov             r4d, 32/4
+    lea             r5, [stride1q * 3]
+    lea             r6, [stride2q * 3]
+
+.loop:
+    movu           m1, [src1q]               ; row 0 of pix0
+    movu           m2, [src2q]               ; row 0 of pix1
+    movu           m3, [src1q + stride1q]    ; row 1 of pix0
+    movu           m4, [src2q + stride2q]    ; row 1 of pix1
+
+    psadbw         m1, m2
+    psadbw         m3, m4
+    paddd          m0, m1
+    paddd          m0, m3
+
+    movu           m1, [src1q + 2 * stride1q] ; row 2 of pix0
+    movu           m2, [src2q + 2 * stride2q] ; row 2 of pix1
+    movu           m3, [src1q + r5]           ; row 3 of pix0
+    movu           m4, [src2q + r6]           ; row 3 of pix1
+
+    psadbw         m1, m2
+    psadbw         m3, m4
+    paddd          m0, m1
+    paddd          m0, m3
+
+    lea            src2q,     [src2q + 4 * stride2q]
+    lea            src1q,     [src1q + 4 * stride1q]
+
+    dec            r4d
+    jnz           .loop
+
+    vextracti128   xm1, m0, 1
+    paddd          xm0, xm1
+    pshufd         xm1, xm0, 2
+    paddd          xm0, xm1
+    movd           eax, xm0
+    RET
+
+;-------------------------------------------------------------------------------
+; int ff_pixelutils_sad_[au]_32x32_avx2(const uint8_t *src1, ptrdiff_t stride1,
+;                                       const uint8_t *src2, ptrdiff_t stride2);
+;-------------------------------------------------------------------------------
+%macro SAD_AVX2_32x32 1
+INIT_YMM avx2
+cglobal pixelutils_sad_%1_32x32, 4,7,3, src1, stride1, src2, stride2
+    pxor           m0, m0
+    mov            r4d, 32/4
+    lea            r5, [stride1q * 3]
+    lea            r6, [stride2q * 3]
+
+.loop:
+    mov%1          m1, [src2q]                ; row 0 of pix1
+    psadbw         m1, [src1q]
+    mov%1          m2, [src2q + stride2q]     ; row 1 of pix1
+    psadbw         m2, [src1q + stride1q]
+
+    paddd          m0, m1
+    paddd          m0, m2
+
+    mov%1          m1, [src2q + 2 * stride2q] ; row 2 of pix1
+    psadbw         m1, [src1q + 2 * stride1q]
+    mov%1          m2, [src2q + r6]           ; row 3 of pix1
+    psadbw         m2, [src1q + r5]
+
+    paddd          m0, m1
+    paddd          m0, m2
+
+    lea            src2q,     [src2q + 4 * stride2q]
+    lea            src1q,     [src1q + 4 * stride1q]
+
+    dec            r4d
+    jnz           .loop
+
+    vextracti128   xm1, m0, 1
+    paddd          xm0, xm1
+    pshufd         xm1, xm0, 2
+    paddd          xm0, xm1
+    movd           eax, xm0
+    RET
+%endmacro
+
+SAD_AVX2_32x32 a
+SAD_AVX2_32x32 u
diff --git a/libavutil/x86/pixelutils_init.c b/libavutil/x86/pixelutils_init.c
index c24a533..dd05421 100644
--- a/libavutil/x86/pixelutils_init.c
+++ b/libavutil/x86/pixelutils_init.c
@@ -35,6 +35,20 @@ int ff_pixelutils_sad_a_16x16_sse2(const uint8_t *src1, ptrdiff_t stride1,
 int ff_pixelutils_sad_u_16x16_sse2(const uint8_t *src1, ptrdiff_t stride1,
                                    const uint8_t *src2, ptrdiff_t stride2);
 
+int ff_pixelutils_sad_32x32_sse2(const uint8_t *src1, ptrdiff_t stride1,
+                                 const uint8_t *src2, ptrdiff_t stride2);
+int ff_pixelutils_sad_a_32x32_sse2(const uint8_t *src1, ptrdiff_t stride1,
+                                   const uint8_t *src2, ptrdiff_t stride2);
+int ff_pixelutils_sad_u_32x32_sse2(const uint8_t *src1, ptrdiff_t stride1,
+                                   const uint8_t *src2, ptrdiff_t stride2);
+
+int ff_pixelutils_sad_32x32_avx2(const uint8_t *src1, ptrdiff_t stride1,
+                                 const uint8_t *src2, ptrdiff_t stride2);
+int ff_pixelutils_sad_a_32x32_avx2(const uint8_t *src1, ptrdiff_t stride1,
+                                   const uint8_t *src2, ptrdiff_t stride2);
+int ff_pixelutils_sad_u_32x32_avx2(const uint8_t *src1, ptrdiff_t stride1,
+                                   const uint8_t *src2, ptrdiff_t stride2);
+
 void ff_pixelutils_sad_init_x86(av_pixelutils_sad_fn *sad, int aligned)
 {
     int cpu_flags = av_get_cpu_flags();
@@ -61,4 +75,20 @@ void ff_pixelutils_sad_init_x86(av_pixelutils_sad_fn *sad, int aligned)
         case 2: sad[3] = ff_pixelutils_sad_a_16x16_sse2; break; // src1   aligned, src2   aligned
         }
     }
+
+    if (EXTERNAL_SSE2(cpu_flags)) {
+        switch (aligned) {
+        case 0: sad[4] = ff_pixelutils_sad_32x32_sse2;   break; // src1 unaligned, src2 unaligned
+        case 1: sad[4] = ff_pixelutils_sad_u_32x32_sse2; break; // src1   aligned, src2 unaligned
+        case 2: sad[4] = ff_pixelutils_sad_a_32x32_sse2; break; // src1   aligned, src2   aligned
+        }
+    }
+
+    if (EXTERNAL_AVX2(cpu_flags)) {
+        switch (aligned) {
+        case 0: sad[4] = ff_pixelutils_sad_32x32_avx2;   break; // src1 unaligned, src2 unaligned
+        case 1: sad[4] = ff_pixelutils_sad_u_32x32_avx2; break; // src1   aligned, src2 unaligned
+        case 2: sad[4] = ff_pixelutils_sad_a_32x32_avx2; break; // src1   aligned, src2   aligned
+        }
+    }
 }