dnxhddec: use dequantization formula from specs

The current one, while correct, does not yield the best possible
results. The specificiations suggest another formula, which results
in quality gains in the decoded output from fate tests. This
justifies changing said formula.

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

16 files changed, 31 insertions, 31 deletions

diff --git a/tests/ref/vsynth/vsynth1-dnxhd-1080i b/tests/ref/vsynth/vsynth1-dnxhd-1080i
index 28d55b6..02f989f 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-1080i
+++ b/tests/ref/vsynth/vsynth1-dnxhd-1080i
@@ -1,4 +1,4 @@
 a0234e0a8516d958f423b119aa9e35c4 *tests/data/fate/vsynth1-dnxhd-1080i.mov
 3031911 tests/data/fate/vsynth1-dnxhd-1080i.mov
-a09132c6db44f415e831dcaa630a351b *tests/data/fate/vsynth1-dnxhd-1080i.out.rawvideo
-stddev:    6.29 PSNR: 32.15 MAXDIFF:   64 bytes:  7603200/   760320
+fed9ed2a5179c9df0ef58772b025e303 *tests/data/fate/vsynth1-dnxhd-1080i.out.rawvideo
+stddev:    6.18 PSNR: 32.31 MAXDIFF:   64 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth1-dnxhd-1080i-colr b/tests/ref/vsynth/vsynth1-dnxhd-1080i-colr
index 16d8953..ac42966 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-1080i-colr
+++ b/tests/ref/vsynth/vsynth1-dnxhd-1080i-colr
@@ -1,4 +1,4 @@
 5fccdb16c0f14dea1b6b603bac90b97e *tests/data/fate/vsynth1-dnxhd-1080i-colr.mov
 3031929 tests/data/fate/vsynth1-dnxhd-1080i-colr.mov
-5835dff88cb84e83bbe70b5ed5edd5ab *tests/data/fate/vsynth1-dnxhd-1080i-colr.out.rawvideo
-stddev:    5.79 PSNR: 32.87 MAXDIFF:   56 bytes:  7603200/   760320
+6f2d5429ffc4529a76acfeb28b560542 *tests/data/fate/vsynth1-dnxhd-1080i-colr.out.rawvideo
+stddev:    5.65 PSNR: 33.09 MAXDIFF:   55 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth1-dnxhd-720p b/tests/ref/vsynth/vsynth1-dnxhd-720p
index fd77e86..16cf20c 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-720p
+++ b/tests/ref/vsynth/vsynth1-dnxhd-720p
@@ -1,4 +1,4 @@
 af03d57b8320568027162132643f7814 *tests/data/fate/vsynth1-dnxhd-720p.dnxhd
 2293760 tests/data/fate/vsynth1-dnxhd-720p.dnxhd
-f074f1b5ed394871b3c73184ad55b895 *tests/data/fate/vsynth1-dnxhd-720p.out.rawvideo
-stddev:    6.26 PSNR: 32.19 MAXDIFF:   65 bytes:  7603200/   760320
+5f9fba5bacda81e77a72d8a816612564 *tests/data/fate/vsynth1-dnxhd-720p.out.rawvideo
+stddev:    6.22 PSNR: 32.24 MAXDIFF:   64 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth1-dnxhd-720p-10bit b/tests/ref/vsynth/vsynth1-dnxhd-720p-10bit
index 8a8f639..ab58807 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-720p-10bit
+++ b/tests/ref/vsynth/vsynth1-dnxhd-720p-10bit
@@ -1,4 +1,4 @@
 f8c4b7aa165a80df2485d526161290a3 *tests/data/fate/vsynth1-dnxhd-720p-10bit.dnxhd
 2293760 tests/data/fate/vsynth1-dnxhd-720p-10bit.dnxhd
-ec26a6cbf53e38ffb9d5c51cbfbf4f7c *tests/data/fate/vsynth1-dnxhd-720p-10bit.out.rawvideo
-stddev:    6.27 PSNR: 32.18 MAXDIFF:   64 bytes:  7603200/   760320
+3cc84f9e8d2e704475b410de27dd9951 *tests/data/fate/vsynth1-dnxhd-720p-10bit.out.rawvideo
+stddev:    6.23 PSNR: 32.23 MAXDIFF:   64 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth1-dnxhd-720p-rd b/tests/ref/vsynth/vsynth1-dnxhd-720p-rd
index 0422776..f030e92 100644
--- a/tests/ref/vsynth/vsynth1-dnxhd-720p-rd
+++ b/tests/ref/vsynth/vsynth1-dnxhd-720p-rd
@@ -1,4 +1,4 @@
 276e5175376051218b0e3eb36f9e9a63 *tests/data/fate/vsynth1-dnxhd-720p-rd.dnxhd
 2293760 tests/data/fate/vsynth1-dnxhd-720p-rd.dnxhd
-28662df973b289798bf6069fbbee8071 *tests/data/fate/vsynth1-dnxhd-720p-rd.out.rawvideo
-stddev:    6.26 PSNR: 32.19 MAXDIFF:   65 bytes:  7603200/   760320
+1c75ad86ccd5cc2112da7d43ace50dbf *tests/data/fate/vsynth1-dnxhd-720p-rd.out.rawvideo
+stddev:    6.23 PSNR: 32.23 MAXDIFF:   64 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-1080i b/tests/ref/vsynth/vsynth2-dnxhd-1080i
index c3a5073..eabb6a2 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-1080i
+++ b/tests/ref/vsynth/vsynth2-dnxhd-1080i
@@ -1,4 +1,4 @@
 2b75889122f8d918e1b068d128b618ca *tests/data/fate/vsynth2-dnxhd-1080i.mov
 3031911 tests/data/fate/vsynth2-dnxhd-1080i.mov
-099001db73036eeb9545c463cf90f0ba *tests/data/fate/vsynth2-dnxhd-1080i.out.rawvideo
-stddev:    1.53 PSNR: 44.43 MAXDIFF:   31 bytes:  7603200/   760320
+e941d2587cfeccddc450da7f41f7f911 *tests/data/fate/vsynth2-dnxhd-1080i.out.rawvideo
+stddev:    1.50 PSNR: 44.56 MAXDIFF:   31 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-1080i-colr b/tests/ref/vsynth/vsynth2-dnxhd-1080i-colr
index ba5e6a1..06731a8 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-1080i-colr
+++ b/tests/ref/vsynth/vsynth2-dnxhd-1080i-colr
@@ -1,4 +1,4 @@
 f9827e9867b0ea4f7585d8e362a58413 *tests/data/fate/vsynth2-dnxhd-1080i-colr.mov
 3031929 tests/data/fate/vsynth2-dnxhd-1080i-colr.mov
-e4cf5528c993b5e7d57a9d0a4d2cd0c6 *tests/data/fate/vsynth2-dnxhd-1080i-colr.out.rawvideo
-stddev:    1.58 PSNR: 44.15 MAXDIFF:   33 bytes:  7603200/   760320
+ec40a8014b819d02951b2f06bee7b514 *tests/data/fate/vsynth2-dnxhd-1080i-colr.out.rawvideo
+stddev:    1.54 PSNR: 44.33 MAXDIFF:   33 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-720p b/tests/ref/vsynth/vsynth2-dnxhd-720p
index f40da38..fa26c2c 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-720p
+++ b/tests/ref/vsynth/vsynth2-dnxhd-720p
@@ -1,4 +1,4 @@
 3bb2d4fe12b49eae830918d68bde0675 *tests/data/fate/vsynth2-dnxhd-720p.dnxhd
 2293760 tests/data/fate/vsynth2-dnxhd-720p.dnxhd
-903e5a7f2b84c0cd362a0f3a69549989 *tests/data/fate/vsynth2-dnxhd-720p.out.rawvideo
-stddev:    1.53 PSNR: 44.41 MAXDIFF:   31 bytes:  7603200/   760320
+bf3c208086609fea7cb40a9ad97b1fd4 *tests/data/fate/vsynth2-dnxhd-720p.out.rawvideo
+stddev:    1.52 PSNR: 44.49 MAXDIFF:   31 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-720p-10bit b/tests/ref/vsynth/vsynth2-dnxhd-720p-10bit
index c57bf7d..5c21985 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-720p-10bit
+++ b/tests/ref/vsynth/vsynth2-dnxhd-720p-10bit
@@ -1,4 +1,4 @@
 e49cb87f69acc809aee55d64990c84a9 *tests/data/fate/vsynth2-dnxhd-720p-10bit.dnxhd
 2293760 tests/data/fate/vsynth2-dnxhd-720p-10bit.dnxhd
-3eb47758e42db9fc704e1254b7abbeb0 *tests/data/fate/vsynth2-dnxhd-720p-10bit.out.rawvideo
-stddev:    1.56 PSNR: 44.25 MAXDIFF:   31 bytes:  7603200/   760320
+a98c4b69d4d036089a455e147d6922a7 *tests/data/fate/vsynth2-dnxhd-720p-10bit.out.rawvideo
+stddev:    1.54 PSNR: 44.36 MAXDIFF:   31 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth2-dnxhd-720p-rd b/tests/ref/vsynth/vsynth2-dnxhd-720p-rd
index 3d97557..f212998 100644
--- a/tests/ref/vsynth/vsynth2-dnxhd-720p-rd
+++ b/tests/ref/vsynth/vsynth2-dnxhd-720p-rd
@@ -1,4 +1,4 @@
 b723c7412a4c93f500b917ad721f6d21 *tests/data/fate/vsynth2-dnxhd-720p-rd.dnxhd
 2293760 tests/data/fate/vsynth2-dnxhd-720p-rd.dnxhd
-f7d437ea7024700cfd61c40197f44852 *tests/data/fate/vsynth2-dnxhd-720p-rd.out.rawvideo
-stddev:    1.53 PSNR: 44.40 MAXDIFF:   31 bytes:  7603200/   760320
+b7c5de5578317654792c6ea31bfeeb7d *tests/data/fate/vsynth2-dnxhd-720p-rd.out.rawvideo
+stddev:    1.52 PSNR: 44.48 MAXDIFF:   31 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth3-dnxhd-1080i-colr b/tests/ref/vsynth/vsynth3-dnxhd-1080i-colr
index 7cc2298..8d7d3b6 100644
--- a/tests/ref/vsynth/vsynth3-dnxhd-1080i-colr
+++ b/tests/ref/vsynth/vsynth3-dnxhd-1080i-colr
@@ -1,4 +1,4 @@
 ee7a70832f37793b62642f770d988bdb *tests/data/fate/vsynth3-dnxhd-1080i-colr.mov
 3031929 tests/data/fate/vsynth3-dnxhd-1080i-colr.mov
-7dd6b261e439cda21df4f01b45336b41 *tests/data/fate/vsynth3-dnxhd-1080i-colr.out.rawvideo
+f907fd2d48bedbc5283fbfc3fb9f61a0 *tests/data/fate/vsynth3-dnxhd-1080i-colr.out.rawvideo
 stddev:    6.92 PSNR: 31.32 MAXDIFF:   50 bytes:    86700/     8670
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-1080i b/tests/ref/vsynth/vsynth_lena-dnxhd-1080i
index 3693d57..16702db 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-1080i
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-1080i
@@ -1,4 +1,4 @@
 f7412afbcb4454692f7492f6710189e3 *tests/data/fate/vsynth_lena-dnxhd-1080i.mov
 3031911 tests/data/fate/vsynth_lena-dnxhd-1080i.mov
-744ba46da5d4c19a28562ea31061d170 *tests/data/fate/vsynth_lena-dnxhd-1080i.out.rawvideo
-stddev:    1.31 PSNR: 45.77 MAXDIFF:   23 bytes:  7603200/   760320
+7d0ca92f12711535d57eff3609462b31 *tests/data/fate/vsynth_lena-dnxhd-1080i.out.rawvideo
+stddev:    1.29 PSNR: 45.87 MAXDIFF:   22 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-1080i-colr b/tests/ref/vsynth/vsynth_lena-dnxhd-1080i-colr
index 1889786..8e43a3f 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-1080i-colr
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-1080i-colr
@@ -1,4 +1,4 @@
 5ba3ddb58b10e5f0069cb4f82d594695 *tests/data/fate/vsynth_lena-dnxhd-1080i-colr.mov
 3031929 tests/data/fate/vsynth_lena-dnxhd-1080i-colr.mov
-864c3d5f49d9edf66ce8f82a2a6725f6 *tests/data/fate/vsynth_lena-dnxhd-1080i-colr.out.rawvideo
-stddev:    1.36 PSNR: 45.45 MAXDIFF:   22 bytes:  7603200/   760320
+ce4993a69ef55c8c4b18138716f17b6f *tests/data/fate/vsynth_lena-dnxhd-1080i-colr.out.rawvideo
+stddev:    1.33 PSNR: 45.59 MAXDIFF:   22 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-720p b/tests/ref/vsynth/vsynth_lena-dnxhd-720p
index 686be54..eaa1af8 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-720p
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-720p
@@ -1,4 +1,4 @@
 4ca9473a8d106bdfe36e9bf7c516b648 *tests/data/fate/vsynth_lena-dnxhd-720p.dnxhd
 2293760 tests/data/fate/vsynth_lena-dnxhd-720p.dnxhd
-d44c4b08cda8a8042ae345124fdfffcc *tests/data/fate/vsynth_lena-dnxhd-720p.out.rawvideo
-stddev:    1.32 PSNR: 45.68 MAXDIFF:   22 bytes:  7603200/   760320
+9ccd48d24b4f1af2323b65abb6d65d7f *tests/data/fate/vsynth_lena-dnxhd-720p.out.rawvideo
+stddev:    1.31 PSNR: 45.77 MAXDIFF:   22 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-720p-10bit b/tests/ref/vsynth/vsynth_lena-dnxhd-720p-10bit
index 794e9c1..1dcadd8 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-720p-10bit
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-720p-10bit
@@ -1,4 +1,4 @@
 e96fc4a7d994b9369c50da32fd325822 *tests/data/fate/vsynth_lena-dnxhd-720p-10bit.dnxhd
 2293760 tests/data/fate/vsynth_lena-dnxhd-720p-10bit.dnxhd
-0449440eb3e8416840a27deb1a8f80b0 *tests/data/fate/vsynth_lena-dnxhd-720p-10bit.out.rawvideo
-stddev:    1.35 PSNR: 45.47 MAXDIFF:   22 bytes:  7603200/   760320
+2b497215c57558910a605ff8c78430d9 *tests/data/fate/vsynth_lena-dnxhd-720p-10bit.out.rawvideo
+stddev:    1.33 PSNR: 45.61 MAXDIFF:   22 bytes:  7603200/   760320
diff --git a/tests/ref/vsynth/vsynth_lena-dnxhd-720p-rd b/tests/ref/vsynth/vsynth_lena-dnxhd-720p-rd
index 453f68f..b9aa690 100644
--- a/tests/ref/vsynth/vsynth_lena-dnxhd-720p-rd
+++ b/tests/ref/vsynth/vsynth_lena-dnxhd-720p-rd
@@ -1,4 +1,4 @@
 b305b03708e905717b42fc0b304367d4 *tests/data/fate/vsynth_lena-dnxhd-720p-rd.dnxhd
 2293760 tests/data/fate/vsynth_lena-dnxhd-720p-rd.dnxhd
-13de1c5ed025abb5120450e134aa623d *tests/data/fate/vsynth_lena-dnxhd-720p-rd.out.rawvideo
-stddev:    1.32 PSNR: 45.66 MAXDIFF:   22 bytes:  7603200/   760320
+3793391df8448937b17d876b6df0d26d *tests/data/fate/vsynth_lena-dnxhd-720p-rd.out.rawvideo
+stddev:    1.31 PSNR: 45.75 MAXDIFF:   22 bytes:  7603200/   760320