From f99654d47030daca949cc6bfb14aa8e8455e8e59 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Wed, 5 Oct 2011 06:27:52 -0700 Subject: swscale: reintroduce full precision in 16-bit output. --- libswscale/swscale.c | 16 +++++++++++----- tests/ref/lavfi/pixdesc | 8 ++++---- tests/ref/lavfi/pixfmts_copy | 8 ++++---- tests/ref/lavfi/pixfmts_crop | 4 ++-- tests/ref/lavfi/pixfmts_hflip | 8 ++++---- tests/ref/lavfi/pixfmts_null | 8 ++++---- tests/ref/lavfi/pixfmts_scale | 12 ++++++------ tests/ref/lavfi/pixfmts_vflip | 8 ++++---- 8 files changed, 39 insertions(+), 33 deletions(-) diff --git a/libswscale/swscale.c b/libswscale/swscale.c index 1d9b965..8ed68c5 100644 --- a/libswscale/swscale.c +++ b/libswscale/swscale.c @@ -202,20 +202,26 @@ yuv2yuvX16_c_template(const int16_t *filter, int filterSize, { #define output_pixel(pos, val) \ if (big_endian) { \ - AV_WB16(pos, av_clip_uint16(val >> shift)); \ + AV_WB16(pos, 0x8000 + av_clip_int16(val >> shift)); \ } else { \ - AV_WL16(pos, av_clip_uint16(val >> shift)); \ + AV_WL16(pos, 0x8000 + av_clip_int16(val >> shift)); \ } int i; - int shift = 15 + 16 - output_bits - 1; + int shift = 15 + 16 - output_bits; for (i = 0; i < dstW; i++) { - int val = 1 << (30-output_bits - 1); + int val = 1 << (30-output_bits); int j; + /* range of val is [0,0x7FFFFFFF], so 31 bits, but with lanczos/spline + * filters (or anything with negative coeffs, the range can be slightly + * wider in both directions. To account for this overflow, we subtract + * a constant so it always fits in the signed range (assuming a + * reasonable filterSize), and re-add that at the end. */ + val -= 0x40000000; for (j = 0; j < filterSize; j++) - val += (src[j][i] * filter[j]) >> 1; + val += src[j][i] * filter[j]; output_pixel(&dest[i], val); } diff --git a/tests/ref/lavfi/pixdesc b/tests/ref/lavfi/pixdesc index c4d7890..90dcbf5 100644 --- a/tests/ref/lavfi/pixdesc +++ b/tests/ref/lavfi/pixdesc @@ -40,16 +40,16 @@ yuv420p9le 16543fda8f87d94a6cf857d2e8d4461a yuv422p c9bba4529821d796a6ab09f6a5fd355a yuv422p10be bdc13b630fd668b34c6fe1aae28dfc71 yuv422p10le d0607c260a45c973e6639f4e449730ad -yuv422p16be 4e9b3b3467aeebb6a528cee5966800ed -yuv422p16le f87c81bf16916b64d201359be0b4b6f4 +yuv422p16be 5499502e1c29534a158a1fe60e889f60 +yuv422p16le e3d61fde6978591596bc36b914386623 yuv422p9be 29b71579946940a8c00fa844c9dff507 yuv422p9le 062b7f9cbb972bf36b5bdb1a7623701a yuv440p 5a064afe2b453bb52cdb3f176b1aa1cf yuv444p 0a98447b78fd476aa39686da6a74fa2e yuv444p10be e65cbae7e4f1892c23defbc8e8052cf6 yuv444p10le 767179dd82846cf00ee4c340c9c1ab74 -yuv444p16be 3ad639fff73e56f3b09dd20c335478d6 -yuv444p16le 8a7e66dc91ab7971fd24a9105ff2699b +yuv444p16be 1c6ea2c2f5e539006112ceec3d4e7d90 +yuv444p16le 20f86bc2f68d2b3f1f2b48b97b2189f4 yuv444p9be 6ab31f4c12b533ce318ecdff83cdd054 yuv444p9le f0606604a5c08becab6ba500124c4b7c yuva420p a29884f3f3dfe1e00b961bc17bef3d47 diff --git a/tests/ref/lavfi/pixfmts_copy b/tests/ref/lavfi/pixfmts_copy index c4d7890..90dcbf5 100644 --- a/tests/ref/lavfi/pixfmts_copy +++ b/tests/ref/lavfi/pixfmts_copy @@ -40,16 +40,16 @@ yuv420p9le 16543fda8f87d94a6cf857d2e8d4461a yuv422p c9bba4529821d796a6ab09f6a5fd355a yuv422p10be bdc13b630fd668b34c6fe1aae28dfc71 yuv422p10le d0607c260a45c973e6639f4e449730ad -yuv422p16be 4e9b3b3467aeebb6a528cee5966800ed -yuv422p16le f87c81bf16916b64d201359be0b4b6f4 +yuv422p16be 5499502e1c29534a158a1fe60e889f60 +yuv422p16le e3d61fde6978591596bc36b914386623 yuv422p9be 29b71579946940a8c00fa844c9dff507 yuv422p9le 062b7f9cbb972bf36b5bdb1a7623701a yuv440p 5a064afe2b453bb52cdb3f176b1aa1cf yuv444p 0a98447b78fd476aa39686da6a74fa2e yuv444p10be e65cbae7e4f1892c23defbc8e8052cf6 yuv444p10le 767179dd82846cf00ee4c340c9c1ab74 -yuv444p16be 3ad639fff73e56f3b09dd20c335478d6 -yuv444p16le 8a7e66dc91ab7971fd24a9105ff2699b +yuv444p16be 1c6ea2c2f5e539006112ceec3d4e7d90 +yuv444p16le 20f86bc2f68d2b3f1f2b48b97b2189f4 yuv444p9be 6ab31f4c12b533ce318ecdff83cdd054 yuv444p9le f0606604a5c08becab6ba500124c4b7c yuva420p a29884f3f3dfe1e00b961bc17bef3d47 diff --git a/tests/ref/lavfi/pixfmts_crop b/tests/ref/lavfi/pixfmts_crop index d11720f..cbbcf90 100644 --- a/tests/ref/lavfi/pixfmts_crop +++ b/tests/ref/lavfi/pixfmts_crop @@ -33,8 +33,8 @@ yuv422p16be 167e4338811a7d272925a4c6417d60da yuv422p16le 3359395d5875d581fa1e975013d30114 yuv440p 2472417d980e395ad6843cbb8b633b29 yuv444p 1f151980486848c96bc5585ced99003e -yuv444p16be 5d0c0ea66ab43c0c590d8c2a9256e43f -yuv444p16le 3c0a747c1b64feb0ab8dfba92f92579a +yuv444p16be 1ce8fcd4712d525af983e6179d6a4f9e +yuv444p16le 5f1441e18345aadb3f881dac99c6c08a yuva420p 7536753dfbc7932560fb50c921369a0e yuvj420p 21f891093006d42d7683b0e1d773a657 yuvj422p 9a43d474c407590ad8f213880586b45e diff --git a/tests/ref/lavfi/pixfmts_hflip b/tests/ref/lavfi/pixfmts_hflip index 5eb1b31..3741713 100644 --- a/tests/ref/lavfi/pixfmts_hflip +++ b/tests/ref/lavfi/pixfmts_hflip @@ -29,12 +29,12 @@ yuv420p 2d5c80f9ba2ddd85b2aeda3564cc7d64 yuv420p16be 758b0c1e2113b15e7afde48da4e4d024 yuv420p16le 480ccd951dcb806bc875d307e02e50a0 yuv422p 6e728f4eb9eae287c224f396d84be6ea -yuv422p16be 69cf0605496c321546899a8442ee64fb -yuv422p16le f0b443fea72f4b6f462859a73b159664 +yuv422p16be a05d43cd62b790087bd37083174557de +yuv422p16le 6954abebcbc62d81068d58d0c62bdd5b yuv440p a99e2b57ed601f39852715c9d675d0d3 yuv444p 947e47f7bb5fdccc659d19b7df2b6fc3 -yuv444p16be bc7d53923cff1d7e98d24540845fb64b -yuv444p16le 5df206a93f85ef8b77f5bdc81d9b0a0b +yuv444p16be 58c012e5ab73b066ef3c2b6411a395f1 +yuv444p16le 32c12794e184042a59738ab2de608c8d yuva420p d83ec0c01498189f179ec574918185f1 yuvj420p df3aaaec3bb157c3bde5f0365af30f4f yuvj422p d113871528d510a192797af59df9c05c diff --git a/tests/ref/lavfi/pixfmts_null b/tests/ref/lavfi/pixfmts_null index c4d7890..90dcbf5 100644 --- a/tests/ref/lavfi/pixfmts_null +++ b/tests/ref/lavfi/pixfmts_null @@ -40,16 +40,16 @@ yuv420p9le 16543fda8f87d94a6cf857d2e8d4461a yuv422p c9bba4529821d796a6ab09f6a5fd355a yuv422p10be bdc13b630fd668b34c6fe1aae28dfc71 yuv422p10le d0607c260a45c973e6639f4e449730ad -yuv422p16be 4e9b3b3467aeebb6a528cee5966800ed -yuv422p16le f87c81bf16916b64d201359be0b4b6f4 +yuv422p16be 5499502e1c29534a158a1fe60e889f60 +yuv422p16le e3d61fde6978591596bc36b914386623 yuv422p9be 29b71579946940a8c00fa844c9dff507 yuv422p9le 062b7f9cbb972bf36b5bdb1a7623701a yuv440p 5a064afe2b453bb52cdb3f176b1aa1cf yuv444p 0a98447b78fd476aa39686da6a74fa2e yuv444p10be e65cbae7e4f1892c23defbc8e8052cf6 yuv444p10le 767179dd82846cf00ee4c340c9c1ab74 -yuv444p16be 3ad639fff73e56f3b09dd20c335478d6 -yuv444p16le 8a7e66dc91ab7971fd24a9105ff2699b +yuv444p16be 1c6ea2c2f5e539006112ceec3d4e7d90 +yuv444p16le 20f86bc2f68d2b3f1f2b48b97b2189f4 yuv444p9be 6ab31f4c12b533ce318ecdff83cdd054 yuv444p9le f0606604a5c08becab6ba500124c4b7c yuva420p a29884f3f3dfe1e00b961bc17bef3d47 diff --git a/tests/ref/lavfi/pixfmts_scale b/tests/ref/lavfi/pixfmts_scale index 83b523e..8ca9cbf 100644 --- a/tests/ref/lavfi/pixfmts_scale +++ b/tests/ref/lavfi/pixfmts_scale @@ -33,23 +33,23 @@ yuv411p 1143e7c5cc28fe0922b051b17733bc4c yuv420p fdad2d8df8985e3d17e73c71f713cb14 yuv420p10be c143e77e97d2f7d62c3b518857ba9f9b yuv420p10le 72d90eccf5c34691ff057dafb7447aa2 -yuv420p16be 9688e33e03b8c8275ab2fb1df0f06bee -yuv420p16le cba8b390ad5e7b8678e419b8ce79c008 +yuv420p16be d7270efce54eb59c7b01c14157a1b890 +yuv420p16le e85abf00bad940a922b623c91c9026d7 yuv420p9be bb87fddca65d1742412c8d2b1caf96c6 yuv420p9le 828eec50014a41258a5423c1fe56ac97 yuv422p 918e37701ee7377d16a8a6c119c56a40 yuv422p10be cea7ca6b0e66d6f29539885896c88603 yuv422p10le a10c4a5837547716f13cd61918b145f9 -yuv422p16be 285993ee0c0f4f8e511ee46f93c5f38c -yuv422p16le 61bfcee8e54465f760164f5a75d40b5e +yuv422p16be e7e34fe9264784763ab6cb406524c0f3 +yuv422p16le c435b76b08204dda6908640fb5fd4621 yuv422p9be 82494823944912f73cebc58ad2979bbd yuv422p9le fc69c8a21f473916a4b4225636b97e06 yuv440p 461503fdb9b90451020aa3b25ddf041c yuv444p 81b2eba962d12e8d64f003ac56f6faf2 yuv444p10be e9d3c8e744b8b0d8187ca092fa203fc9 yuv444p10le 02f0a336e9da062a64df1ba487e102c5 -yuv444p16be 2677f3074d255f9dab625e9e2e092ca5 -yuv444p16le 65fa92521ef97088599ea83f9508cd5b +yuv444p16be 0da9bed80f5542682ab286f3261cf24c +yuv444p16le a0c5d3c7bf3f181db503cf8e450d1335 yuv444p9be 9ac2643ce7f7e5c4e17c8c9fd8494d4a yuv444p9le 896a1cc9cccca1ba410dd53942d33cc4 yuva420p 8673a9131fb47de69788863f93a50eb7 diff --git a/tests/ref/lavfi/pixfmts_vflip b/tests/ref/lavfi/pixfmts_vflip index 66d803c..4aaef16 100644 --- a/tests/ref/lavfi/pixfmts_vflip +++ b/tests/ref/lavfi/pixfmts_vflip @@ -40,16 +40,16 @@ yuv420p9le 7e606310d3f5ff12badf911e8f333471 yuv422p d7f5cb44d9b0210d66d6a8762640ab34 yuv422p10be 588fe319b96513c32e21d3e32b45447f yuv422p10le 11b57f2bd9661024153f3973b9090cdb -yuv422p16be c092d083548c2a144c372a98c46875c7 -yuv422p16le c071b9397a416d51cbe339345cbcba84 +yuv422p16be 9bd8f8c961822b586fa4cf992be54acc +yuv422p16le 9c4a1239605c7952b736ac3130163f14 yuv422p9be 7c6f1e140b3999ee7d923854e507752a yuv422p9le 51f10d79c07989060dd06e767e6d7d60 yuv440p 876385e96165acf51271b20e5d85a416 yuv444p 9c3c667d1613b72d15bc6d851c5eb8f7 yuv444p10be 944a4997c4edb3a8dd0f0493cfd5a1fd yuv444p10le 2d0947ae89ecc6a501eee6832cb27e06 -yuv444p16be 6a954614fd2a8ae0df53e4fd76937af8 -yuv444p16le 65613965fb58cc4c3cd480a68b6540ea +yuv444p16be de2dedfc6f12073ffead113f86e07ecf +yuv444p16le 8e83323cf102d6c823a03ae8a7b7e033 yuv444p9be 6ac92b7dc9ab2fc59bee99204886899a yuv444p9le 85aef13a654953d3455d89770b0d74bd yuva420p c705d1cf061d8c6580ac690b55f92276 -- cgit v1.1