/* * Copyright (c) 2017 Richard Ling * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ /* * Normalize RGB video (aka histogram stretching, contrast stretching). * See: https://en.wikipedia.org/wiki/Normalization_(image_processing) * * For each channel of each frame, the filter computes the input range and maps * it linearly to the user-specified output range. The output range defaults * to the full dynamic range from pure black to pure white. * * Naively maximising the dynamic range of each frame of video in isolation * may cause flickering (rapid changes in brightness of static objects in the * scene) when small dark or bright objects enter or leave the scene. This * filter can apply temporal smoothing to the input range to reduce flickering. * Temporal smoothing is similar to the auto-exposure (automatic gain control) * on a video camera, which performs the same function; and, like a video * camera, it may cause a period of over- or under-exposure of the video. * * The filter can normalize the R,G,B channels independently, which may cause * color shifting, or link them together as a single channel, which prevents * color shifting. More precisely, linked normalization preserves hue (as it's * defined in HSV/HSL color spaces) while independent normalization does not. * Independent normalization can be used to remove color casts, such as the * blue cast from underwater video, restoring more natural colors. The filter * can also combine independent and linked normalization in any ratio. * * Finally the overall strength of the filter can be adjusted, from no effect * to full normalization. * * The 5 AVOptions are: * blackpt, Colors which define the output range. The minimum input value * whitept is mapped to the blackpt. The maximum input value is mapped to * the whitept. The defaults are black and white respectively. * Specifying white for blackpt and black for whitept will give * color-inverted, normalized video. Shades of grey can be used * to reduce the dynamic range (contrast). Specifying saturated * colors here can create some interesting effects. * * smoothing The amount of temporal smoothing, expressed in frames (>=0). * the minimum and maximum input values of each channel are * smoothed using a rolling average over the current frame and * that many previous frames of video. Defaults to 0 (no temporal * smoothing). * * independence * Controls the ratio of independent (color shifting) channel * normalization to linked (color preserving) normalization. 0.0 * is fully linked, 1.0 is fully independent. Defaults to fully * independent. * * strength Overall strength of the filter. 1.0 is full strength. 0.0 is * a rather expensive no-op. Values in between can give a gentle * boost to low-contrast video without creating an artificial * over-processed look. The default is full strength. */ #include "libavutil/imgutils.h" #include "libavutil/opt.h" #include "libavutil/pixdesc.h" #include "avfilter.h" #include "drawutils.h" #include "formats.h" #include "internal.h" #include "video.h" typedef struct NormalizeHistory { uint8_t *history; // History entries. uint32_t history_sum; // Sum of history entries. } NormalizeHistory; typedef struct NormalizeLocal { uint8_t in; // Original input byte value for this frame. float smoothed; // Smoothed input value [0,255]. float out; // Output value [0,255] } NormalizeLocal; typedef struct NormalizeContext { const AVClass *class; // Storage for the corresponding AVOptions uint8_t blackpt[4]; uint8_t whitept[4]; int smoothing; float independence; float strength; uint8_t co[4]; // Offsets to R,G,B,A bytes respectively in each pixel int num_components; // Number of components in the pixel format int step; int history_len; // Number of frames to average; based on smoothing factor int frame_num; // Increments on each frame, starting from 0. // Per-extremum, per-channel history, for temporal smoothing. NormalizeHistory min[3], max[3]; // Min and max for each channel in {R,G,B}. uint8_t *history_mem; // Single allocation for above history entries } NormalizeContext; #define OFFSET(x) offsetof(NormalizeContext, x) #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM static const AVOption normalize_options[] = { { "blackpt", "output color to which darkest input color is mapped", OFFSET(blackpt), AV_OPT_TYPE_COLOR, { .str = "black" }, CHAR_MIN, CHAR_MAX, FLAGS }, { "whitept", "output color to which brightest input color is mapped", OFFSET(whitept), AV_OPT_TYPE_COLOR, { .str = "white" }, CHAR_MIN, CHAR_MAX, FLAGS }, { "smoothing", "amount of temporal smoothing of the input range, to reduce flicker", OFFSET(smoothing), AV_OPT_TYPE_INT, {.i64=0}, 0, INT_MAX/8, FLAGS }, { "independence", "proportion of independent to linked channel normalization", OFFSET(independence), AV_OPT_TYPE_FLOAT, {.dbl=1.0}, 0.0, 1.0, FLAGS }, { "strength", "strength of filter, from no effect to full normalization", OFFSET(strength), AV_OPT_TYPE_FLOAT, {.dbl=1.0}, 0.0, 1.0, FLAGS }, { NULL } }; AVFILTER_DEFINE_CLASS(normalize); // This function is the main guts of the filter. Normalizes the input frame // into the output frame. The frames are known to have the same dimensions // and pixel format. static void normalize(NormalizeContext *s, AVFrame *in, AVFrame *out) { // Per-extremum, per-channel local variables. NormalizeLocal min[3], max[3]; // Min and max for each channel in {R,G,B}. float rgb_min_smoothed; // Min input range for linked normalization float rgb_max_smoothed; // Max input range for linked normalization uint8_t lut[3][256]; // Lookup table int x, y, c; // First, scan the input frame to find, for each channel, the minimum // (min.in) and maximum (max.in) values present in the channel. for (c = 0; c < 3; c++) min[c].in = max[c].in = in->data[0][s->co[c]]; for (y = 0; y < in->height; y++) { uint8_t *inp = in->data[0] + y * in->linesize[0]; for (x = 0; x < in->width; x++) { for (c = 0; c < 3; c++) { min[c].in = FFMIN(min[c].in, inp[s->co[c]]); max[c].in = FFMAX(max[c].in, inp[s->co[c]]); } inp += s->step; } } // Next, for each channel, push min.in and max.in into their respective // histories, to determine the min.smoothed and max.smoothed for this frame. { int history_idx = s->frame_num % s->history_len; // Assume the history is not yet full; num_history_vals is the number // of frames received so far including the current frame. int num_history_vals = s->frame_num + 1; if (s->frame_num >= s->history_len) { //The history is full; drop oldest value and cap num_history_vals. for (c = 0; c < 3; c++) { s->min[c].history_sum -= s->min[c].history[history_idx]; s->max[c].history_sum -= s->max[c].history[history_idx]; } num_history_vals = s->history_len; } // For each extremum, update history_sum and calculate smoothed value // as the rolling average of the history entries. for (c = 0; c < 3; c++) { s->min[c].history_sum += (s->min[c].history[history_idx] = min[c].in); min[c].smoothed = s->min[c].history_sum / (float)num_history_vals; s->max[c].history_sum += (s->max[c].history[history_idx] = max[c].in); max[c].smoothed = s->max[c].history_sum / (float)num_history_vals; } } // Determine the input range for linked normalization. This is simply the // minimum of the per-channel minimums, and the maximum of the per-channel // maximums. rgb_min_smoothed = FFMIN3(min[0].smoothed, min[1].smoothed, min[2].smoothed); rgb_max_smoothed = FFMAX3(max[0].smoothed, max[1].smoothed, max[2].smoothed); // Now, process each channel to determine the input and output range and // build the lookup tables. for (c = 0; c < 3; c++) { int in_val; // Adjust the input range for this channel [min.smoothed,max.smoothed] // by mixing in the correct proportion of the linked normalization // input range [rgb_min_smoothed,rgb_max_smoothed]. min[c].smoothed = (min[c].smoothed * s->independence) + (rgb_min_smoothed * (1.0f - s->independence)); max[c].smoothed = (max[c].smoothed * s->independence) + (rgb_max_smoothed * (1.0f - s->independence)); // Calculate the output range [min.out,max.out] as a ratio of the full- // strength output range [blackpt,whitept] and the original input range // [min.in,max.in], based on the user-specified filter strength. min[c].out = (s->blackpt[c] * s->strength) + (min[c].in * (1.0f - s->strength)); max[c].out = (s->whitept[c] * s->strength) + (max[c].in * (1.0f - s->strength)); // Now, build a lookup table which linearly maps the adjusted input range // [min.smoothed,max.smoothed] to the output range [min.out,max.out]. // Perform the linear interpolation for each x: // lut[x] = (int)(float(x - min.smoothed) * scale + max.out + 0.5) // where scale = (max.out - min.out) / (max.smoothed - min.smoothed) if (min[c].smoothed == max[c].smoothed) { // There is no dynamic range to expand. No mapping for this channel. for (in_val = min[c].in; in_val <= max[c].in; in_val++) lut[c][in_val] = min[c].out; } else { // We must set lookup values for all values in the original input // range [min.in,max.in]. Since the original input range may be // larger than [min.smoothed,max.smoothed], some output values may // fall outside the [0,255] dynamic range. We need to clamp them. float scale = (max[c].out - min[c].out) / (max[c].smoothed - min[c].smoothed); for (in_val = min[c].in; in_val <= max[c].in; in_val++) { int out_val = (in_val - min[c].smoothed) * scale + min[c].out + 0.5f; out_val = FFMAX(out_val, 0); out_val = FFMIN(out_val, 255); lut[c][in_val] = out_val; } } } // Finally, process the pixels of the input frame using the lookup tables. for (y = 0; y < in->height; y++) { uint8_t *inp = in->data[0] + y * in->linesize[0]; uint8_t *outp = out->data[0] + y * out->linesize[0]; for (x = 0; x < in->width; x++) { for (c = 0; c < 3; c++) outp[s->co[c]] = lut[c][inp[s->co[c]]]; if (s->num_components == 4) // Copy alpha as-is. outp[s->co[3]] = inp[s->co[3]]; inp += s->step; outp += s->step; } } s->frame_num++; } // Now we define all the functions accessible from the ff_vf_normalize class, // which is ffmpeg's interface to our filter. See doc/filter_design.txt and // doc/writing_filters.txt for descriptions of what these interface functions // are expected to do. // Set the pixel formats that our filter supports. We should be able to process // any 8-bit RGB formats. 16-bit support might be useful one day. static int query_formats(AVFilterContext *ctx) { static const enum AVPixelFormat pixel_fmts[] = { AV_PIX_FMT_RGB24, AV_PIX_FMT_BGR24, AV_PIX_FMT_ARGB, AV_PIX_FMT_RGBA, AV_PIX_FMT_ABGR, AV_PIX_FMT_BGRA, AV_PIX_FMT_0RGB, AV_PIX_FMT_RGB0, AV_PIX_FMT_0BGR, AV_PIX_FMT_BGR0, AV_PIX_FMT_NONE }; // According to filter_design.txt, using ff_set_common_formats() this way // ensures the pixel formats of the input and output will be the same. That // saves a bit of effort possibly needing to handle format conversions. AVFilterFormats *formats = ff_make_format_list(pixel_fmts); if (!formats) return AVERROR(ENOMEM); return ff_set_common_formats(ctx, formats); } // At this point we know the pixel format used for both input and output. We // can also access the frame rate of the input video and allocate some memory // appropriately static int config_input(AVFilterLink *inlink) { NormalizeContext *s = inlink->dst->priv; // Store offsets to R,G,B,A bytes respectively in each pixel const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); int c; ff_fill_rgba_map(s->co, inlink->format); s->num_components = desc->nb_components; s->step = av_get_padded_bits_per_pixel(desc) >> 3; // Convert smoothing value to history_len (a count of frames to average, // must be at least 1). Currently this is a direct assignment, but the // smoothing value was originally envisaged as a number of seconds. In // future it would be nice to set history_len using a number of seconds, // but VFR video is currently an obstacle to doing so. s->history_len = s->smoothing + 1; // Allocate the history buffers -- there are 6 -- one for each extrema. // s->smoothing is limited to INT_MAX/8, so that (s->history_len * 6) // can't overflow on 32bit causing a too-small allocation. s->history_mem = av_malloc(s->history_len * 6); if (s->history_mem == NULL) return AVERROR(ENOMEM); for (c = 0; c < 3; c++) { s->min[c].history = s->history_mem + (c*2) * s->history_len; s->max[c].history = s->history_mem + (c*2+1) * s->history_len; } return 0; } // Free any memory allocations here static av_cold void uninit(AVFilterContext *ctx) { NormalizeContext *s = ctx->priv; av_freep(&s->history_mem); } // This function is pretty much standard from doc/writing_filters.txt. It // tries to do in-place filtering where possible, only allocating a new output // frame when absolutely necessary. static int filter_frame(AVFilterLink *inlink, AVFrame *in) { AVFilterContext *ctx = inlink->dst; AVFilterLink *outlink = ctx->outputs[0]; NormalizeContext *s = ctx->priv; AVFrame *out; // Set 'direct' if we can modify the input frame in-place. Otherwise we // need to retrieve a new frame from the output link. int direct = av_frame_is_writable(in) && !ctx->is_disabled; if (direct) { out = in; } else { out = ff_get_video_buffer(outlink, outlink->w, outlink->h); if (!out) { av_frame_free(&in); return AVERROR(ENOMEM); } av_frame_copy_props(out, in); } // Now we've got the input and output frames (which may be the same frame) // perform the filtering with our custom function. normalize(s, in, out); if (ctx->is_disabled) { av_frame_free(&out); return ff_filter_frame(outlink, in); } if (!direct) av_frame_free(&in); return ff_filter_frame(outlink, out); } static const AVFilterPad inputs[] = { { .name = "default", .type = AVMEDIA_TYPE_VIDEO, .filter_frame = filter_frame, .config_props = config_input, }, { NULL } }; static const AVFilterPad outputs[] = { { .name = "default", .type = AVMEDIA_TYPE_VIDEO, }, { NULL } }; AVFilter ff_vf_normalize = { .name = "normalize", .description = NULL_IF_CONFIG_SMALL("Normalize RGB video."), .priv_size = sizeof(NormalizeContext), .priv_class = &normalize_class, .uninit = uninit, .query_formats = query_formats, .inputs = inputs, .outputs = outputs, .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL, };