| AOMedia AV1 Codec

temporal_filter.h
1/*
2 * Copyright (c) 2016, Alliance for Open Media. All rights reserved.
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12#ifndef AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
13#define AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
14
15#include <stdbool.h>
16
17#include "aom_util/aom_pthread.h"
18
19#ifdef __cplusplus
20 extern "C" {
21#endif
23 struct AV1_COMP;
24 struct AV1EncoderConfig;
25 struct ThreadData;
26// TODO(wtc): These two variables are only used in avx2, sse2, neon
27// implementations, where the block size is still hard coded to TF_BLOCK_SIZE.
28// This should be fixed to align with the c implementation.
29#define BH 64
30#define BW 64
31
32// Block size used in temporal filtering.
33#define TF_BLOCK_SIZE BLOCK_64X64
34
35// Window size for temporal filtering.
36#define TF_WINDOW_LENGTH 5
37
38// Number of 16x16 blocks within one 64x64 TF block.
39#define NUM_16X16 16
40
41// A constant number, sqrt(pi / 2), used for noise estimation.
42 static const double SQRT_PI_BY_2 = 1.25331413732;
43
44// Hyper-parameters used to compute filtering weight. These hyper-parameters can
45// be tuned for a better performance.
46// 0. A scale factor used in temporal filtering to raise the filter weight from
47// `double` with range [0, 1] to `int` with range [0, 1000].
48#define TF_WEIGHT_SCALE 1000
49// 1. Weight factor used to balance the weighted-average between window error
50// and block error. The weight is for window error while the weight for block
51// error is always set as 1.
52#define TF_WINDOW_BLOCK_BALANCE_WEIGHT 5
53// 2. Threshold for using q to adjust the filtering weight. Concretely, when
54// using a small q (high bitrate), we would like to reduce the filtering
55// strength such that more detailed information can be preserved. Hence, when
56// q is smaller than this threshold, we will adjust the filtering weight
57// based on the q-value.
58#define TF_Q_DECAY_THRESHOLD 20
59// 3. Normalization factor used to normalize the motion search error. Since the
60// motion search error can be large and uncontrollable, we will simply
61// normalize it before using it to compute the filtering weight.
62#define TF_SEARCH_ERROR_NORM_WEIGHT 20
63// 4. Threshold for using `arnr_strength` to adjust the filtering strength.
64// Concretely, users can use `arnr_strength` arguments to control the
65// strength of temporal filtering. When `arnr_strength` is small enough (
66// i.e., smaller than this threshold), we will adjust the filtering weight
67// based on the strength value.
68#define TF_STRENGTH_THRESHOLD 4
69// 5. Threshold for using motion search distance to adjust the filtering weight.
70// Concretely, larger motion search vector leads to a higher probability of
71// unreliable search. Hence, we would like to reduce the filtering strength
72// when the distance is large enough. Considering that the distance actually
73// relies on the frame size, this threshold is also a resolution-based
74// threshold. Taking 720p videos as an instance, if this field equals to 0.1,
75// then the actual threshold will be 720 * 0.1 = 72. Similarly, the threshold
76// for 360p videos will be 360 * 0.1 = 36.
77#define TF_SEARCH_DISTANCE_THRESHOLD 0.1
78// 6. Threshold to identify if the q is in a relative high range.
79// Above this cutoff q, a stronger filtering is applied.
80// For a high q, the quantization throws away more information, and thus a
81// stronger filtering is less likely to distort the encoded quality, while a
82// stronger filtering could reduce bit rates.
83// Ror a low q, more details are expected to be retained. Filtering is thus
84// more conservative.
85#define TF_QINDEX_CUTOFF 128
86
87#define NOISE_ESTIMATION_EDGE_THRESHOLD 50
88
89// Sum and SSE source vs filtered frame difference returned by
90// temporal filter.
91 typedef struct {
92 int64_t sum;
93 int64_t sse;
94} FRAME_DIFF;
95
97
101 typedef struct {
105 YV12_BUFFER_CONFIG *frames[MAX_LAG_BUFFERS];
109 int num_frames;
110
114 YV12_BUFFER_CONFIG *output_frame;
115
119 int filter_frame_idx;
123 int compute_frame_diff;
127 struct scale_factors sf;
131 double noise_levels[MAX_MB_PLANE];
135 int num_pels;
139 int mb_rows;
143 int mb_cols;
147 int is_highbitdepth;
151 int q_factor;
152} TemporalFilterCtx;
153
159#define TF_INFO_BUF_COUNT 2
160
164 typedef struct TEMPORAL_FILTER_INFO {
170 int is_temporal_filter_on;
175 YV12_BUFFER_CONFIG tf_buf[TF_INFO_BUF_COUNT];
176
183 YV12_BUFFER_CONFIG tf_buf_second_arf;
187 FRAME_DIFF frame_diff[TF_INFO_BUF_COUNT];
191 int tf_buf_gf_index[TF_INFO_BUF_COUNT];
195 int tf_buf_display_index_offset[TF_INFO_BUF_COUNT];
199 int tf_buf_valid[TF_INFO_BUF_COUNT];
200} TEMPORAL_FILTER_INFO;
201
207 int av1_is_temporal_filter_on(const struct AV1EncoderConfig *oxcf);
208
215 bool av1_tf_info_alloc(TEMPORAL_FILTER_INFO *tf_info,
216 const struct AV1_COMP *cpi);
217
221 void av1_tf_info_free(TEMPORAL_FILTER_INFO *tf_info);
222
226 void av1_tf_info_reset(TEMPORAL_FILTER_INFO *tf_info);
227
233 void av1_tf_info_filtering(TEMPORAL_FILTER_INFO *tf_info, struct AV1_COMP *cpi,
234 const GF_GROUP *gf_group);
235
242YV12_BUFFER_CONFIG *av1_tf_info_get_filtered_buf(TEMPORAL_FILTER_INFO *tf_info,
243 int gf_index,
244 FRAME_DIFF *frame_diff);
245
247
248// Data related to temporal filtering.
249 typedef struct {
250 // Source vs filtered frame error.
251 FRAME_DIFF diff;
252 // Pointer to temporary block info used to store state in temporal filtering
253 // process.
254 MB_MODE_INFO *tmp_mbmi;
255 // Pointer to accumulator buffer used in temporal filtering process.
256 uint32_t *accum;
257 // Pointer to count buffer used in temporal filtering process.
258 uint16_t *count;
259 // Pointer to predictor used in temporal filtering process.
260 uint8_t *pred;
261} TemporalFilterData;
262
263// Data related to temporal filter multi-thread synchronization.
264 typedef struct {
265#if CONFIG_MULTITHREAD
266 // Mutex lock used for dispatching jobs.
267 pthread_mutex_t *mutex_;
268#endif // CONFIG_MULTITHREAD
269 // Next temporal filter block row to be filtered.
270 int next_tf_row;
271 // Initialized to false, set to true by the worker thread that encounters an
272 // error in order to abort the processing of other worker threads.
273 bool tf_mt_exit;
274} AV1TemporalFilterSync;
275
276// Estimates noise level from a given frame using a single plane (Y, U, or V).
277// This is an adaptation of the mehtod in the following paper:
278// Shen-Chuan Tai, Shih-Ming Yang, "A fast method for image noise
279// estimation using Laplacian operator and adaptive edge detection",
280// Proc. 3rd International Symposium on Communications, Control and
281// Signal Processing, 2008, St Julians, Malta.
282// Inputs:
283// frame: Pointer to the frame to estimate noise level from.
284// noise_level: Pointer to store the estimated noise.
285// plane_from: Index of the starting plane used for noise estimation.
286// Commonly, 0 for Y-plane, 1 for U-plane, and 2 for V-plane.
287// plane_to: Index of the end plane used for noise estimation.
288// bit_depth: Actual bit-depth instead of the encoding bit-depth of the frame.
289// edge_thresh: Edge threshold.
290 void av1_estimate_noise_level(const YV12_BUFFER_CONFIG *frame,
291 double *noise_level, int plane_from, int plane_to,
292 int bit_depth, int edge_thresh);
294
306 void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td,
307 int mb_row);
308
333 void av1_temporal_filter(struct AV1_COMP *cpi,
334 const int filter_frame_lookahead_idx,
335 int gf_frame_index, FRAME_DIFF *frame_diff,
336 YV12_BUFFER_CONFIG *output_frame);
337
354 int av1_check_show_filtered_frame(const YV12_BUFFER_CONFIG *frame,
355 const FRAME_DIFF *frame_diff, int q_index,
356 aom_bit_depth_t bit_depth, int enable_overlay,
357 int is_second_arf);
358
360// Allocates memory for members of TemporalFilterData.
361// Inputs:
362// tf_data: Pointer to the structure containing temporal filter related data.
363// num_pels: Number of pixels in the block across all planes.
364// is_high_bitdepth: Whether the frame is high-bitdepth or not.
365// Returns:
366// True if allocation is successful and false otherwise.
367 static inline bool tf_alloc_and_reset_data(TemporalFilterData *tf_data,
368 int num_pels, int is_high_bitdepth) {
369 tf_data->tmp_mbmi = (MB_MODE_INFO *)aom_calloc(1, sizeof(*tf_data->tmp_mbmi));
370 tf_data->accum =
371 (uint32_t *)aom_memalign(16, num_pels * sizeof(*tf_data->accum));
372 tf_data->count =
373 (uint16_t *)aom_memalign(16, num_pels * sizeof(*tf_data->count));
374 if (is_high_bitdepth)
375 tf_data->pred = CONVERT_TO_BYTEPTR(
376 aom_memalign(32, num_pels * 2 * sizeof(*tf_data->pred)));
377 else
378 tf_data->pred =
379 (uint8_t *)aom_memalign(32, num_pels * sizeof(*tf_data->pred));
380 // In case of an allocation failure, other successfully allocated buffers will
381 // be freed by the tf_dealloc_data() call in encoder_destroy().
382 if (!(tf_data->tmp_mbmi && tf_data->accum && tf_data->count && tf_data->pred))
383 return false;
384 memset(&tf_data->diff, 0, sizeof(tf_data->diff));
385 return true;
386}
387
388// Setup macroblockd params for temporal filtering process.
389// Inputs:
390// mbd: Pointer to the block for filtering.
391// tf_data: Pointer to the structure containing temporal filter related data.
392// scale: Scaling factor.
393// Returns:
394// Nothing will be returned. Contents of mbd will be modified.
395 static inline void tf_setup_macroblockd(MACROBLOCKD *mbd,
396 TemporalFilterData *tf_data,
397 const struct scale_factors *scale) {
398 mbd->block_ref_scale_factors[0] = scale;
399 mbd->block_ref_scale_factors[1] = scale;
400 mbd->mi = &tf_data->tmp_mbmi;
401 mbd->mi[0]->motion_mode = SIMPLE_TRANSLATION;
402}
403
404// Deallocates the memory allocated for members of TemporalFilterData.
405// Inputs:
406// tf_data: Pointer to the structure containing temporal filter related data.
407// is_high_bitdepth: Whether the frame is high-bitdepth or not.
408// Returns:
409// Nothing will be returned.
410 static inline void tf_dealloc_data(TemporalFilterData *tf_data,
411 int is_high_bitdepth) {
412 if (is_high_bitdepth)
413 tf_data->pred = (uint8_t *)CONVERT_TO_SHORTPTR(tf_data->pred);
414 aom_free(tf_data->tmp_mbmi);
415 tf_data->tmp_mbmi = NULL;
416 aom_free(tf_data->accum);
417 tf_data->accum = NULL;
418 aom_free(tf_data->count);
419 tf_data->count = NULL;
420 aom_free(tf_data->pred);
421 tf_data->pred = NULL;
422}
423
424// Saves the state prior to temporal filter process.
425// Inputs:
426// mbd: Pointer to the block for filtering.
427// input_mbmi: Backup block info to save input state.
428// input_buffer: Backup buffer pointer to save input state.
429// num_planes: Number of planes.
430// Returns:
431// Nothing will be returned. Contents of input_mbmi and input_buffer will be
432// modified.
433 static inline void tf_save_state(MACROBLOCKD *mbd, MB_MODE_INFO ***input_mbmi,
434 uint8_t **input_buffer, int num_planes) {
435 for (int i = 0; i < num_planes; i++) {
436 input_buffer[i] = mbd->plane[i].pre[0].buf;
437 }
438 *input_mbmi = mbd->mi;
439}
440
441// Restores the initial state after temporal filter process.
442// Inputs:
443// mbd: Pointer to the block for filtering.
444// input_mbmi: Backup block info from where input state is restored.
445// input_buffer: Backup buffer pointer from where input state is restored.
446// num_planes: Number of planes.
447// Returns:
448// Nothing will be returned. Contents of mbd will be modified.
449 static inline void tf_restore_state(MACROBLOCKD *mbd, MB_MODE_INFO **input_mbmi,
450 uint8_t **input_buffer, int num_planes) {
451 for (int i = 0; i < num_planes; i++) {
452 mbd->plane[i].pre[0].buf = input_buffer[i];
453 }
454 mbd->mi = input_mbmi;
455}
456
458#ifdef __cplusplus
459} // extern "C"
460#endif
461
462#endif // AOM_AV1_ENCODER_TEMPORAL_FILTER_H_
aom_bit_depth_t
enum aom_bit_depth aom_bit_depth_t
Bit depth for codecThis enumeration determines the bit depth of the codec.
av1_temporal_filter
void av1_temporal_filter(struct AV1_COMP *cpi, const int filter_frame_lookahead_idx, int gf_frame_index, FRAME_DIFF *frame_diff, YV12_BUFFER_CONFIG *output_frame)
Performs temporal filtering if needed on a source frame. For example to create a filtered alternate r...
av1_check_show_filtered_frame
int av1_check_show_filtered_frame(const YV12_BUFFER_CONFIG *frame, const FRAME_DIFF *frame_diff, int q_index, aom_bit_depth_t bit_depth, int enable_overlay, int is_second_arf)
Check whether a filtered frame can be show directly.
av1_tf_do_filtering_row
void av1_tf_do_filtering_row(struct AV1_COMP *cpi, struct ThreadData *td, int mb_row)
Does temporal filter for a given macroblock row.
Definition temporal_filter.c:1034
AV1EncoderConfig
Main encoder configuration data structure.
Definition encoder.h:934
AV1_COMP
Top level encoder structure.
Definition encoder.h:2897
GF_GROUP
Data related to the current GF/ARF group and the individual frames within the group.
Definition firstpass.h:343
MB_MODE_INFO
Stores the prediction/txfm mode of the current coding block.
Definition blockd.h:222
MB_MODE_INFO::motion_mode
MOTION_MODE motion_mode
The motion mode used by the inter prediction.
Definition blockd.h:250
TEMPORAL_FILTER_INFO
Temporal filter info for a gop.
Definition temporal_filter.h:164
TEMPORAL_FILTER_INFO::tf_buf_display_index_offset
int tf_buf_display_index_offset[2]
Definition temporal_filter.h:195
TEMPORAL_FILTER_INFO::tf_buf
YV12_BUFFER_CONFIG tf_buf[2]
Definition temporal_filter.h:175
TEMPORAL_FILTER_INFO::tf_buf_gf_index
int tf_buf_gf_index[2]
Definition temporal_filter.h:191
TEMPORAL_FILTER_INFO::frame_diff
FRAME_DIFF frame_diff[2]
Definition temporal_filter.h:187
TEMPORAL_FILTER_INFO::is_temporal_filter_on
int is_temporal_filter_on
Definition temporal_filter.h:170
TEMPORAL_FILTER_INFO::tf_buf_valid
int tf_buf_valid[2]
Definition temporal_filter.h:199
TEMPORAL_FILTER_INFO::tf_buf_second_arf
YV12_BUFFER_CONFIG tf_buf_second_arf
Definition temporal_filter.h:183
TemporalFilterCtx
Parameters related to temporal filtering.
Definition temporal_filter.h:101
TemporalFilterCtx::sf
struct scale_factors sf
Definition temporal_filter.h:127
TemporalFilterCtx::output_frame
YV12_BUFFER_CONFIG * output_frame
Definition temporal_filter.h:114
TemporalFilterCtx::q_factor
int q_factor
Definition temporal_filter.h:151
TemporalFilterCtx::num_pels
int num_pels
Definition temporal_filter.h:135
TemporalFilterCtx::num_frames
int num_frames
Definition temporal_filter.h:109
TemporalFilterCtx::compute_frame_diff
int compute_frame_diff
Definition temporal_filter.h:123
TemporalFilterCtx::mb_rows
int mb_rows
Definition temporal_filter.h:139
TemporalFilterCtx::mb_cols
int mb_cols
Definition temporal_filter.h:143
TemporalFilterCtx::noise_levels
double noise_levels[MAX_MB_PLANE]
Definition temporal_filter.h:131
TemporalFilterCtx::is_highbitdepth
int is_highbitdepth
Definition temporal_filter.h:147
TemporalFilterCtx::filter_frame_idx
int filter_frame_idx
Definition temporal_filter.h:119
TemporalFilterCtx::frames
YV12_BUFFER_CONFIG * frames[MAX_LAG_BUFFERS]
Definition temporal_filter.h:105
macroblockd::plane
struct macroblockd_plane plane[3]
Definition blockd.h:606
macroblockd::block_ref_scale_factors
const struct scale_factors * block_ref_scale_factors[2]
Definition blockd.h:687
macroblockd::mi
MB_MODE_INFO ** mi
Definition blockd.h:617