diff --git a/doc/reST/cli.rst b/doc/reST/cli.rst index 31304fe90..4494099c5 100755 --- a/doc/reST/cli.rst +++ b/doc/reST/cli.rst @@ -1358,15 +1358,33 @@ Temporal / motion search options Motion-compensated spatio-temporal filtering (MCSTF) improves the compression efficiency of videos that contain a high level of noise. It introduces a - temporal filter before encoding and this filter is applied only to the I- and P-frames. + temporal filter before encoding, and this filter is applied unconditionally + to every I- and P-frame regardless of how noisy the source content actually is. It utilizes previously generated motion vectors across different video content resolutions to find the best temporal correspondence for low-pass filtering. Here, motion estimation is applied between the central picture and each future or past picture, thereby generating multiple motion-compensated predictions, which are then combined by using adaptive filtering to produce a final noise-reduced picture. + To filter frames selectively based on their estimated noise level instead, use + :option:`--selective-mcstf`. Default: disabled - Note : MCSTF should be enabled only with frame threads 1 +.. option:: --mcstf-ref-range + + Number of reference frames to consider on each side (past and future) of the + current frame during Motion-compensated spatio-temporal filtering. A higher value + includes more temporal neighbors for filtering, which can improve noise reduction + at the cost of increased memory usage and processing time. The valid range is 1 to 4. + Default: 2 + +.. option:: --selective-mcstf, --no-selective-mcstf + + Gate MCSTF with a per-GOP noise estimate: skip the filter for GOPs whose + starting I/IDR frame (or scenecut) is estimated as clean (low noise), and + apply it only where it helps. Implicitly enables :option:`--mcstf` if it + was not already turned on. When :option:`--csv-log-level` is 2 or higher, + adds a Frame Noise and IsMCSTFEnabled column to the per-frame CSV log. + Default: disabled Spatial/intra options ===================== diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index c9380d301..3177af99a 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -32,7 +32,7 @@ option(NATIVE_BUILD "Target the build CPU" OFF) option(STATIC_LINK_CRT "Statically link C and C++ runtimes for release builds" OFF) mark_as_advanced(FPROFILE_USE FPROFILE_GENERATE NATIVE_BUILD) # X265_BUILD must be incremented each time the public API is changed -set(X265_BUILD 216) +set(X265_BUILD 217) configure_file("${PROJECT_SOURCE_DIR}/x265.def.in" "${PROJECT_BINARY_DIR}/x265.def") configure_file("${PROJECT_SOURCE_DIR}/x265_config.h.in" diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index b17c0cd87..c920c4c7a 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -22,9 +22,10 @@ if(ENABLE_ASSEMBLY AND X86) set(SSE3 vec/dct-sse3.cpp) set(SSSE3 vec/dct-ssse3.cpp) set(SSE41 vec/dct-sse41.cpp) + set(AVX2 vec/temporalfilter-avx2.cpp) if(MSVC) - set(PRIMITIVES ${SSE3} ${SSSE3} ${SSE41}) + set(PRIMITIVES ${SSE3} ${SSSE3} ${SSE41} ${AVX2}) set(WARNDISABLE "/wd4100") # unreferenced formal parameter if(INTEL_CXX) add_definitions(/Qwd111) # statement is unreachable @@ -39,6 +40,7 @@ if(ENABLE_ASSEMBLY AND X86) # x64 implies SSE4, so only add /arch:SSE2 if building for Win32 set_source_files_properties(${SSE3} ${SSSE3} ${SSE41} PROPERTIES COMPILE_FLAGS "${WARNDISABLE} /arch:SSE2") endif() + set_source_files_properties(${AVX2} PROPERTIES COMPILE_FLAGS "${WARNDISABLE} /arch:AVX2") endif() if(GCC) if(CLANG) @@ -48,10 +50,11 @@ if(ENABLE_ASSEMBLY AND X86) set(WARNDISABLE "-Wno-unused-parameter") endif() if(INTEL_CXX OR CLANG OR (NOT CC_VERSION VERSION_LESS 4.3)) - set(PRIMITIVES ${SSE3} ${SSSE3} ${SSE41}) + set(PRIMITIVES ${SSE3} ${SSSE3} ${SSE41} ${AVX2}) set_source_files_properties(${SSE3} PROPERTIES COMPILE_FLAGS "${WARNDISABLE} -msse3") set_source_files_properties(${SSSE3} PROPERTIES COMPILE_FLAGS "${WARNDISABLE} -mssse3") set_source_files_properties(${SSE41} PROPERTIES COMPILE_FLAGS "${WARNDISABLE} -msse4.1") + set_source_files_properties(${AVX2} PROPERTIES COMPILE_FLAGS "${WARNDISABLE} -mavx2") endif() endif() set(VEC_PRIMITIVES vec/vec-primitives.cpp ${PRIMITIVES}) diff --git a/source/common/loongarch64/asm-primitives.cpp b/source/common/loongarch64/asm-primitives.cpp index 6f88d2bd5..94e6ff4ec 100644 --- a/source/common/loongarch64/asm-primitives.cpp +++ b/source/common/loongarch64/asm-primitives.cpp @@ -936,7 +936,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // mc p.frameInitLowres = PFX(frame_init_lowres_core_lsx); p.frameInitLowerRes = PFX(frame_init_lowres_core_lsx); - p.frameSubSampleLuma = PFX(frame_subsample_luma_lsx); p.fix8Unpack = PFX(cutree_fix8_unpack_lsx); p.fix8Pack = PFX(cutree_fix8_pack_lsx); @@ -1333,7 +1332,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // mc p.frameInitLowres = PFX(frame_init_lowres_core_lasx); p.frameInitLowerRes = PFX(frame_init_lowres_core_lasx); - p.frameSubSampleLuma = PFX(frame_subsample_luma_lasx); p.fix8Unpack = PFX(cutree_fix8_unpack_lasx); p.fix8Pack = PFX(cutree_fix8_pack_lasx); diff --git a/source/common/loongarch64/mc-a2.S b/source/common/loongarch64/mc-a2.S index 8b3b37c9e..78fb6d8c5 100644 --- a/source/common/loongarch64/mc-a2.S +++ b/source/common/loongarch64/mc-a2.S @@ -278,123 +278,6 @@ function x265_frame_init_lowres_core_lasx blt zero, t0, .LOOP_HEIGHT_FRAME_INIT_LOWRES_CORE_LASX endfunc -.macro CALCULATE_FRAMESUBSAMPLELUMA_LSX_8 offset0 - // load inRow[offset0 ~ offset0+15], inRowBelow[offset0 ~ offset0+15] - vldx vr1, a0, \offset0 - vldx vr3, t0, \offset0 - vaddwev.h.bu vr4, vr1, vr3 - vaddwod.h.bu vr5, vr1, vr3 - vsrari.h vr4, vr4, 1 - vsrari.h vr5, vr5, 1 - vadd.h vr6, vr4, vr5 - vsrarni.b.h vr6, vr6, 1 -.endm - -/*----------------------------------------------------------------------------- -void x265_frame_subsample_luma_lsx(const pixel* src0, pixel* dst0, - intptr_t src_stride, intptr_t dst_stride, int width, int height) ------------------------------------------------------------------------------*/ -function x265_frame_subsample_luma_lsx - andi t4, a4, 7 // width % 8 - srli.d a4, a4, 3 // width / 8 - -.LOOP_HEIGHT_FRAME_SUBSAMPLE_LUMA_LSX: - add.d t0, a0, a2 // *inRowBelow - addi.d t1, zero, 0 - addi.d t2, zero, 0 - addi.d t8, zero, 0 // index - -.LOOP_WIDTH_FRAME_SUBSAMPLE_LUMA_LSX: - bge t8, a4, .HANDLING_MANTISSA_FRAMESUBSAMPLELUMA_LSX_7 - CALCULATE_FRAMESUBSAMPLELUMA_LSX_8 t1 - add.d t3, a1, t2 - vstelm.d vr6, t3, 0, 0 - - // prepare for next inner-loop - addi.d t1, t1, 16 - addi.d t2, t2, 8 - addi.d t8, t8, 1 // index++ - b .LOOP_WIDTH_FRAME_SUBSAMPLE_LUMA_LSX - -.HANDLING_MANTISSA_FRAMESUBSAMPLELUMA_LSX_7: - bge zero, t4, .PREPARE_FOR_NEXT_OUTERLOOP_FRAMESUBSAMPLELUMA_LSX - CALCULATE_FRAMESUBSAMPLELUMA_LSX_8 t1 // calculate mantissa - addi.d t8, zero, 0 - add.d t3, a1, t2 -.irp offset0, 0, 1, 2, 3, 4, 5, 6 - vstelm.b vr6, t3, 0, \offset0 - addi.d t8, t8, 1 - addi.d t3, t3, 1 - bge t8, t4, .PREPARE_FOR_NEXT_OUTERLOOP_FRAMESUBSAMPLELUMA_LSX -.endr - -.PREPARE_FOR_NEXT_OUTERLOOP_FRAMESUBSAMPLELUMA_LSX: - // prepare for next outer-loop - add.d a0, a0, a2 - add.d a0, a0, a2 - add.d a1, a1, a3 - addi.d a5, a5, -1 - blt zero, a5, .LOOP_HEIGHT_FRAME_SUBSAMPLE_LUMA_LSX -endfunc - -.macro CALCULATE_FRAMESUBSAMPLELUMA_LASX_16 offset0 - // load inRow[offset0 ~ offset0+31], inRowBelow[offset0 ~ offset0+31] - xvldx xr1, a0, \offset0 - xvldx xr3, t0, \offset0 - xvaddwev.h.bu xr4, xr1, xr3 - xvaddwod.h.bu xr5, xr1, xr3 - xvsrari.h xr4, xr4, 1 - xvsrari.h xr5, xr5, 1 - xvadd.h xr6, xr4, xr5 - xvsrarni.b.h xr6, xr6, 1 - xvpermi.d xr6, xr6, 0b00001000 -.endm - -/*----------------------------------------------------------------------------- -void x265_frame_subsample_luma_lasx(const pixel* src0, pixel* dst0, - intptr_t src_stride, intptr_t dst_stride, int width, int height) ------------------------------------------------------------------------------*/ -function x265_frame_subsample_luma_lasx - andi t4, a4, 15 // width % 16 - srli.d a4, a4, 4 // width / 16 - -.LOOP_HEIGHT_FRAME_SUBSAMPLE_LUMA_LASX: - add.d t0, a0, a2 // *inRowBelow - addi.d t1, zero, 0 - addi.d t2, zero, 0 - addi.d t8, zero, 0 // index - -.LOOP_WIDTH_FRAME_SUBSAMPLE_LUMA_LASX: - bge t8, a4, .HANDLING_MANTISSA_FRAMESUBSAMPLELUMA_LASX_15 - CALCULATE_FRAMESUBSAMPLELUMA_LASX_16 t1 - vstx vr6, a1, t2 - addi.d t1, t1, 32 - addi.d t2, t2, 16 - addi.d t8, t8, 1 // index++ - b .LOOP_WIDTH_FRAME_SUBSAMPLE_LUMA_LASX - -.HANDLING_MANTISSA_FRAMESUBSAMPLELUMA_LASX_15: - bge zero, t4, .PREPARE_FOR_NEXT_OUTERLOOP_FRAMESUBSAMPLELUMA_LASX - CALCULATE_FRAMESUBSAMPLELUMA_LASX_16 t1 // calculate mantissa - addi.d t8, zero, 0 - add.d t3, a1, t2 -.irp offset0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 - vstelm.b vr6, t3, 0, \offset0 - addi.d t8, t8, 1 - addi.d t3, t3, 1 - bge t8, t4, .PREPARE_FOR_NEXT_OUTERLOOP_FRAMESUBSAMPLELUMA_LASX -.endr - -.PREPARE_FOR_NEXT_OUTERLOOP_FRAMESUBSAMPLELUMA_LASX: - // prepare for next loop - add.d a0, a0, a2 - add.d a0, a0, a2 - add.d a1, a1, a3 - addi.d a5, a5, -1 - blt zero, a5, .LOOP_HEIGHT_FRAME_SUBSAMPLE_LUMA_LASX -endfunc - - .macro HANDLING_MANTISSA_MC_A2_3 size, dst0, dst1, dst2, count, dst_adress, \ dst_stride, index0, index1, index2, terminate_lable addi.d \count, \count, -1 diff --git a/source/common/loongarch64/pixel.h b/source/common/loongarch64/pixel.h index 731664e70..9da01c6d5 100644 --- a/source/common/loongarch64/pixel.h +++ b/source/common/loongarch64/pixel.h @@ -372,8 +372,6 @@ void x265_addAvg_64x64_lasx(const int16_t* src0, const int16_t* src1, pixel* dst void x265_frame_init_lowres_core_lsx(const pixel *src0, pixel *dst0, pixel *dsth, pixel *dstv, pixel *dstc, intptr_t src_stride, intptr_t dst_stride, int width, int height); void x265_frame_init_lowres_core_lasx(const pixel *src0, pixel *dst0, pixel *dsth, pixel *dstv, pixel *dstc, intptr_t src_stride, intptr_t dst_stride, int width, int height); -void x265_frame_subsample_luma_lsx(const pixel *src0, pixel *dst0, intptr_t src_stride, intptr_t dst_stride, int width, int height); -void x265_frame_subsample_luma_lasx(const pixel *src0, pixel *dst0, intptr_t src_stride, intptr_t dst_stride, int width, int height); void x265_cutree_fix8_unpack_lsx(double *dst, uint16_t *src, int count); void x265_cutree_fix8_unpack_lasx(double *dst, uint16_t *src, int count); void x265_cutree_fix8_pack_lsx(uint16_t *dst, double *src, int count); diff --git a/source/common/lowres.cpp b/source/common/lowres.cpp index 52953a812..b15dc194d 100644 --- a/source/common/lowres.cpp +++ b/source/common/lowres.cpp @@ -26,6 +26,12 @@ #include "lowres.h" #include "mv.h" +namespace X265_NS { + // forward declaration - defined in pixel.cpp, no SIMD override for MCSTF path + void frameInitLowresCoreMCSTF(const pixel* src0, pixel* dst0, pixel* dsth, pixel* dstv, pixel* dstc, + intptr_t src_stride, intptr_t dst_stride, int width, int height); +} + using namespace X265_NS; /* @@ -194,11 +200,6 @@ bool Lowres::create(x265_param* param, PicYuv *origPic, uint32_t qgSize) } } - for (int i = 0; i < 4; i++) - { - CHECKED_MALLOC(lowresMcstfMvs[0][i], MV, cuCount); - } - for (int i = 0; i < bframes + 2; i++) { CHECKED_MALLOC(lowresMvs[0][i], MV, cuCount); @@ -287,10 +288,6 @@ void Lowres::destroy(x265_param* param) } } - for (int i = 0; i < 4; i++) - { - X265_FREE(lowresMcstfMvs[0][i]); - } X265_FREE(qpAqOffset); X265_FREE(invQscaleFactor); X265_FREE(qpCuTreeOffset); @@ -344,11 +341,13 @@ void Lowres::destroy(x265_param* param) } } // (re) initialize lowres state -void Lowres::init(PicYuv *origPic, int poc) +void Lowres::init(PicYuv* origPic, int poc, bool bEnableTemporalFilter) { bLastMiniGopBFrame = false; bKeyframe = false; // Not a keyframe unless identified by lookahead bIsFadeEnd = false; + filterThisGOP = false; + noiseScore = -1; frameNum = poc; leadingBframes = 0; indB = 0; @@ -368,11 +367,6 @@ void Lowres::init(PicYuv *origPic, int poc) lowresMvs[1][i][0].x = 0x7FFF; } - for (int i = 0; i < 4; i++) - { - lowresMcstfMvs[0][i][0].x = 0x7FFF; - } - for (int i = 0; i < bframes + 2; i++) intraMbs[i] = 0; if (origPic->m_param->rc.vbvBufferSize) @@ -380,9 +374,10 @@ void Lowres::init(PicYuv *origPic, int poc) plannedType[i] = X265_TYPE_AUTO; /* downscale and generate 4 hpel planes for lookahead */ - primitives.frameInitLowres(origPic->m_picOrg[0], - lowresPlane[0], lowresPlane[1], lowresPlane[2], lowresPlane[3], - origPic->m_stride, lumaStride, width, lines); + if (bEnableTemporalFilter) + frameInitLowresCoreMCSTF(origPic->m_picOrg[0], lowresPlane[0], lowresPlane[1], lowresPlane[2], lowresPlane[3], origPic->m_stride, lumaStride, width, lines); + else + primitives.frameInitLowres(origPic->m_picOrg[0], lowresPlane[0], lowresPlane[1], lowresPlane[2], lowresPlane[3], origPic->m_stride, lumaStride, width, lines); /* extend hpel planes for motion search */ extendPicBorder(lowresPlane[0], lumaStride, width, lines, origPic->m_lumaMarginX, origPic->m_lumaMarginY); @@ -392,9 +387,11 @@ void Lowres::init(PicYuv *origPic, int poc) if (origPic->m_param->bEnableHME || origPic->m_param->bEnableTemporalFilter) { - primitives.frameInitLowerRes(lowresPlane[0], - lowerResPlane[0], lowerResPlane[1], lowerResPlane[2], lowerResPlane[3], - lumaStride, lumaStride/2, (width / 2), (lines / 2)); + if (bEnableTemporalFilter) + frameInitLowresCoreMCSTF(lowresPlane[0], lowerResPlane[0], lowerResPlane[1], lowerResPlane[2], lowerResPlane[3], lumaStride, lumaStride / 2, (width / 2), (lines / 2)); + else + primitives.frameInitLowerRes(lowresPlane[0], lowerResPlane[0], lowerResPlane[1], lowerResPlane[2], lowerResPlane[3], lumaStride, lumaStride / 2, (width / 2), (lines / 2)); + extendPicBorder(lowerResPlane[0], lumaStride/2, width/2, lines/2, origPic->m_lumaMarginX/2, origPic->m_lumaMarginY/2); extendPicBorder(lowerResPlane[1], lumaStride/2, width/2, lines/2, origPic->m_lumaMarginX/2, origPic->m_lumaMarginY/2); extendPicBorder(lowerResPlane[2], lumaStride/2, width/2, lines/2, origPic->m_lumaMarginX/2, origPic->m_lumaMarginY/2); diff --git a/source/common/lowres.h b/source/common/lowres.h index f4b843f5e..fa64ea4f5 100644 --- a/source/common/lowres.h +++ b/source/common/lowres.h @@ -177,6 +177,8 @@ struct Lowres : public ReferencePlanes bool bKeyframe; bool bLastMiniGopBFrame; bool bIsFadeEnd; + bool filterThisGOP; // Set by lookahead noise gate: true = GOP start was noisy, apply MCSTF + int32_t noiseScore; // Raw noise score from estimateNoise() at GOP start; -1 for non-GOP-start frames double ipCostRatio; @@ -192,7 +194,6 @@ struct Lowres : public ReferencePlanes uint16_t* lowresCosts[X265_BFRAME_MAX + 2][X265_BFRAME_MAX + 2]; int32_t* lowresMvCosts[2][X265_BFRAME_MAX + 2]; MV* lowresMvs[2][X265_BFRAME_MAX + 2]; - MV* lowresMcstfMvs[2][4]; uint32_t maxBlocksInRow; uint32_t maxBlocksInCol; uint32_t maxBlocksInRowFullRes; @@ -253,7 +254,7 @@ struct Lowres : public ReferencePlanes bool create(x265_param* param, PicYuv *origPic, uint32_t qgSize); void destroy(x265_param* param); - void init(PicYuv *origPic, int poc); + void init(PicYuv* origPic, int poc, bool bEnableTemporalFilter); }; } diff --git a/source/common/param.cpp b/source/common/param.cpp index c09214d37..88d4069a7 100755 --- a/source/common/param.cpp +++ b/source/common/param.cpp @@ -418,10 +418,7 @@ void x265_param_default(x265_param* param) /* MCSTF */ param->bEnableTemporalFilter = 0; - param->temporalFilterStrength = 0.95; - param->searchRangeForLayer0 = 3; - param->searchRangeForLayer1 = 3; - param->searchRangeForLayer2 = 3; + param->bSelectiveMCSTF = 0; /* Threaded ME */ param->tmeTaskBlockSize = 1; @@ -1500,6 +1497,8 @@ int x265_param_parse(x265_param* p, const char* name, const char* value) OPT("film-grain") p->filmGrain = (char* )value; OPT("aom-film-grain") p->aomFilmGrain = (char*)value; OPT("mcstf") p->bEnableTemporalFilter = atobool(value); + OPT("mcstf-ref-range") p->mcstfFrameRange = atoi(value); + OPT("selective-mcstf") p->bSelectiveMCSTF = atobool(value); OPT("sbrc") p->bEnableSBRC = atobool(value); #if ENABLE_ALPHA OPT("alpha") @@ -1959,6 +1958,11 @@ int x265_check_params(x265_param* param) } } } + if (param->mcstfFrameRange > 4) + { + param->mcstfFrameRange = 4; + x265_log(param, X265_LOG_WARNING, "MCSTF reference range should not exceed 4. Setting MCSTF reference range to 4\n"); + } if (param->bEnableHME) { for (int level = 0; level < 3; level++) @@ -2229,7 +2233,11 @@ void x265_print_params(x265_param* param) TOOLOPT(param->toneMapFile != NULL, "dhdr10-info"); #endif if(param->bEnableTemporalFilter) + { TOOLOPT(param->bEnableTemporalFilter, "mcstf"); + TOOLVAL(param->mcstfFrameRange, "mcstf-ref-range=%d"); + TOOLOPT(param->bSelectiveMCSTF, "selective-mcstf"); + } x265_log(param, X265_LOG_INFO, "tools:%s\n", buf); fflush(stderr); } @@ -2493,6 +2501,11 @@ char *x265_param2string(x265_param* p, int padx, int pady) if (p->aomFilmGrain) s += snprintf(s, bufSize - (s - buf), " aom-film-grain=%s", p->aomFilmGrain); BOOL(p->bEnableTemporalFilter, "mcstf"); + if (p->bEnableTemporalFilter) + { + s += snprintf(s, bufSize - (s - buf), " mcstf-ref-range=%d", p->mcstfFrameRange); + BOOL(p->bSelectiveMCSTF, "selective-mcstf"); + } #if ENABLE_ALPHA BOOL(p->bEnableAlpha, "alpha"); #endif @@ -2727,6 +2740,7 @@ bool parseMaskingStrength(x265_param* p, const char* value) void x265_copy_params(x265_param* dst, x265_param* src) { dst->mcstfFrameRange = src->mcstfFrameRange; + dst->bSelectiveMCSTF = src->bSelectiveMCSTF; dst->cpuid = src->cpuid; dst->frameNumThreads = src->frameNumThreads; if (strlen(src->numaPools)) snprintf(dst->numaPools, X265_MAX_STRING_SIZE, "%s", src->numaPools); @@ -3025,10 +3039,6 @@ void x265_copy_params(x265_param* dst, x265_param* src) } dst->bField = src->bField; dst->bEnableTemporalFilter = src->bEnableTemporalFilter; - dst->temporalFilterStrength = src->temporalFilterStrength; - dst->searchRangeForLayer0 = src->searchRangeForLayer0; - dst->searchRangeForLayer1 = src->searchRangeForLayer1; - dst->searchRangeForLayer2 = src->searchRangeForLayer2; dst->confWinRightOffset = src->confWinRightOffset; dst->confWinBottomOffset = src->confWinBottomOffset; dst->bliveVBV2pass = src->bliveVBV2pass; diff --git a/source/common/pixel.cpp b/source/common/pixel.cpp index 06e5732bd..e90ef5207 100644 --- a/source/common/pixel.cpp +++ b/source/common/pixel.cpp @@ -619,23 +619,6 @@ void frame_init_lowres_core(const pixel* src0, pixel* dst0, pixel* dsth, pixel* } } -static -void frame_subsample_luma(const pixel* src0, pixel* dst0, intptr_t src_stride, intptr_t dst_stride, int width, int height) -{ - for (int y = 0; y < height; y++, src0 += 2 * src_stride, dst0 += dst_stride) - { - const pixel *inRow = src0; - const pixel *inRowBelow = src0 + src_stride; - pixel *target = dst0; - for (int x = 0; x < width; x++) - { - target[x] = (((inRow[0] + inRowBelow[0] + 1) >> 1) + ((inRow[1] + inRowBelow[1] + 1) >> 1) + 1) >> 1; - inRow += 2; - inRowBelow += 2; - } - } -} - /* structural similarity metric */ static void ssim_4x4x2_core(const pixel* pix1, intptr_t stride1, const pixel* pix2, intptr_t stride2, int sums[2][4]) { @@ -1030,6 +1013,31 @@ static pixel planeClipAndMax_c(pixel *src, intptr_t stride, int width, int heigh namespace X265_NS { // x265 private namespace +/* Scalar lowres downscale for MCSTF path - HM-equivalent filter, no SIMD override */ + void frameInitLowresCoreMCSTF(const pixel* src0, pixel* dst0, pixel* dsth, pixel* dstv, pixel* dstc, + intptr_t src_stride, intptr_t dst_stride, int width, int height) + { + for (int y = 0; y < height; y++) + { + const pixel* src1 = src0 + src_stride; + const pixel* src2 = src1 + src_stride; + for (int x = 0; x < width; x++) + { +#define FILTER(a, b, c, d) ((a + b + c + d + 2) >> 2) + dst0[x] = FILTER(src0[2 * x], src1[2 * x], src0[2 * x + 1], src1[2 * x + 1]); + dsth[x] = FILTER(src0[2 * x + 1], src1[2 * x + 1], src0[2 * x + 2], src1[2 * x + 2]); + dstv[x] = FILTER(src1[2 * x], src2[2 * x], src1[2 * x + 1], src2[2 * x + 1]); + dstc[x] = FILTER(src1[2 * x + 1], src2[2 * x + 1], src1[2 * x + 2], src2[2 * x + 2]); +#undef FILTER + } + src0 += src_stride * 2; + dst0 += dst_stride; + dsth += dst_stride; + dstv += dst_stride; + dstc += dst_stride; + } + } + /* Extend the edges of a picture so that it may safely be used for motion * compensation. This function assumes the picture is stored in a buffer with * sufficient padding for the X and Y margins */ @@ -1364,7 +1372,5 @@ void setupPixelPrimitives_c(EncoderPrimitives &p) p.cu[BLOCK_16x16].normFact = normFact_c; p.cu[BLOCK_32x32].normFact = normFact_c; p.cu[BLOCK_64x64].normFact = normFact_c; - /* SubSample Luma*/ - p.frameSubSampleLuma = frame_subsample_luma; } } diff --git a/source/common/primitives.cpp b/source/common/primitives.cpp index 55f702c7b..427fbc1df 100644 --- a/source/common/primitives.cpp +++ b/source/common/primitives.cpp @@ -23,6 +23,7 @@ #include "common.h" #include "primitives.h" +#include "temporalfilter.h" namespace X265_NS { // x265 private namespace @@ -253,6 +254,9 @@ void x265_setup_primitives(x265_param *param) { setupCPrimitives(primitives); + /* MCSTF primitives: scalar defaults, then x86 SIMD overrides on capable CPUs */ + setupMCSTFPrimitives_scalar(mcstfPrim); + /* We do not want the encoder to use the un-optimized intra all-angles * C references. It is better to call the individual angle functions * instead. We must check for NULL before using this primitive */ diff --git a/source/common/primitives.h b/source/common/primitives.h index fd343f5fe..16a4c6e02 100644 --- a/source/common/primitives.h +++ b/source/common/primitives.h @@ -355,8 +355,7 @@ struct EncoderPrimitives downscale_t frameInitLowres; downscale_t frameInitLowerRes; - /* Sub Sample Luma */ - downscaleluma_t frameSubSampleLuma; + cutree_propagate_cost propagateCost; cutree_fix8_unpack fix8Unpack; cutree_fix8_pack fix8Pack; diff --git a/source/common/temporalfilter.cpp b/source/common/temporalfilter.cpp index e26628db2..7c3bdc683 100644 --- a/source/common/temporalfilter.cpp +++ b/source/common/temporalfilter.cpp @@ -23,7 +23,7 @@ #include "common.h" #include "temporalfilter.h" #include "primitives.h" - +#include "threadpool.h" #include "frame.h" #include "slice.h" #include "framedata.h" @@ -31,6 +31,230 @@ using namespace X265_NS; +namespace X265_NS { + + /* MCSTF scalar fallbacks - used when the runtime CPU lacks the required ISA.*/ + + static int motionErrorLumaFrac_c( + const pixel* origOrigin, intptr_t origStride, + const pixel* buffOrigin, intptr_t buffStride, + int x, int y, int dx, int dy, + int bs, int besterror, int bitDepth) + { + const int* xFilter = s_interpolationFilter[dx & 0xF]; + const int* yFilter = s_interpolationFilter[dy & 0xF]; + int tempArray[64 + 8][64]; + int error = 0; + + for (int y1 = 1; y1 < bs + 7; y1++) + { + const int yOffset = y + y1 + (dy >> 4) - 3; + const pixel* sourceRow = buffOrigin + yOffset * buffStride; + for (int x1 = 0; x1 < bs; x1++) + { + int iBase = x + x1 + (dx >> 4) - 3; + const pixel* rowStart = sourceRow + iBase; + int iSum = 0; + iSum += xFilter[1] * rowStart[1]; + iSum += xFilter[2] * rowStart[2]; + iSum += xFilter[3] * rowStart[3]; + iSum += xFilter[4] * rowStart[4]; + iSum += xFilter[5] * rowStart[5]; + iSum += xFilter[6] * rowStart[6]; + tempArray[y1][x1] = iSum; + } + } + + const int maxSampleValue = (1 << bitDepth) - 1; + for (int y1 = 0; y1 < bs; y1++) + { + const pixel* origRow = origOrigin + (y + y1) * origStride; + for (int x1 = 0; x1 < bs; x1++) + { + int iSum = 0; + iSum += yFilter[1] * tempArray[y1 + 1][x1]; + iSum += yFilter[2] * tempArray[y1 + 2][x1]; + iSum += yFilter[3] * tempArray[y1 + 3][x1]; + iSum += yFilter[4] * tempArray[y1 + 4][x1]; + iSum += yFilter[5] * tempArray[y1 + 5][x1]; + iSum += yFilter[6] * tempArray[y1 + 6][x1]; + iSum = (iSum + (1 << 11)) >> 12; + iSum = iSum < 0 ? 0 : (iSum > maxSampleValue ? maxSampleValue : iSum); + + int diff = iSum - origRow[x + x1]; + error += diff * diff; + } + if (error > besterror) + return error; + } + return error; + } + static void applyMotion_c( + const pixel* pSrcImage, int srcStride, + pixel* pDstImage, int dstStride, + int width, int height, + int blockSizeX, int blockSizeY, + uint32_t mvsStride, const MV* mvs, + int csx, int csy, + int blockRow, int rowSize, int vShift) + { + static const int numFilterTaps = 7; + static const int centreTapOffset = 3; + const int maxValue = (1 << X265_DEPTH) - 1; + + const int blkRowStart = (!rowSize) ? 0 : (blockRow * rowSize) >> vShift; + const int blkRowEnd = (!rowSize) ? height : X265_MIN((blockRow * rowSize + rowSize) >> vShift, height); + int blockNumY = (!rowSize) ? 0 : blkRowStart / blockSizeY; + + for (int y = blkRowStart; + y + blockSizeY <= blkRowEnd; + y += blockSizeY, blockNumY++) + { + for (int x = 0, blockNumX = 0; + x + blockSizeX <= width; + x += blockSizeX, blockNumX++) + { + const int mvIdx = blockNumY * (int)mvsStride + blockNumX; + const MV& mv = mvs[mvIdx]; + + const int dx = mv.x >> csx; + const int dy = mv.y >> csy; + const int xInt = mv.x >> (4 + csx); + const int yInt = mv.y >> (4 + csy); + + const int* xFilter = s_interpolationFilter[dx & 0xf]; + const int* yFilter = s_interpolationFilter[dy & 0xf]; + + int tempArray[8 + numFilterTaps][8]; + + for (int by = 1; by < blockSizeY + numFilterTaps; by++) + { + const int yOffset = y + by + yInt - centreTapOffset; + const pixel* sourceRow = pSrcImage + yOffset * srcStride; + + for (int bx = 0; bx < blockSizeX; bx++) + { + int iBase = x + bx + xInt - centreTapOffset; + const pixel* rowStart = sourceRow + iBase; + + int iSum = 0; + iSum += xFilter[1] * rowStart[1]; + iSum += xFilter[2] * rowStart[2]; + iSum += xFilter[3] * rowStart[3]; + iSum += xFilter[4] * rowStart[4]; + iSum += xFilter[5] * rowStart[5]; + iSum += xFilter[6] * rowStart[6]; + + tempArray[by][bx] = iSum; + } + } + + pixel* pDstRow = pDstImage + y * dstStride; + + for (int by = 0; by < blockSizeY; by++, pDstRow += dstStride) + { + pixel* pDstPel = pDstRow + x; + + for (int bx = 0; bx < blockSizeX; bx++, pDstPel++) + { + int iSum = 0; + + iSum += yFilter[1] * tempArray[by + 1][bx]; + iSum += yFilter[2] * tempArray[by + 2][bx]; + iSum += yFilter[3] * tempArray[by + 3][bx]; + iSum += yFilter[4] * tempArray[by + 4][bx]; + iSum += yFilter[5] * tempArray[by + 5][bx]; + iSum += yFilter[6] * tempArray[by + 6][bx]; + + iSum = (iSum + (1 << 11)) >> 12; + iSum = iSum < 0 ? 0 : (iSum > maxValue ? maxValue : iSum); + + *pDstPel = (pixel)iSum; + } + } + } + } + } + + static void computeBlockStats_c( + const pixel* srcPel, intptr_t srcStride, + const pixel* refPel, intptr_t refStride, + int blkSize, int* outVariance, int* outDiffsum) + { + int variance = 0, diffsum = 0; + for (int y1 = 0; y1 < blkSize; y1++) + { + for (int x1 = 0; x1 < blkSize; x1++) + { + int diff = *(srcPel + srcStride * y1 + x1) + - *(refPel + refStride * y1 + x1); + variance += diff * diff; + if (x1 != blkSize - 1) + { + int diffR = *(srcPel + srcStride * y1 + x1 + 1) + - *(refPel + refStride * y1 + x1 + 1); + diffsum += (diffR - diff) * (diffR - diff); + } + if (y1 != blkSize - 1) + { + int diffD = *(srcPel + srcStride * (y1 + 1) + x1) + - *(refPel + refStride * (y1 + 1) + x1); + diffsum += (diffD - diff) * (diffD - diff); + } + } + } + *outVariance = variance; + *outDiffsum = diffsum; + } + + static void bilateralFilter_c( + const pixel* srcBlk, intptr_t srcStride, + int numRefs, + const pixel* const* refBlks, + const intptr_t* refStrides, + const double* vww, + const double* vsw, + double bdw, + double maxSample, + int blkSize, + pixel* dstBlk, intptr_t dstStride) + { + for (int y = 0; y < blkSize; y++) + { + for (int x = 0; x < blkSize; x++) + { + const int orgVal = (int)srcBlk[y * srcStride + x]; + double temporalWeightSum = 1.0; + double newVal = (double)orgVal; + for (int i = 0; i < numRefs; i++) + { + const int refVal = (int)refBlks[i][y * refStrides[i] + x]; + double diff = (double)(refVal - orgVal) * bdw; + double diffSq = diff * diff; + const double weight = vww[i] * exp(-diffSq / vsw[i]); + newVal += weight * refVal; + temporalWeightSum += weight; + } + newVal /= temporalWeightSum; + double sampleVal = round(newVal); + sampleVal = (sampleVal < 0 ? 0 : (sampleVal > maxSample ? maxSample : sampleVal)); + dstBlk[y * dstStride + x] = (pixel)sampleVal; + } + } + } + + /* Global MCSTF primitives table */ + MCSTFPrimitives mcstfPrim; + + void setupMCSTFPrimitives_scalar(MCSTFPrimitives& p) + { + p.motionErrorLumaFrac = motionErrorLumaFrac_c; + p.applyMotion = applyMotion_c; + p.computeBlockStats = computeBlockStats_c; + p.bilateralFilter = bilateralFilter_c; + } +}/* namespace X265_NS */ + void OrigPicBuffer::addPicture(Frame* inFrame) { m_mcstfPicList.pushFrontMCSTF(*inFrame); @@ -149,6 +373,7 @@ TemporalFilter::TemporalFilter() m_chromaFactor = 0.55; m_sigmaMultiplier = 9.0; m_sigmaZeroPoint = 10.0; + m_overallStrength = 0.95; } TemporalFilter::~TemporalFilter() @@ -193,113 +418,7 @@ int TemporalFilter::createRefPicInfo(TemporalFilterRefPicInfo* refFrame, x265_pa return 0; } -int MotionEstimatorTLD::motionErrorLumaSAD(MotionEstimatorTLD& m_metld, - pixel* src, - int stride, - pixel* buf, - int x, - int y, - int dx, - int dy, - int bs, - int besterror) -{ - - pixel* origOrigin = src; - intptr_t origStride = stride; - pixel *buffOrigin = buf; - intptr_t buffStride = stride; - int error = 0;// dx * 10 + dy * 10; - if (((dx | dy) & 0xF) == 0) - { - dx /= m_motionVectorFactor; - dy /= m_motionVectorFactor; - - const pixel* bufferRowStart = buffOrigin + (y + dy) * buffStride + (x + dx); -#if 0 - const pixel* origRowStart = origOrigin + y *origStride + x; - - for (int y1 = 0; y1 < bs; y1++) - { - for (int x1 = 0; x1 < bs; x1++) - { - int diff = origRowStart[x1] - bufferRowStart[x1]; - error += abs(diff); - } - - origRowStart += origStride; - bufferRowStart += buffStride; - } -#else - int partEnum = partitionFromSizes(bs, bs); - /* copy PU block into cache */ - primitives.pu[partEnum].copy_pp(predPUYuv.m_buf[0], FENC_STRIDE, bufferRowStart, buffStride); - - error = m_metld.me.bufSAD(predPUYuv.m_buf[0], FENC_STRIDE); -#endif - if (error > besterror) - { - return error; - } - } - else - { - const int *xFilter = s_interpolationFilter[dx & 0xF]; - const int *yFilter = s_interpolationFilter[dy & 0xF]; - int tempArray[64 + 8][64]; - - int iSum, iBase; - for (int y1 = 1; y1 < bs + 7; y1++) - { - const int yOffset = y + y1 + (dy >> 4) - 3; - const pixel *sourceRow = buffOrigin + (yOffset)*buffStride + 0; - for (int x1 = 0; x1 < bs; x1++) - { - iSum = 0; - iBase = x + x1 + (dx >> 4) - 3; - const pixel *rowStart = sourceRow + iBase; - - iSum += xFilter[1] * rowStart[1]; - iSum += xFilter[2] * rowStart[2]; - iSum += xFilter[3] * rowStart[3]; - iSum += xFilter[4] * rowStart[4]; - iSum += xFilter[5] * rowStart[5]; - iSum += xFilter[6] * rowStart[6]; - - tempArray[y1][x1] = iSum; - } - } - - const pixel maxSampleValue = (1 << m_bitDepth) - 1; - for (int y1 = 0; y1 < bs; y1++) - { - const pixel *origRow = origOrigin + (y + y1)*origStride + 0; - for (int x1 = 0; x1 < bs; x1++) - { - iSum = 0; - iSum += yFilter[1] * tempArray[y1 + 1][x1]; - iSum += yFilter[2] * tempArray[y1 + 2][x1]; - iSum += yFilter[3] * tempArray[y1 + 3][x1]; - iSum += yFilter[4] * tempArray[y1 + 4][x1]; - iSum += yFilter[5] * tempArray[y1 + 5][x1]; - iSum += yFilter[6] * tempArray[y1 + 6][x1]; - - iSum = (iSum + (1 << 11)) >> 12; - iSum = iSum < 0 ? 0 : (iSum > maxSampleValue ? maxSampleValue : iSum); - - error += abs(iSum - origRow[x + x1]); - } - if (error > besterror) - { - return error; - } - } - } - return error; -} - -int MotionEstimatorTLD::motionErrorLumaSSD(MotionEstimatorTLD& m_metld, - pixel* src, +int MotionEstimatorTLD::motionErrorLumaSSD(pixel* src, int stride, pixel* buf, int x, @@ -340,7 +459,7 @@ int MotionEstimatorTLD::motionErrorLumaSSD(MotionEstimatorTLD& m_metld, /* copy PU block into cache */ primitives.pu[partEnum].copy_pp(predPUYuv.m_buf[0], FENC_STRIDE, bufferRowStart, buffStride); - error = (int)primitives.cu[partEnum].sse_pp(m_metld.me.fencPUYuv.m_buf[0], FENC_STRIDE, predPUYuv.m_buf[0], FENC_STRIDE); + error = (int)primitives.cu[partEnum].sse_pp(me.fencPUYuv.m_buf[0], FENC_STRIDE, predPUYuv.m_buf[0], FENC_STRIDE); #endif if (error > besterror) @@ -350,61 +469,15 @@ int MotionEstimatorTLD::motionErrorLumaSSD(MotionEstimatorTLD& m_metld, } else { - const int *xFilter = s_interpolationFilter[dx & 0xF]; - const int *yFilter = s_interpolationFilter[dy & 0xF]; - int tempArray[64 + 8][64]; - - int iSum, iBase; - for (int y1 = 1; y1 < bs + 7; y1++) - { - const int yOffset = y + y1 + (dy >> 4) - 3; - const pixel *sourceRow = buffOrigin + (yOffset)*buffStride + 0; - for (int x1 = 0; x1 < bs; x1++) - { - iSum = 0; - iBase = x + x1 + (dx >> 4) - 3; - const pixel *rowStart = sourceRow + iBase; - - iSum += xFilter[1] * rowStart[1]; - iSum += xFilter[2] * rowStart[2]; - iSum += xFilter[3] * rowStart[3]; - iSum += xFilter[4] * rowStart[4]; - iSum += xFilter[5] * rowStart[5]; - iSum += xFilter[6] * rowStart[6]; - - tempArray[y1][x1] = iSum; - } - } - - const pixel maxSampleValue = (1 << m_bitDepth) - 1; - for (int y1 = 0; y1 < bs; y1++) - { - const pixel *origRow = origOrigin + (y + y1)*origStride + 0; - for (int x1 = 0; x1 < bs; x1++) - { - iSum = 0; - iSum += yFilter[1] * tempArray[y1 + 1][x1]; - iSum += yFilter[2] * tempArray[y1 + 2][x1]; - iSum += yFilter[3] * tempArray[y1 + 3][x1]; - iSum += yFilter[4] * tempArray[y1 + 4][x1]; - iSum += yFilter[5] * tempArray[y1 + 5][x1]; - iSum += yFilter[6] * tempArray[y1 + 6][x1]; - - iSum = (iSum + (1 << 11)) >> 12; - iSum = iSum < 0 ? 0 : (iSum > maxSampleValue ? maxSampleValue : iSum); - - error += (iSum - origRow[x + x1]) * (iSum - origRow[x + x1]); - } - if (error > besterror) - { - return error; - } - } + error = mcstfPrim.motionErrorLumaFrac( + origOrigin, origStride, buffOrigin, buffStride, + x, y, dx, dy, bs, besterror, m_bitDepth); + if (error > besterror) return error; } return error; } -void TemporalFilter::applyMotion(MV *mvs, uint32_t mvsStride, PicYuv *input, PicYuv *output) +void TemporalFilter::applyMotion(MV *mvs, uint32_t mvsStride, PicYuv *input, PicYuv *output, const int blockRow, const int rowSize) { static const int lumaBlockSize = 8; int srcStride = 0; @@ -412,10 +485,9 @@ void TemporalFilter::applyMotion(MV *mvs, uint32_t mvsStride, PicYuv *input, Pic int csx = 0, csy = 0; for (int c = 0; c < m_numComponents; c++) { - const pixel maxValue = (1 << X265_DEPTH) - 1; - const pixel *pSrcImage = input->m_picOrg[c]; - pixel *pDstImage = output->m_picOrg[c]; + const pixel* pSrcImage = input->m_picOrg[c]; + pixel* pDstImage = output->m_picOrg[c]; if (!c) { @@ -434,201 +506,85 @@ void TemporalFilter::applyMotion(MV *mvs, uint32_t mvsStride, PicYuv *input, Pic const int height = input->m_picHeight >> csy; const int width = input->m_picWidth >> csx; - for (int y = 0, blockNumY = 0; y + blockSizeY <= height; y += blockSizeY, blockNumY++) - { - for (int x = 0, blockNumX = 0; x + blockSizeX <= width; x += blockSizeX, blockNumX++) - { - int mvIdx = blockNumY * mvsStride + blockNumX; - const MV &mv = mvs[mvIdx]; - const int dx = mv.x >> csx; - const int dy = mv.y >> csy; - const int xInt = mv.x >> (4 + csx); - const int yInt = mv.y >> (4 + csy); - - const int *xFilter = s_interpolationFilter[dx & 0xf]; - const int *yFilter = s_interpolationFilter[dy & 0xf]; // will add 6 bit. - const int numFilterTaps = 7; - const int centreTapOffset = 3; - - int tempArray[lumaBlockSize + numFilterTaps][lumaBlockSize]; - - for (int by = 1; by < blockSizeY + numFilterTaps; by++) - { - const int yOffset = y + by + yInt - centreTapOffset; - const pixel *sourceRow = pSrcImage + yOffset * srcStride; - for (int bx = 0; bx < blockSizeX; bx++) - { - int iBase = x + bx + xInt - centreTapOffset; - const pixel *rowStart = sourceRow + iBase; - - int iSum = 0; - iSum += xFilter[1] * rowStart[1]; - iSum += xFilter[2] * rowStart[2]; - iSum += xFilter[3] * rowStart[3]; - iSum += xFilter[4] * rowStart[4]; - iSum += xFilter[5] * rowStart[5]; - iSum += xFilter[6] * rowStart[6]; + const int vShift = (!c) ? 0 : csy; - tempArray[by][bx] = iSum; - } - } - - pixel *pDstRow = pDstImage + y * dstStride; - for (int by = 0; by < blockSizeY; by++, pDstRow += dstStride) - { - pixel *pDstPel = pDstRow + x; - for (int bx = 0; bx < blockSizeX; bx++, pDstPel++) - { - int iSum = 0; - - iSum += yFilter[1] * tempArray[by + 1][bx]; - iSum += yFilter[2] * tempArray[by + 2][bx]; - iSum += yFilter[3] * tempArray[by + 3][bx]; - iSum += yFilter[4] * tempArray[by + 4][bx]; - iSum += yFilter[5] * tempArray[by + 5][bx]; - iSum += yFilter[6] * tempArray[by + 6][bx]; - - iSum = (iSum + (1 << 11)) >> 12; - iSum = iSum < 0 ? 0 : (iSum > maxValue ? maxValue : iSum); - *pDstPel = (pixel)iSum; - } - } - } - } + mcstfPrim.applyMotion(pSrcImage, srcStride, pDstImage, dstStride, width, height, blockSizeX, blockSizeY, mvsStride, mvs, csx, csy, blockRow, rowSize, vShift); } } -void TemporalFilter::bilateralFilter(Frame* frame, - TemporalFilterRefPicInfo* m_mcstfRefList, - double overallStrength) +void TemporalFilter::bilateralFilterCore(Frame* frame, TemporalFilterRefPicInfo* m_mcstfRefList, int numRefs, int blockRow, int rowSize) { - const int numRefs = frame->m_mcstf->m_numRef; - - for (int i = 0; i < numRefs; i++) - { - TemporalFilterRefPicInfo *ref = &m_mcstfRefList[i]; - applyMotion(m_mcstfRefList[i].mvs, m_mcstfRefList[i].mvsStride, m_mcstfRefList[i].picBuffer, ref->compensatedPic); - } - - int refStrengthRow = 2; - if (numRefs == m_range * 2) - { - refStrengthRow = 0; - } - else if (numRefs == m_range) - { - refStrengthRow = 1; - } + int refStrengthRow = 0; const double lumaSigmaSq = (m_QP - m_sigmaZeroPoint) * (m_QP - m_sigmaZeroPoint) * m_sigmaMultiplier; const double chromaSigmaSq = 30 * 30; - PicYuv* orgPic = frame->m_fencPic; - for (int c = 0; c < m_numComponents; c++) + for (int i = 0; i < numRefs; i++) { - int height, width; - pixel *srcPelRow = NULL; - intptr_t srcStride, correctedPicsStride = 0; - - if (!c) - { - height = orgPic->m_picHeight; - width = orgPic->m_picWidth; - srcPelRow = orgPic->m_picOrg[c]; - srcStride = orgPic->m_stride; - } - else - { - int csx = CHROMA_H_SHIFT(m_internalCsp); - int csy = CHROMA_V_SHIFT(m_internalCsp); - - height = orgPic->m_picHeight >> csy; - width = orgPic->m_picWidth >> csx; - srcPelRow = orgPic->m_picOrg[c]; - srcStride = (int)orgPic->m_strideC; - } - - const double sigmaSq = (!c) ? lumaSigmaSq : chromaSigmaSq; - const double weightScaling = overallStrength * ( (!c) ? 0.4 : m_chromaFactor); + TemporalFilterRefPicInfo* ref = &m_mcstfRefList[i]; + applyMotion(m_mcstfRefList[i].mvs, m_mcstfRefList[i].mvsStride, m_mcstfRefList[i].picBuffer, ref->compensatedPic, blockRow, rowSize); + } + for (int c = 0; c < m_numComponents; c++) + { + const int csx = (!c) ? 0 : CHROMA_H_SHIFT(m_internalCsp); + const int csy = (!c) ? 0 : CHROMA_V_SHIFT(m_internalCsp); + const int height = (!c) ? orgPic->m_picHeight : orgPic->m_picHeight >> csy; + const int width = (!c) ? orgPic->m_picWidth : orgPic->m_picWidth >> csx; + pixel* srcPelPlane = orgPic->m_picOrg[c]; + const intptr_t srcStride = (!c) ? orgPic->m_stride : (intptr_t)orgPic->m_strideC; + + const double sigmaSq = (!c) ? lumaSigmaSq : chromaSigmaSq; + const double weightScaling = m_overallStrength * ((!c) ? 0.4 : m_chromaFactor); const double maxSampleValue = (1 << m_bitDepth) - 1; const double bitDepthDiffWeighting = 1024.0 / (maxSampleValue + 1); - const int blkSize = (!c) ? 8 : 4; - for (int y = 0; y < height; y++, srcPelRow += srcStride) - { - pixel *srcPel = srcPelRow; + const int vShift = (!c) ? 0 : csy; + const int planeRowStart = (!rowSize) ? 0 : (blockRow * rowSize) >> vShift; + const int planeRowEnd = (!rowSize) ? height : X265_MIN((blockRow * rowSize + rowSize) >> vShift, height); + const int blkRowStart = (planeRowStart / blkSize) * blkSize; + const int blkRowEnd = X265_MIN(((planeRowEnd + blkSize - 1) / blkSize) * blkSize, height); - for (int x = 0; x < width; x++, srcPel++) + for (int by = blkRowStart; by + blkSize <= blkRowEnd; by += blkSize) + { + for (int bx = 0; bx + blkSize <= width; bx += blkSize) { - const int orgVal = (int)*srcPel; - double temporalWeightSum = 1.0; - double newVal = (double)orgVal; - - if ((y % blkSize == 0) && (x % blkSize == 0)) - { - for (int i = 0; i < numRefs; i++) - { - TemporalFilterRefPicInfo *refPicInfo = &m_mcstfRefList[i]; - - if (!c) - correctedPicsStride = refPicInfo->compensatedPic->m_stride; - else - correctedPicsStride = refPicInfo->compensatedPic->m_strideC; - - const intptr_t pelOffset = y * correctedPicsStride + x; - primitives.pu[1].copy_pp(m_metld->me.fencPUYuv.m_buf[0], FENC_STRIDE, refPicInfo->compensatedPic->m_picOrg[c] + pelOffset, correctedPicsStride); + const pixel* srcPel = srcPelPlane + by * srcStride + bx; - double variance = 0, diffsum = 0; - for (int y1 = 0; y1 < blkSize - 1; y1++) - { - for (int x1 = 0; x1 < blkSize - 1; x1++) - { - int pix = *(srcPel + x1); - int pixR = *(srcPel + x1 + 1); - int pixD = *(srcPel + x1 + srcStride); - - int ref = *(m_metld->me.fencPUYuv.m_buf[0] + ((y1)*FENC_STRIDE + x1)); - int refR = *(m_metld->me.fencPUYuv.m_buf[0] + ((y1)*FENC_STRIDE + x1 + 1)); - int refD = *(m_metld->me.fencPUYuv.m_buf[0] + ((y1 + 1) * FENC_STRIDE + x1)); - - int diff = pix - ref; - int diffR = pixR - refR; - int diffD = pixD - refD; - - variance += diff * diff; - diffsum += (diffR - diff) * (diffR - diff); - diffsum += (diffD - diff) * (diffD - diff); - } - } - - refPicInfo->noise[(y / blkSize) * refPicInfo->mvsStride + (x / blkSize)] = (int)round((300 * variance + 50) / (10 * diffsum + 50)); - } - } - - double minError = 9999999; + // Step 1: noise computation via SIMD primitive + double minError = DBL_MAX; for (int i = 0; i < numRefs; i++) { - TemporalFilterRefPicInfo *refPicInfo = &m_mcstfRefList[i]; - minError = X265_MIN(minError, (double)refPicInfo->error[(y / blkSize) * refPicInfo->mvsStride + (x / blkSize)]); + TemporalFilterRefPicInfo* refPicInfo = &m_mcstfRefList[i]; + const intptr_t refStride = (!c) ? refPicInfo->compensatedPic->m_stride + : refPicInfo->compensatedPic->m_strideC; + const pixel* refPel = refPicInfo->compensatedPic->m_picOrg[c] + + by * refStride + bx; + + int iVariance = 0, iDiffsum = 0; + mcstfPrim.computeBlockStats( + srcPel, srcStride, refPel, refStride, + blkSize, &iVariance, &iDiffsum); + + const int cntV = blkSize * blkSize; + const int cntD = 2 * cntV - blkSize - blkSize; + refPicInfo->noise[(by / blkSize) * refPicInfo->mvsStride + (bx / blkSize)] = + (int)round((15.0 * cntD / cntV * iVariance + 5.0) / (iDiffsum + 5.0)); + minError = X265_MIN(minError, + (double)refPicInfo->error[(by / blkSize) * refPicInfo->mvsStride + (bx / blkSize)]); } + // Step 2: pre-compute vww / vsw (block-level) + double vww[MCSTF_MAX_REFS] = {}; + double vsw[MCSTF_MAX_REFS] = {}; for (int i = 0; i < numRefs; i++) { - TemporalFilterRefPicInfo *refPicInfo = &m_mcstfRefList[i]; - - const int error = refPicInfo->error[(y / blkSize) * refPicInfo->mvsStride + (x / blkSize)]; - const int noise = refPicInfo->noise[(y / blkSize) * refPicInfo->mvsStride + (x / blkSize)]; - - const pixel *pCorrectedPelPtr = refPicInfo->compensatedPic->m_picOrg[c] + (y * correctedPicsStride + x); - const int refVal = (int)*pCorrectedPelPtr; - double diff = (double)(refVal - orgVal); - diff *= bitDepthDiffWeighting; - double diffSq = diff * diff; - + TemporalFilterRefPicInfo* refPicInfo = &m_mcstfRefList[i]; + const int error = refPicInfo->error[(by / blkSize) * refPicInfo->mvsStride + (bx / blkSize)]; + const int noise = refPicInfo->noise[(by / blkSize) * refPicInfo->mvsStride + (bx / blkSize)]; const int index = X265_MIN(3, std::abs(refPicInfo->origOffset) - 1); double ww = 1, sw = 1; ww *= (noise < 25) ? 1 : 1.2; @@ -636,40 +592,78 @@ void TemporalFilter::bilateralFilter(Frame* frame, ww *= (error < 50) ? 1.2 : ((error > 100) ? 0.8 : 1); sw *= (error < 50) ? 1.3 : 1; ww *= ((minError + 1) / (error + 1)); - const double weight = weightScaling * s_refStrengths[refStrengthRow][index] * ww * exp(-diffSq / (2 * sw * sigmaSq)); + vww[i] = weightScaling * s_refStrengths[refStrengthRow][index] * ww; + vsw[i] = 2 * sw * sigmaSq; + } - newVal += weight * refVal; - temporalWeightSum += weight; + // Step 3: pixel filtering via SIMD primitive + const pixel* refBlkPtrs[MCSTF_MAX_REFS]; + intptr_t refBlkStrides[MCSTF_MAX_REFS]; + for (int i = 0; i < numRefs; i++) + { + TemporalFilterRefPicInfo* refPicInfo = &m_mcstfRefList[i]; + refBlkStrides[i] = (!c) ? refPicInfo->compensatedPic->m_stride + : refPicInfo->compensatedPic->m_strideC; + refBlkPtrs[i] = refPicInfo->compensatedPic->m_picOrg[c] + + by * refBlkStrides[i] + bx; } - newVal /= temporalWeightSum; - double sampleVal = round(newVal); - sampleVal = (sampleVal < 0 ? 0 : (sampleVal > maxSampleValue ? maxSampleValue : sampleVal)); - *srcPel = (pixel)sampleVal; + mcstfPrim.bilateralFilter(srcPel, srcStride, numRefs, refBlkPtrs, refBlkStrides, vww, vsw, bitDepthDiffWeighting, maxSampleValue, + blkSize, srcPelPlane + by * srcStride + bx, srcStride); } } } } -void MotionEstimatorTLD::motionEstimationLuma(MotionEstimatorTLD& m_metld, MV *mvs, uint32_t mvStride, pixel* src,int stride, int height, int width, pixel* buf, int blockSize, - int sRange, MV* previous, uint32_t prevMvStride, int factor) +// Splits the frame into 64-row blocks, dispatches jobs to the threadpool via BilateralFilterGroup, then returns. +void TemporalFilter::bilateralFilter(Frame* curFrame, TemporalFilterRefPicInfo* mcstfRefList, ThreadPool* pool) { + const int numRef = curFrame->m_mcstf->m_numRef; + const int rowSize = 64; + const int frameHeight = curFrame->m_fencPic->m_picHeight; + const int numBlockRows = (frameHeight + rowSize - 1) / rowSize; - int range = sRange; + if (numRef == 0) + return; + + if (!pool) + { + bilateralFilterCore(curFrame, mcstfRefList, numRef, 0, 0); + return; + } + + BilateralFilterGroup filterGroup(*this, pool); + + for (int row = 0; row < numBlockRows; row++) + filterGroup.add(curFrame, mcstfRefList, numRef, row, rowSize); + + filterGroup.finishBatch(); +} + +void MotionEstimatorTLD::motionEstimationLuma(MV *mvs, uint32_t mvStride, pixel* src,int stride, int height, int width, pixel* buf, + int row, const int rowSize, const MV* previous, uint32_t prevMvStride, int factor) +{ + int range = m_searchRange; + int stepSize, blockSize; - const int stepSize = blockSize; + stepSize = blockSize = m_blockSize; const int origWidth = width; const int origHeight = height; + int rowStart = row * rowSize; + if (rowStart > height) + return; + + int rowEnd = (!rowSize) ? height : X265_MIN(rowStart + rowSize, height); int error; - for (int blockY = 0; blockY + blockSize <= origHeight; blockY += stepSize) + for (int blockY = rowStart; blockY + blockSize <= rowEnd; blockY += stepSize) { for (int blockX = 0; blockX + blockSize <= origWidth; blockX += stepSize) { const intptr_t pelOffset = blockY * stride + blockX; - m_metld.me.setSourcePU(src, stride, pelOffset, blockSize, blockSize, X265_HEX_SEARCH, 1); + me.setSourcePU(src, stride, pelOffset, blockSize, blockSize, X265_HEX_SEARCH, 1); MV best(0, 0); @@ -677,7 +671,7 @@ void MotionEstimatorTLD::motionEstimationLuma(MotionEstimatorTLD& m_metld, MV *m if (previous == NULL) { - range = sRange; + range = m_searchRange; } else { @@ -695,10 +689,7 @@ void MotionEstimatorTLD::motionEstimationLuma(MotionEstimatorTLD& m_metld, MV *m int mvIdx = testy * prevMvStride + testx; MV old = previous[mvIdx]; - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, src, stride, buf, blockX, blockY, old.x * factor, old.y * factor, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, src, stride, buf, blockX, blockY, old.x * factor, old.y * factor, blockSize, leastError); + error = motionErrorLumaSSD(src, stride, buf, blockX, blockY, old.x * factor, old.y * factor, blockSize, leastError); if (error < leastError) { @@ -709,10 +700,7 @@ void MotionEstimatorTLD::motionEstimationLuma(MotionEstimatorTLD& m_metld, MV *m } } - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, src, stride, buf, blockX, blockY, 0, 0, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, src, stride, buf, blockX, blockY, 0, 0, blockSize, leastError); + error = motionErrorLumaSSD(src, stride, buf, blockX, blockY, 0, 0, blockSize, leastError); if (error < leastError) { @@ -727,10 +715,8 @@ void MotionEstimatorTLD::motionEstimationLuma(MotionEstimatorTLD& m_metld, MV *m { for (int x2 = prevBest.x / m_motionVectorFactor - range; x2 <= prevBest.x / m_motionVectorFactor + range; x2++) { - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, src, stride, buf, blockX, blockY, x2 * m_motionVectorFactor, y2 * m_motionVectorFactor, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, src, stride, buf, blockX, blockY, x2 * m_motionVectorFactor, y2 * m_motionVectorFactor, blockSize, leastError); + error = motionErrorLumaSSD(src, stride, buf, blockX, blockY, x2 * m_motionVectorFactor, y2 * m_motionVectorFactor, blockSize, leastError); + if (error < leastError) { best.set(x2 * m_motionVectorFactor, y2 * m_motionVectorFactor); @@ -739,32 +725,14 @@ void MotionEstimatorTLD::motionEstimationLuma(MotionEstimatorTLD& m_metld, MV *m } } - if (blockY > 0) - { - int idx = ((blockY - stepSize) / stepSize) * mvStride + (blockX / stepSize); - MV aboveMV = mvs[idx]; - - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, src, stride, buf, blockX, blockY, aboveMV.x, aboveMV.y, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, src, stride, buf, blockX, blockY, aboveMV.x, aboveMV.y, blockSize, leastError); - - if (error < leastError) - { - best.set(aboveMV.x, aboveMV.y); - leastError = error; - } - } + /* Removed above block's Motion estimation dependency as the atomicity cost outweighs quality benefit */ if (blockX > 0) { int idx = ((blockY / stepSize) * mvStride + (blockX - stepSize) / stepSize); MV leftMV = mvs[idx]; - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, src, stride, buf, blockX, blockY, leftMV.x, leftMV.y, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, src, stride, buf, blockX, blockY, leftMV.x, leftMV.y, blockSize, leastError); + error = motionErrorLumaSSD(src, stride, buf, blockX, blockY, leftMV.x, leftMV.y, blockSize, leastError); if (error < leastError) { @@ -773,30 +741,6 @@ void MotionEstimatorTLD::motionEstimationLuma(MotionEstimatorTLD& m_metld, MV *m } } - // calculate average - double avg = 0.0; - for (int x1 = 0; x1 < blockSize; x1++) - { - for (int y1 = 0; y1 < blockSize; y1++) - { - avg = avg + *(src + (blockX + x1 + stride * (blockY + y1))); - } - } - avg = avg / (blockSize * blockSize); - - // calculate variance - double variance = 0; - for (int x1 = 0; x1 < blockSize; x1++) - { - for (int y1 = 0; y1 < blockSize; y1++) - { - int pix = *(src + (blockX + x1 + stride * (blockY + y1))); - variance = variance + (pix - avg) * (pix - avg); - } - } - - leastError = (int)(20 * ((leastError + 5.0) / (variance + 5.0)) + (leastError / (blockSize * blockSize)) / 50); - int mvIdx = (blockY / stepSize) * mvStride + (blockX / stepSize); mvs[mvIdx] = best; } @@ -804,27 +748,33 @@ void MotionEstimatorTLD::motionEstimationLuma(MotionEstimatorTLD& m_metld, MV *m } -void MotionEstimatorTLD::motionEstimationLumaDoubleRes(MotionEstimatorTLD& m_metld, MV *mvs, uint32_t mvStride, PicYuv *orig, PicYuv *buffer, int blockSize, - MV *previous, uint32_t prevMvStride, int factor, int* minError) +void MotionEstimatorTLD::motionEstimationLumaDoubleRes(MV *mvs, uint32_t mvStride, PicYuv *orig, PicYuv *buffer, + const MV *previous, uint32_t prevMvStride, int factor, int* minError, int row, const int rowSize) { int range = 0; + int stepSize, blockSize; + stepSize = blockSize = m_blockSize / 2; - const int stepSize = blockSize; - - const int origWidth = orig->m_picWidth; + const int origWidth = orig->m_picWidth; const int origHeight = orig->m_picHeight; + int rowStart = row * rowSize; + + if (rowStart > origHeight) + return; // row beyond frame edge — nothing to do + + int rowEnd = (!rowSize) ? origHeight : X265_MIN(rowStart + rowSize, origHeight); int error; - for (int blockY = 0; blockY + blockSize <= origHeight; blockY += stepSize) + for (int blockY = rowStart; blockY + blockSize <= rowEnd; blockY += stepSize) { for (int blockX = 0; blockX + blockSize <= origWidth; blockX += stepSize) { const intptr_t pelOffset = blockY * orig->m_stride + blockX; - m_metld.me.setSourcePU(orig->m_picOrg[0], orig->m_stride, pelOffset, blockSize, blockSize, X265_HEX_SEARCH, 1); + me.setSourcePU(orig->m_picOrg[0], orig->m_stride, pelOffset, blockSize, blockSize, X265_HEX_SEARCH, 1); MV best(0, 0); int leastError = INT_MAX; @@ -849,10 +799,7 @@ void MotionEstimatorTLD::motionEstimationLumaDoubleRes(MotionEstimatorTLD& m_met int mvIdx = testy * prevMvStride + testx; MV old = previous[mvIdx]; - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, old.x * factor, old.y * factor, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, old.x * factor, old.y * factor, blockSize, leastError); + error = motionErrorLumaSSD(orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, old.x * factor, old.y * factor, blockSize, leastError); if (error < leastError) { @@ -863,10 +810,7 @@ void MotionEstimatorTLD::motionEstimationLumaDoubleRes(MotionEstimatorTLD& m_met } } - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, 0, 0, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, 0, 0, blockSize, leastError); + error = motionErrorLumaSSD(orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, 0, 0, blockSize, leastError); if (error < leastError) { @@ -881,10 +825,7 @@ void MotionEstimatorTLD::motionEstimationLumaDoubleRes(MotionEstimatorTLD& m_met { for (int x2 = prevBest.x / m_motionVectorFactor - range; x2 <= prevBest.x / m_motionVectorFactor + range; x2++) { - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, x2 * m_motionVectorFactor, y2 * m_motionVectorFactor, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, x2 * m_motionVectorFactor, y2 * m_motionVectorFactor, blockSize, leastError); + error = motionErrorLumaSSD(orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, x2 * m_motionVectorFactor, y2 * m_motionVectorFactor, blockSize, leastError); if (error < leastError) { @@ -900,10 +841,7 @@ void MotionEstimatorTLD::motionEstimationLumaDoubleRes(MotionEstimatorTLD& m_met { for (int x2 = prevBest.x - doubleRange; x2 <= prevBest.x + doubleRange; x2++) { - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, x2, y2, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, x2, y2, blockSize, leastError); + error = motionErrorLumaSSD(orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, x2, y2, blockSize, leastError); if (error < leastError) { @@ -913,16 +851,13 @@ void MotionEstimatorTLD::motionEstimationLumaDoubleRes(MotionEstimatorTLD& m_met } } - - if (blockY > 0) + /* Using Above block's Motion vector only when above block is available within the same thread */ + if (blockY != rowStart) { int idx = ((blockY - stepSize) / stepSize) * mvStride + (blockX / stepSize); MV aboveMV = mvs[idx]; - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, aboveMV.x, aboveMV.y, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, aboveMV.x, aboveMV.y, blockSize, leastError); + error = motionErrorLumaSSD(orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, aboveMV.x, aboveMV.y, blockSize, leastError); if (error < leastError) { @@ -936,10 +871,7 @@ void MotionEstimatorTLD::motionEstimationLumaDoubleRes(MotionEstimatorTLD& m_met int idx = ((blockY / stepSize) * mvStride + (blockX - stepSize) / stepSize); MV leftMV = mvs[idx]; - if (m_useSADinME) - error = motionErrorLumaSAD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, leftMV.x, leftMV.y, blockSize, leastError); - else - error = motionErrorLumaSSD(m_metld, orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, leftMV.x, leftMV.y, blockSize, leastError); + error = motionErrorLumaSSD(orig->m_picOrg[0], (int)orig->m_stride, buffer->m_picOrg[0], blockX, blockY, leftMV.x, leftMV.y, blockSize, leastError); if (error < leastError) { @@ -948,27 +880,12 @@ void MotionEstimatorTLD::motionEstimationLumaDoubleRes(MotionEstimatorTLD& m_met } } - // calculate average - double avg = 0.0; - for (int x1 = 0; x1 < blockSize; x1++) - { - for (int y1 = 0; y1 < blockSize; y1++) - { - avg = avg + *(orig->m_picOrg[0] + (blockX + x1 + orig->m_stride * (blockY + y1))); - } - } - avg = avg / (blockSize * blockSize); - - // calculate variance - double variance = 0; - for (int x1 = 0; x1 < blockSize; x1++) - { - for (int y1 = 0; y1 < blockSize; y1++) - { - int pix = *(orig->m_picOrg[0] + (blockX + x1 + orig->m_stride * (blockY + y1))); - variance = variance + (pix - avg) * (pix - avg); - } - } + const pixel* blkOrigin = orig->m_picOrg[0] + blockY * orig->m_stride + blockX; + uint64_t sumSqr = primitives.cu[partitionFromSizes(blockSize, blockSize)].var(blkOrigin, orig->m_stride); + uint32_t sum = (uint32_t)sumSqr; + uint32_t sqr = (uint32_t)(sumSqr >> 32); + const int N = blockSize * blockSize; + double variance = (double)sqr - ((double)sum * sum) / N; leastError = (int)(20 * ((leastError + 5.0) / (variance + 5.0)) + (leastError / (blockSize * blockSize)) / 50); diff --git a/source/common/temporalfilter.h b/source/common/temporalfilter.h index c8c226b7c..7d080b53e 100644 --- a/source/common/temporalfilter.h +++ b/source/common/temporalfilter.h @@ -30,6 +30,8 @@ #include "piclist.h" #include "yuv.h" #include "motion.h" +#include "threadpool.h" +#include const int s_interpolationFilter[16][8] = { @@ -58,8 +60,39 @@ const double s_refStrengths[3][4] = {1.13, 0.97, 0.81, 0.57}, // m_range {0.30, 0.30, 0.30, 0.30} // otherwise }; +#define MCSTF_MAX_REFS 16 namespace X265_NS { + + /* MCSTF runtime SIMD dispatch + * Function-pointer table for MCSTF SIMD kernels. Defaults are scalar + * implementations defined in temporalfilter.cpp; setupIntrinsicMCSTF_avx2 + * (in common/vec/temporalfilter-avx2.cpp) overrides them with AVX2 variants, + * called from setupIntrinsicPrimitives() when the runtime CPU supports it.*/ + + struct MCSTFPrimitives + { + int (*motionErrorLumaFrac)(const pixel* origOrigin, intptr_t origStride, const pixel* buffOrigin, intptr_t buffStride, int x, int y, + int dx, int dy, int bs, int besterror, int bitDepth); + + void (*applyMotion)(const pixel* pSrcImage, int srcStride, pixel* pDstImage, int dstStride, int width, int height, int blockSizeX, + int blockSizeY, uint32_t mvsStride, const MV* mvs, int csx, int csy, int blockRow, int rowSize, int vShift); + + void (*computeBlockStats)(const pixel* srcPel, intptr_t srcStride, const pixel* refPel, intptr_t refStride, int blkSize, int* outVariance, int* outDiffsum); + + void (*bilateralFilter)(const pixel* srcBlk, intptr_t srcStride, int numRefs, const pixel* const* refBlks, const intptr_t* refStrides, + const double* vww, const double* vsw, double bdw, double maxSample, int blkSize, pixel* dstBlk, intptr_t dstStride); + }; + + extern MCSTFPrimitives mcstfPrim; + + void setupMCSTFPrimitives_scalar(MCSTFPrimitives& p); +#if X265_ARCH_X86 + /* defined in common/vec/temporalfilter-avx2.cpp; exposed here so the test + * bench can exercise it directly without going through the global mcstfPrim */ + void setupIntrinsicMCSTF_avx2(MCSTFPrimitives& p); +#endif + class OrigPicBuffer { public: @@ -85,34 +118,26 @@ namespace X265_NS { me.init(X265_CSP_I400); me.setQP(X265_LOOKAHEAD_QP); predPUYuv.create(FENC_STRIDE, X265_CSP_I400); - m_useSADinME = 1; m_motionVectorFactor = 16; + m_searchRange = 3; + m_blockSize = 16; } Yuv predPUYuv; - int m_useSADinME; int m_motionVectorFactor; int32_t m_bitDepth; + int m_searchRange; + int m_blockSize; void init(const x265_param* param); - void motionEstimationLuma(MotionEstimatorTLD& m_tld, MV* mvs, uint32_t mvStride, pixel* src, int stride, int height, int width, pixel* buf, int bs, int sRange, - MV* previous = 0, uint32_t prevmvStride = 0, int factor = 1); - - void motionEstimationLumaDoubleRes(MotionEstimatorTLD& m_tld, MV* mvs, uint32_t mvStride, PicYuv* orig, PicYuv* buffer, int blockSize, - MV* previous, uint32_t prevMvStride, int factor, int* minError); + void motionEstimationLuma(MV* mvs, uint32_t mvStride, pixel* src, int stride, int height, int width, pixel* buf, int row, const int rowSize, + const MV* previous = 0, uint32_t prevmvStride = 0, int factor = 1); - int motionErrorLumaSSD(MotionEstimatorTLD& m_tld, pixel* src, - int stride, - pixel* buf, - int x, - int y, - int dx, - int dy, - int bs, - int besterror = 8 * 8 * 1024 * 1024); + void motionEstimationLumaDoubleRes(MV* mvs, uint32_t mvStride, PicYuv* orig, PicYuv* buffer, + const MV* previous, uint32_t prevMvStride, int factor, int* minError, int row, const int rowSize); - int motionErrorLumaSAD(MotionEstimatorTLD& m_tld, pixel* src, + int motionErrorLumaSSD(pixel* src, int stride, pixel* buf, int x, @@ -185,15 +210,84 @@ namespace X265_NS { uint8_t m_sliceTypeConfig; MotionEstimatorTLD* m_metld; + double m_overallStrength; int createRefPicInfo(TemporalFilterRefPicInfo* refFrame, x265_param* param); - void bilateralFilter(Frame* frame, TemporalFilterRefPicInfo* mctfRefList, double overallStrength); + void bilateralFilter(Frame* frame, + TemporalFilterRefPicInfo* mcstfRefList, + ThreadPool* pool); + + void bilateralFilterCore(Frame* frame, + TemporalFilterRefPicInfo* mcstfRefList, + int numRef, + int blockRow, + int blockSize); void destroyRefPicInfo(TemporalFilterRefPicInfo* curFrame); - void applyMotion(MV *mvs, uint32_t mvsStride, PicYuv *input, PicYuv *output); + void applyMotion(MV *mvs, uint32_t mvsStride, PicYuv *input, PicYuv *output, const int blockRow = 0, const int rowSize = 0); + + }; + + class BilateralFilterGroup : public BondedTaskGroup + { + public: + TemporalFilter& m_filter; + ThreadPool* m_pool; + + struct FilterJob + { + Frame* frame; + TemporalFilterRefPicInfo* mcstfRefList; + int numRef; + int blockRow; + int rowSize; + }; + + static const int MAX_FILTER_JOBS = 512; + FilterJob m_jobs[MAX_FILTER_JOBS]; + + BilateralFilterGroup(TemporalFilter& f, ThreadPool* pool) + : m_filter(f), m_pool(pool), m_jobs() {} + void add(Frame* frame, TemporalFilterRefPicInfo* mcstfRefList, + int numRef, int blockRow, int rowSize) + { + X265_CHECK(m_jobTotal < MAX_FILTER_JOBS, + "BilateralFilterGroup overflow\n"); + FilterJob& j = m_jobs[m_jobTotal++]; + j.frame = frame; + j.mcstfRefList = mcstfRefList; + j.numRef = numRef; + j.blockRow = blockRow; + j.rowSize = rowSize; + } + + void finishBatch() + { + if (m_pool) + tryBondPeers(*m_pool, m_jobTotal); + processTasks(-1); + waitForExit(); + m_jobTotal = m_jobAcquired = 0; + } + + void processTasks(int /*workerThreadID*/) + { + m_lock.acquire(); + while (m_jobAcquired < m_jobTotal) + { + const int i = m_jobAcquired++; + m_lock.release(); + + const FilterJob& j = m_jobs[i]; + m_filter.bilateralFilterCore(j.frame, j.mcstfRefList, j.numRef, + j.blockRow, j.rowSize); + m_lock.acquire(); + } + m_lock.release(); + } }; } #endif diff --git a/source/common/vec/temporalfilter-avx2.cpp b/source/common/vec/temporalfilter-avx2.cpp new file mode 100644 index 000000000..f5857ab7d --- /dev/null +++ b/source/common/vec/temporalfilter-avx2.cpp @@ -0,0 +1,659 @@ +/***************************************************************************** +* Copyright (C) 2013-2021 MulticoreWare, Inc +* + * Authors: gunasrij + * +* This program is free software; you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation; either version 2 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program; if not, write to the Free Software +* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. +* +* This program is also available under a commercial proprietary license. +* For more information, contact us at license @ x265.com. +*****************************************************************************/ + +/* MCSTF SIMD kernels - Provides AVX2 implementations of MCSTF primitives + * declared in temporalfilter.h (MCSTFPrimitives). + * + * Entry point: setupMCSTFPrimitives_x86(MCSTFPrimitives&, int cpuMask) + * Called from primitives.cpp after setupMCSTFPrimitives_scalar(). + * Overrides only the function pointers whose ISA requirement is met. + */ + + +#include /* SSE2 */ +#include /* SSSE3 */ +#include /* SSE4.1 */ +#include /* AVX2 */ + +#include "common.h" +#include "primitives.h" +#include "temporalfilter.h" +#include "mv.h" +#include "cpu.h" + +using namespace X265_NS; + +namespace X265_NS { + +#if defined(_MSC_VER) && !defined(__clang__) && !defined(__INTEL_COMPILER) +# define MCSTF_TARGET_AVX2 +#else +# define MCSTF_TARGET_AVX2 __attribute__((target("avx2"))) +#endif + + + /* Shared helpers */ + + MCSTF_TARGET_AVX2 + static inline __m256d loadPix4_pd(const pixel* p) + { +#if X265_DEPTH > 8 + __m128i v = _mm_loadl_epi64(reinterpret_cast(p)); + return _mm256_cvtepi32_pd(_mm_cvtepu16_epi32(v)); +#else + __m128i v = _mm_cvtsi32_si128(*reinterpret_cast(p)); + return _mm256_cvtepi32_pd(_mm_cvtepu8_epi32(v)); +#endif + } + + MCSTF_TARGET_AVX2 + static inline void storePix4_pd(pixel* p, __m256d v) + { + __m128i i32 = _mm256_cvtpd_epi32(v); +#if X265_DEPTH > 8 + _mm_storel_epi64(reinterpret_cast<__m128i*>(p), + _mm_packus_epi32(i32, i32)); +#else + __m128i u16 = _mm_packus_epi32(i32, i32); + *reinterpret_cast(p) = _mm_cvtsi128_si32(_mm_packus_epi16(u16, u16)); +#endif + } + + static inline int hsum_epi32(__m128i v) + { + __m128i hi = _mm_srli_si128(v, 8); + v = _mm_add_epi32(v, hi); + hi = _mm_srli_si128(v, 4); + v = _mm_add_epi32(v, hi); + return _mm_cvtsi128_si32(v); + } + + MCSTF_TARGET_AVX2 + static inline int hsum_epi32_avx(__m256i v) + { + __m128i lo = _mm256_castsi256_si128(v); + __m128i hi = _mm256_extracti128_si256(v, 1); + __m128i sum = _mm_add_epi32(lo, hi); + + __m128i hi2 = _mm_srli_si128(sum, 8); + sum = _mm_add_epi32(sum, hi2); + __m128i hi3 = _mm_srli_si128(sum, 4); + sum = _mm_add_epi32(sum, hi3); + + return _mm_cvtsi128_si32(sum); + } + + MCSTF_TARGET_AVX2 + static inline __m256i load8px_epi32(const pixel* p) + { +#if X265_DEPTH > 8 + return _mm256_cvtepu16_epi32(_mm_loadu_si128(reinterpret_cast(p))); +#else + return _mm256_cvtepu8_epi32(_mm_loadl_epi64(reinterpret_cast(p))); +#endif + } + + MCSTF_TARGET_AVX2 + static inline __m256i load4px_epi32(const pixel* p) + { +#if X265_DEPTH > 8 + return _mm256_cvtepu16_epi32(_mm_loadl_epi64(reinterpret_cast(p))); +#else + return _mm256_cvtepu8_epi32(_mm_cvtsi32_si128(*reinterpret_cast(p))); +#endif + } + + MCSTF_TARGET_AVX2 + static inline void store8px(pixel* dst, __m256i v) + { +#if X265_DEPTH > 8 + /* _mm256_packus_epi32(v,v) → [lane0-3, lane0-3 | lane4-7, lane4-7]. + * permute4x64(0x08) moves qwords 0 and 2 into the lower 128 bits. */ + __m256i packed = _mm256_packus_epi32(v, v); + __m128i out = _mm256_castsi256_si128( + _mm256_permute4x64_epi64(packed, 0x08)); + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), out); +#else + __m256i as16 = _mm256_packs_epi32(v, v); + __m128i lo = _mm256_castsi256_si128(as16); + __m128i hi = _mm256_extracti128_si256(as16, 1); + __m128i merged = _mm_unpacklo_epi64(lo, hi); + __m128i out8 = _mm_packus_epi16(merged, merged); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dst), out8); +#endif + } + + MCSTF_TARGET_AVX2 + static inline void store4px(pixel* dst, __m256i v) + { + __m128i lo = _mm256_castsi256_si128(v); /* only lower 4 lanes valid */ +#if X265_DEPTH > 8 + __m128i out = _mm_packus_epi32(lo, _mm_setzero_si128()); + _mm_storel_epi64(reinterpret_cast<__m128i*>(dst), out); +#else + __m128i as16 = _mm_packs_epi32(lo, _mm_setzero_si128()); + __m128i out8 = _mm_packus_epi16(as16, _mm_setzero_si128()); + *reinterpret_cast(dst) = _mm_cvtsi128_si32(out8); +#endif + } + + MCSTF_TARGET_AVX2 + static int motionErrorLumaFrac_avx2( + const pixel* origOrigin, intptr_t origStride, + const pixel* buffOrigin, intptr_t buffStride, + int x, int y, int dx, int dy, + int bs, int besterror, int bitDepth) + { + const int* xFilter = s_interpolationFilter[dx & 0xF]; + const int* yFilter = s_interpolationFilter[dy & 0xF]; + + X265_CHECK(bs <= 16, "Unsupported block size\n"); + int tempArray[64 + 8][64]; + const int int_dx = dx >> 4; + const int int_dy = dy >> 4; + int error = 0; + + // HEVC interpolation filters always have zero-valued end taps + // (xFilter[0] and xFilter[7]), so only taps 1..6 are processed. + const __m128i xf12_128 = _mm_unpacklo_epi16( + _mm_set1_epi16((int16_t)xFilter[1]), + _mm_set1_epi16((int16_t)xFilter[2])); + const __m128i xf34_128 = _mm_unpacklo_epi16( + _mm_set1_epi16((int16_t)xFilter[3]), + _mm_set1_epi16((int16_t)xFilter[4])); + const __m128i xf56_128 = _mm_unpacklo_epi16( + _mm_set1_epi16((int16_t)xFilter[5]), + _mm_set1_epi16((int16_t)xFilter[6])); + + const __m256i xf12 = _mm256_set_m128i(xf12_128, xf12_128); + const __m256i xf34 = _mm256_set_m128i(xf34_128, xf34_128); + const __m256i xf56 = _mm256_set_m128i(xf56_128, xf56_128); + + for (int y1 = 1; y1 < bs + 7; y1++) + { + const pixel* rowStart0 = buffOrigin + + (y + y1 + int_dy - 3) * buffStride + + (x + int_dx - 3); + + for (int x1 = 0; x1 < bs; x1 += 8) + { + const pixel* rowStart = rowStart0 + x1; + + +#if X265_DEPTH > 8 + /* uint16 pixels: load 8× uint16 directly into 128-bit reg. + * _mm256_set_m128i places lo_half in [127:0] and hi_half in + * [255:128]. We build the 256-bit interleaved pairs: + * pairs_ab[255:0] = { unpackhi(s_a, s_b) | unpacklo(s_a, s_b) } + * so madd lane i gives s_a[i]*f_a + s_b[i]*f_b. */ + __m128i s1 = _mm_loadu_si128(reinterpret_cast(&rowStart[1])); + __m128i s2 = _mm_loadu_si128(reinterpret_cast(&rowStart[2])); + __m128i s3 = _mm_loadu_si128(reinterpret_cast(&rowStart[3])); + __m128i s4 = _mm_loadu_si128(reinterpret_cast(&rowStart[4])); + __m128i s5 = _mm_loadu_si128(reinterpret_cast(&rowStart[5])); + __m128i s6 = _mm_loadu_si128(reinterpret_cast(&rowStart[6])); +#else + __m128i s1 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast(&rowStart[1]))); + __m128i s2 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast(&rowStart[2]))); + __m128i s3 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast(&rowStart[3]))); + __m128i s4 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast(&rowStart[4]))); + __m128i s5 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast(&rowStart[5]))); + __m128i s6 = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast(&rowStart[6]))); +#endif + + __m256i pairs12 = _mm256_set_m128i( + _mm_unpackhi_epi16(s1, s2), + _mm_unpacklo_epi16(s1, s2)); + __m256i pairs34 = _mm256_set_m128i( + _mm_unpackhi_epi16(s3, s4), + _mm_unpacklo_epi16(s3, s4)); + __m256i pairs56 = _mm256_set_m128i( + _mm_unpackhi_epi16(s5, s6), + _mm_unpacklo_epi16(s5, s6)); + + __m256i h_out = _mm256_add_epi32( + _mm256_add_epi32( + _mm256_madd_epi16(pairs12, xf12), + _mm256_madd_epi16(pairs34, xf34)), + _mm256_madd_epi16(pairs56, xf56)); + + _mm256_storeu_si256(reinterpret_cast<__m256i*>(&tempArray[y1][x1]), h_out); + } + } + + // End taps yFilter[0] and yFilter[7] are always zero. + const __m256i yt1 = _mm256_set1_epi32(yFilter[1]); + const __m256i yt2 = _mm256_set1_epi32(yFilter[2]); + const __m256i yt3 = _mm256_set1_epi32(yFilter[3]); + const __m256i yt4 = _mm256_set1_epi32(yFilter[4]); + const __m256i yt5 = _mm256_set1_epi32(yFilter[5]); + const __m256i yt6 = _mm256_set1_epi32(yFilter[6]); + const __m256i vmax = _mm256_set1_epi32((1 << bitDepth) - 1); + const __m256i vmin = _mm256_setzero_si256(); + const __m256i round_v = _mm256_set1_epi32(1 << 11); + + __m256i xerror = _mm256_setzero_si256(); + + for (int outY = 0; outY < bs; outY++) + { + const pixel* origRowBase = origOrigin + (y + outY) * origStride + x; + + for (int x1 = 0; x1 < bs; x1 += 8) + { + __m256i t1 = _mm256_loadu_si256(reinterpret_cast(&tempArray[outY + 1][x1])); + __m256i t2 = _mm256_loadu_si256(reinterpret_cast(&tempArray[outY + 2][x1])); + __m256i t3 = _mm256_loadu_si256(reinterpret_cast(&tempArray[outY + 3][x1])); + __m256i t4 = _mm256_loadu_si256(reinterpret_cast(&tempArray[outY + 4][x1])); + __m256i t5 = _mm256_loadu_si256(reinterpret_cast(&tempArray[outY + 5][x1])); + __m256i t6 = _mm256_loadu_si256(reinterpret_cast(&tempArray[outY + 6][x1])); + + __m256i v = _mm256_add_epi32( + _mm256_add_epi32( + _mm256_add_epi32(_mm256_mullo_epi32(t1, yt1), + _mm256_mullo_epi32(t2, yt2)), + _mm256_add_epi32(_mm256_mullo_epi32(t3, yt3), + _mm256_mullo_epi32(t4, yt4))), + _mm256_add_epi32(_mm256_mullo_epi32(t5, yt5), + _mm256_mullo_epi32(t6, yt6))); + + v = _mm256_srai_epi32(_mm256_add_epi32(v, round_v), 12); + v = _mm256_min_epi32(_mm256_max_epi32(v, vmin), vmax); + +#if X265_DEPTH > 8 + __m256i orig = _mm256_cvtepu16_epi32( + _mm_loadu_si128(reinterpret_cast(&origRowBase[x1]))); +#else + __m128i xorig = _mm_loadl_epi64(reinterpret_cast(&origRowBase[x1])); + __m256i orig = _mm256_set_m128i( + _mm_cvtepu8_epi32(_mm_srli_si128(xorig, 4)), + _mm_cvtepu8_epi32(xorig)); +#endif + + __m256i diff = _mm256_sub_epi32(v, orig); + + __m256i row_err = _mm256_mullo_epi32(diff, diff); /* SSD */ + + xerror = _mm256_add_epi32(xerror, row_err); + } + + error = hsum_epi32_avx(xerror); + if (error > besterror) + return error; + } + + return error; + } + + /* Per-block separable 6-tap filter worker */ + MCSTF_TARGET_AVX2 + static void applyMotionBlock_avx2( + const pixel* pSrcImage, int srcStride, + pixel* pDstImage, int dstStride, + int x, int y, + int blockSizeX, int blockSizeY, + int xInt, int yInt, + const int* xFilter, + const int* yFilter) + { + X265_CHECK(blockSizeX == 4 || blockSizeX == 8, "Unsupported block width\n"); + + static const int numFilterTaps = 7; + static const int maxBlockSize = 8; + int tempArray[maxBlockSize + numFilterTaps][8]; + + const __m256i xf1 = _mm256_set1_epi32(xFilter[1]); + const __m256i xf2 = _mm256_set1_epi32(xFilter[2]); + const __m256i xf3 = _mm256_set1_epi32(xFilter[3]); + const __m256i xf4 = _mm256_set1_epi32(xFilter[4]); + const __m256i xf5 = _mm256_set1_epi32(xFilter[5]); + const __m256i xf6 = _mm256_set1_epi32(xFilter[6]); + + const __m256i yf1 = _mm256_set1_epi32(yFilter[1]); + const __m256i yf2 = _mm256_set1_epi32(yFilter[2]); + const __m256i yf3 = _mm256_set1_epi32(yFilter[3]); + const __m256i yf4 = _mm256_set1_epi32(yFilter[4]); + const __m256i yf5 = _mm256_set1_epi32(yFilter[5]); + const __m256i yf6 = _mm256_set1_epi32(yFilter[6]); + + const __m256i vround = _mm256_set1_epi32(1 << 11); + const __m256i vmin = _mm256_setzero_si256(); + const __m256i vmax = _mm256_set1_epi32((1 << X265_DEPTH) - 1); + + const int hColBase = x + xInt - 3; + const int hRowEnd = blockSizeY + 5; + + /* H-PASS */ + if (blockSizeX == 8) + { + for (int by = 1; by <= hRowEnd; by++) + { + const pixel* srcRow = pSrcImage + (y + by + yInt - 3) * srcStride; + + __m256i s1 = load8px_epi32(srcRow + hColBase + 1); + __m256i s2 = load8px_epi32(srcRow + hColBase + 2); + __m256i s3 = load8px_epi32(srcRow + hColBase + 3); + __m256i s4 = load8px_epi32(srcRow + hColBase + 4); + __m256i s5 = load8px_epi32(srcRow + hColBase + 5); + __m256i s6 = load8px_epi32(srcRow + hColBase + 6); + + __m256i acc = _mm256_mullo_epi32(s1, xf1); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s2, xf2)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s3, xf3)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s4, xf4)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s5, xf5)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s6, xf6)); + + _mm256_storeu_si256(reinterpret_cast<__m256i*>(&tempArray[by][0]), acc); + } + } + else + { + for (int by = 1; by <= hRowEnd; by++) + { + const pixel* srcRow = pSrcImage + (y + by + yInt - 3) * srcStride; + + __m256i s1 = load4px_epi32(srcRow + hColBase + 1); + __m256i s2 = load4px_epi32(srcRow + hColBase + 2); + __m256i s3 = load4px_epi32(srcRow + hColBase + 3); + __m256i s4 = load4px_epi32(srcRow + hColBase + 4); + __m256i s5 = load4px_epi32(srcRow + hColBase + 5); + __m256i s6 = load4px_epi32(srcRow + hColBase + 6); + + __m256i acc = _mm256_mullo_epi32(s1, xf1); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s2, xf2)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s3, xf3)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s4, xf4)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s5, xf5)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(s6, xf6)); + + _mm_storeu_si128(reinterpret_cast<__m128i*>(&tempArray[by][0]), + _mm256_castsi256_si128(acc)); + } + } + + /* V-PASS */ + if (blockSizeX == 8) + { + for (int by = 0; by < blockSizeY; by++) + { + pixel* pDstPel = pDstImage + (y + by) * dstStride + x; + + __m256i t1 = _mm256_loadu_si256(reinterpret_cast(&tempArray[by + 1][0])); + __m256i t2 = _mm256_loadu_si256(reinterpret_cast(&tempArray[by + 2][0])); + __m256i t3 = _mm256_loadu_si256(reinterpret_cast(&tempArray[by + 3][0])); + __m256i t4 = _mm256_loadu_si256(reinterpret_cast(&tempArray[by + 4][0])); + __m256i t5 = _mm256_loadu_si256(reinterpret_cast(&tempArray[by + 5][0])); + __m256i t6 = _mm256_loadu_si256(reinterpret_cast(&tempArray[by + 6][0])); + + __m256i acc = _mm256_mullo_epi32(t1, yf1); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t2, yf2)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t3, yf3)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t4, yf4)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t5, yf5)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t6, yf6)); + + acc = _mm256_add_epi32(acc, vround); + acc = _mm256_srai_epi32(acc, 12); + + acc = _mm256_max_epi32(acc, vmin); + acc = _mm256_min_epi32(acc, vmax); + + store8px(pDstPel, acc); + } + } + else + { + for (int by = 0; by < blockSizeY; by++) + { + pixel* pDstPel = pDstImage + (y + by) * dstStride + x; + + __m256i t1 = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast(&tempArray[by + 1][0]))); + __m256i t2 = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast(&tempArray[by + 2][0]))); + __m256i t3 = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast(&tempArray[by + 3][0]))); + __m256i t4 = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast(&tempArray[by + 4][0]))); + __m256i t5 = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast(&tempArray[by + 5][0]))); + __m256i t6 = _mm256_castsi128_si256(_mm_loadu_si128(reinterpret_cast(&tempArray[by + 6][0]))); + + __m256i acc = _mm256_mullo_epi32(t1, yf1); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t2, yf2)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t3, yf3)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t4, yf4)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t5, yf5)); + acc = _mm256_add_epi32(acc, _mm256_mullo_epi32(t6, yf6)); + + acc = _mm256_add_epi32(acc, vround); + acc = _mm256_srai_epi32(acc, 12); + acc = _mm256_max_epi32(acc, vmin); + acc = _mm256_min_epi32(acc, vmax); + + store4px(pDstPel, acc); + } + } + } + + MCSTF_TARGET_AVX2 + static void applyMotion_avx2( + const pixel* pSrcImage, int srcStride, + pixel* pDstImage, int dstStride, + int width, int height, + int blockSizeX, int blockSizeY, + uint32_t mvsStride, const MV* mvs, + int csx, int csy, + int blockRow, int rowSize, int vShift) + { + const int blkRowStart = (blockRow * rowSize) >> vShift; + const int blkRowEnd = X265_MIN((blockRow * rowSize + rowSize) >> vShift, height); + const int rowStart = (!rowSize) ? 0 : blkRowStart; + const int rowEnd = (!rowSize) ? height : blkRowEnd; + int blockNumY = (!rowSize) ? 0 : blkRowStart / blockSizeY; + + for (int y = rowStart; + y + blockSizeY <= rowEnd; + y += blockSizeY, blockNumY++) + { + for (int x = 0, blockNumX = 0; + x + blockSizeX <= width; + x += blockSizeX, blockNumX++) + { + const int mvIdx = blockNumY * (int)mvsStride + blockNumX; + const MV& mv = mvs[mvIdx]; + + const int dx = mv.x >> csx; + const int dy = mv.y >> csy; + const int xInt = mv.x >> (4 + csx); + const int yInt = mv.y >> (4 + csy); + + const int* xFilter = s_interpolationFilter[dx & 0xf]; + const int* yFilter = s_interpolationFilter[dy & 0xf]; + + applyMotionBlock_avx2(pSrcImage, srcStride, pDstImage, dstStride, x, y, blockSizeX, blockSizeY, xInt, yInt, xFilter, yFilter); + } + } + } + + MCSTF_TARGET_AVX2 + static void computeBlockStats_avx2(const pixel* srcPel, intptr_t srcStride, const pixel* refPel, intptr_t refStride, int blkSize, int* outVariance, int* outDiffsum) + { + if (blkSize == 8) + { + __m128i drow[8]; + for (int y = 0; y < 8; y++) + { +#if X265_DEPTH > 8 + __m128i s = _mm_loadu_si128(reinterpret_cast(srcPel + y * srcStride)); + __m128i r = _mm_loadu_si128(reinterpret_cast(refPel + y * refStride)); +#else + __m128i s = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast(srcPel + y * srcStride))); + __m128i r = _mm_cvtepu8_epi16(_mm_loadl_epi64(reinterpret_cast(refPel + y * refStride))); +#endif + drow[y] = _mm_sub_epi16(s, r); + } + + static const int16_t maskdata[8] = { -1,-1,-1,-1,-1,-1,-1, 0 }; + const __m128i mask7 = _mm_loadu_si128(reinterpret_cast(maskdata)); + + __m256i vVar = _mm256_setzero_si256(); + __m256i vDiff = _mm256_setzero_si256(); + + for (int y = 0; y < 8; y++) + { + __m256i d = _mm256_cvtepi16_epi32(drow[y]); + vVar = _mm256_add_epi32(vVar, _mm256_mullo_epi32(d, d)); + + __m128i hdiff = _mm_and_si128( + _mm_sub_epi16(_mm_srli_si128(drow[y], 2), drow[y]), + mask7); + __m256i hd = _mm256_cvtepi16_epi32(hdiff); + vDiff = _mm256_add_epi32(vDiff, _mm256_mullo_epi32(hd, hd)); + + if (y < 7) + { + __m256i vd = _mm256_cvtepi16_epi32(_mm_sub_epi16(drow[y + 1], drow[y])); + vDiff = _mm256_add_epi32(vDiff, _mm256_mullo_epi32(vd, vd)); + } + } + + *outVariance = hsum_epi32_avx(vVar); + *outDiffsum = hsum_epi32_avx(vDiff); + } + else + { + __m128i drow[4]; + for (int y = 0; y < 4; y++) + { +#if X265_DEPTH > 8 + __m128i s = _mm_loadl_epi64(reinterpret_cast(srcPel + y * srcStride)); + __m128i r = _mm_loadl_epi64(reinterpret_cast(refPel + y * refStride)); +#else + __m128i s = _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast(srcPel + y * srcStride))); + __m128i r = _mm_cvtepu8_epi16(_mm_cvtsi32_si128(*reinterpret_cast(refPel + y * refStride))); +#endif + drow[y] = _mm_sub_epi16(s, r); + } + + static const int16_t maskdata3[8] = { -1,-1,-1, 0, 0,0,0,0 }; + const __m128i mask3 = _mm_loadu_si128(reinterpret_cast(maskdata3)); + + __m128i vVar128 = _mm_setzero_si128(); + __m128i vDiff128 = _mm_setzero_si128(); + + for (int y = 0; y < 4; y++) + { + __m128i d = _mm_cvtepi16_epi32(drow[y]); + vVar128 = _mm_add_epi32(vVar128, _mm_mullo_epi32(d, d)); + + __m128i hdiff = _mm_and_si128( + _mm_sub_epi16(_mm_srli_si128(drow[y], 2), drow[y]), + mask3); + __m128i hd = _mm_cvtepi16_epi32(hdiff); + vDiff128 = _mm_add_epi32(vDiff128, _mm_mullo_epi32(hd, hd)); + + if (y < 3) + { + __m128i vd = _mm_cvtepi16_epi32(_mm_sub_epi16(drow[y + 1], drow[y])); + vDiff128 = _mm_add_epi32(vDiff128, _mm_mullo_epi32(vd, vd)); + } + } + + *outVariance = hsum_epi32(vVar128); + *outDiffsum = hsum_epi32(vDiff128); + } + } + + MCSTF_TARGET_AVX2 + static void bilateralFilter_avx2( + const pixel* srcBlk, intptr_t srcStride, + int numRefs, + const pixel* const* refBlks, + const intptr_t* refStrides, + const double* vww, + const double* vsw, + double bdw, + double maxSample, + int blkSize, + pixel* dstBlk, intptr_t dstStride) + { + const __m256d vBdw = _mm256_set1_pd(bdw); + const __m256d vOne = _mm256_set1_pd(1.0); + const __m256d vZero = _mm256_setzero_pd(); + const __m256d vMax = _mm256_set1_pd(maxSample); + const __m256d vHalf = _mm256_set1_pd(0.5); + + double neg_inv_vsw[MCSTF_MAX_REFS]; + for (int i = 0; i < numRefs; i++) + neg_inv_vsw[i] = -1.0 / vsw[i]; + + ALIGN_VAR_32(double, expArgs[4]); + ALIGN_VAR_32(double, expVals[4]); + + for (int y = 0; y < blkSize; y++) + { + for (int xg = 0; xg < blkSize; xg += 4) + { + __m256d vOrgVal = loadPix4_pd(srcBlk + y * srcStride + xg); + __m256d vNewVal = vOrgVal; + __m256d vWeightSum = vOne; + + for (int i = 0; i < numRefs; i++) + { + __m256d vRefVal = loadPix4_pd(refBlks[i] + y * refStrides[i] + xg); + + __m256d vDiff = _mm256_mul_pd(_mm256_sub_pd(vRefVal, vOrgVal), vBdw); + __m256d vDiffSq = _mm256_mul_pd(vDiff, vDiff); + + __m256d vExpArg = _mm256_mul_pd(vDiffSq, _mm256_set1_pd(neg_inv_vsw[i])); + + _mm256_store_pd(expArgs, vExpArg); + expVals[0] = exp(expArgs[0]); + expVals[1] = exp(expArgs[1]); + expVals[2] = exp(expArgs[2]); + expVals[3] = exp(expArgs[3]); + __m256d vExpResult = _mm256_load_pd(expVals); + + __m256d vWeight = _mm256_mul_pd(_mm256_set1_pd(vww[i]), vExpResult); + + vNewVal = _mm256_add_pd(vNewVal, _mm256_mul_pd(vWeight, vRefVal)); + vWeightSum = _mm256_add_pd(vWeightSum, vWeight); + } + + vNewVal = _mm256_div_pd(vNewVal, vWeightSum); + + vNewVal = _mm256_floor_pd(_mm256_add_pd(vNewVal, vHalf)); + + vNewVal = _mm256_max_pd(vZero, _mm256_min_pd(vMax, vNewVal)); + + storePix4_pd(dstBlk + y * dstStride + xg, vNewVal); + } + } + } +} // anonymous namespace + +namespace X265_NS { + void setupIntrinsicMCSTF_avx2(MCSTFPrimitives & p) + { + p.motionErrorLumaFrac = motionErrorLumaFrac_avx2; + p.applyMotion = applyMotion_avx2; + p.computeBlockStats = computeBlockStats_avx2; + p.bilateralFilter = bilateralFilter_avx2; + } +} // namespace X265_NS diff --git a/source/common/vec/vec-primitives.cpp b/source/common/vec/vec-primitives.cpp index d37f7ac3d..c1d951cca 100644 --- a/source/common/vec/vec-primitives.cpp +++ b/source/common/vec/vec-primitives.cpp @@ -23,6 +23,7 @@ #include "primitives.h" #include "x265.h" +#include "temporalfilter.h" /* The #if logic here must match the file lists in CMakeLists.txt */ #if X265_ARCH_X86 @@ -57,6 +58,7 @@ namespace X265_NS { void setupIntrinsicDCT_sse3(EncoderPrimitives&); void setupIntrinsicDCT_ssse3(EncoderPrimitives&); void setupIntrinsicDCT_sse41(EncoderPrimitives&); +void setupIntrinsicMCSTF_avx2(MCSTFPrimitives&); /* Use primitives for the best available vector architecture */ void setupIntrinsicPrimitives(EncoderPrimitives &p, int cpuMask) @@ -78,6 +80,10 @@ void setupIntrinsicPrimitives(EncoderPrimitives &p, int cpuMask) { setupIntrinsicDCT_sse41(p); } +#endif +#ifdef HAVE_AVX2 + if (cpuMask & X265_CPU_AVX2) + setupIntrinsicMCSTF_avx2(mcstfPrim); #endif (void)p; (void)cpuMask; diff --git a/source/common/x86/asm-primitives.cpp b/source/common/x86/asm-primitives.cpp index bb32f32cd..72a207862 100644 --- a/source/common/x86/asm-primitives.cpp +++ b/source/common/x86/asm-primitives.cpp @@ -1091,7 +1091,7 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main10 p.frameInitLowres = PFX(frame_init_lowres_core_sse2); p.frameInitLowerRes = PFX(frame_init_lowres_core_sse2); - p.frameSubSampleLuma = PFX(frame_subsample_luma_sse2); + // TODO: the planecopy_sp is really planecopy_SC now, must be fix it //p.planecopy_sp = PFX(downShift_16_sse2); p.planecopy_sp_shl = PFX(upShift_16_sse2); @@ -1122,7 +1122,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main10 { ASSIGN2(p.scale1D_128to64, scale1D_128to64_ssse3); p.scale2D_64to32 = PFX(scale2D_64to32_ssse3); - p.frameSubSampleLuma = PFX(frame_subsample_luma_ssse3); // p.pu[LUMA_4x4].satd = p.cu[BLOCK_4x4].sa8d = PFX(pixel_satd_4x4_ssse3); this one is broken ALL_LUMA_PU(satd, pixel_satd, ssse3); @@ -1464,7 +1463,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main10 p.pu[LUMA_64x48].copy_pp = (copy_pp_t)PFX(blockcopy_ss_64x48_avx); p.pu[LUMA_64x64].copy_pp = (copy_pp_t)PFX(blockcopy_ss_64x64_avx); p.propagateCost = PFX(mbtree_propagate_cost_avx); - p.frameSubSampleLuma = PFX(frame_subsample_luma_avx); } if (cpuMask & X265_CPU_XOP) { @@ -1476,7 +1474,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main10 LUMA_VAR(xop); p.frameInitLowres = PFX(frame_init_lowres_core_xop); p.frameInitLowerRes = PFX(frame_init_lowres_core_xop); - p.frameSubSampleLuma = PFX(frame_subsample_luma_xop); } if (cpuMask & X265_CPU_AVX2) { @@ -2306,8 +2303,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main10 p.frameInitLowres = PFX(frame_init_lowres_core_avx2); p.frameInitLowerRes = PFX(frame_init_lowres_core_avx2); - p.frameSubSampleLuma = PFX(frame_subsample_luma_avx2); - p.propagateCost = PFX(mbtree_propagate_cost_avx2); p.fix8Unpack = PFX(cutree_fix8_unpack_avx2); p.fix8Pack = PFX(cutree_fix8_pack_avx2); @@ -3307,7 +3302,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main //p.frameInitLowres = PFX(frame_init_lowres_core_mmx2); p.frameInitLowres = PFX(frame_init_lowres_core_sse2); p.frameInitLowerRes = PFX(frame_init_lowres_core_sse2); - p.frameSubSampleLuma = PFX(frame_subsample_luma_sse2); ALL_LUMA_TU(blockfill_s[NONALIGNED], blockfill_s, sse2); ALL_LUMA_TU(blockfill_s[ALIGNED], blockfill_s, sse2); @@ -3432,8 +3426,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main ASSIGN2(p.scale1D_128to64, scale1D_128to64_ssse3); p.scale2D_64to32 = PFX(scale2D_64to32_ssse3); - p.frameSubSampleLuma = PFX(frame_subsample_luma_ssse3); - ASSIGN2(p.pu[LUMA_8x4].convert_p2s, filterPixelToShort_8x4_ssse3); ASSIGN2(p.pu[LUMA_8x8].convert_p2s, filterPixelToShort_8x8_ssse3); ASSIGN2(p.pu[LUMA_8x16].convert_p2s, filterPixelToShort_8x16_ssse3); @@ -3701,7 +3693,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main p.frameInitLowres = PFX(frame_init_lowres_core_avx); p.frameInitLowerRes = PFX(frame_init_lowres_core_avx); p.propagateCost = PFX(mbtree_propagate_cost_avx); - p.frameSubSampleLuma = PFX(frame_subsample_luma_avx); } if (cpuMask & X265_CPU_XOP) { @@ -3713,7 +3704,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main p.cu[BLOCK_16x16].sse_pp = PFX(pixel_ssd_16x16_xop); p.frameInitLowres = PFX(frame_init_lowres_core_xop); p.frameInitLowerRes = PFX(frame_init_lowres_core_xop); - p.frameSubSampleLuma = PFX(frame_subsample_luma_xop); } #if X86_64 @@ -4696,8 +4686,6 @@ void setupAssemblyPrimitives(EncoderPrimitives &p, int cpuMask) // Main p.saoCuStatsE2 = PFX(saoCuStatsE2_avx2); p.saoCuStatsE3 = PFX(saoCuStatsE3_avx2); - p.frameSubSampleLuma = PFX(frame_subsample_luma_avx2); - if (cpuMask & X265_CPU_BMI2) { p.scanPosLast = PFX(scanPosLast_avx2_bmi2); diff --git a/source/encoder/api.cpp b/source/encoder/api.cpp index 9e3e63291..efcc6bdb6 100644 --- a/source/encoder/api.cpp +++ b/source/encoder/api.cpp @@ -1417,6 +1417,8 @@ FILE* x265_csvlog_open(const x265_param* param) else if (param->rc.rateControlMode == X265_RC_CQP) fprintf(csvfp, ", Target QP"); } + if (param->bSelectiveMCSTF) + fprintf(csvfp, ", Frame Noise, IsMCSTFEnabled"); } fprintf(csvfp, "\n"); } @@ -1556,6 +1558,13 @@ void x265_csvlog_frame(const x265_param* param, const x265_picture* pic) else if (param->rc.rateControlMode == X265_RC_CQP) fprintf(param->csvfpt, ", %d", frameStats->currTrQP); } + if (param->bSelectiveMCSTF) + { + if (frameStats->frameNoise >= 0) + fprintf(param->csvfpt, ", %d, %d", frameStats->frameNoise, frameStats->isMCSTFEnabled); + else + fprintf(param->csvfpt, ", -, %d", frameStats->isMCSTFEnabled); + } } fprintf(param->csvfpt, "\n"); fflush(stderr); diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp index 2c8f8b73b..d6d746903 100644 --- a/source/encoder/encoder.cpp +++ b/source/encoder/encoder.cpp @@ -2924,11 +2924,10 @@ void Encoder::printSummary() for (int i = 0; i < m_numPools; i++) totalWorkerCount += m_threadPool[i].m_numWorkers; - int64_t batchElapsedTime, coopSliceElapsedTime; - uint64_t batchCount, coopSliceCount; - m_lookahead->getWorkerStats(batchElapsedTime, batchCount, coopSliceElapsedTime, coopSliceCount); + int64_t framecostBatchElapsedTime, coopSliceElapsedTime, mcstfBatchElapsedTime; + m_lookahead->getWorkerStats(framecostBatchElapsedTime, coopSliceElapsedTime, mcstfBatchElapsedTime); int64_t lookaheadWorkerTime = m_lookahead->m_slicetypeDecideElapsedTime + m_lookahead->m_preLookaheadElapsedTime + - batchElapsedTime + coopSliceElapsedTime; + framecostBatchElapsedTime + coopSliceElapsedTime + mcstfBatchElapsedTime; int64_t totalWorkerTime = cuStats.totalCTUTime + cuStats.loopFilterElapsedTime + cuStats.pmodeTime + cuStats.pmeTime + lookaheadWorkerTime + cuStats.weightAnalyzeTime + cuStats.tmeTime; @@ -3015,10 +3014,17 @@ void Encoder::printSummary() ELAPSED_MSEC(cuStats.pmodeTime) / cuStats.countPModeTasks); } - x265_log(m_param, X265_LOG_INFO, "CU: %%%05.2lf time spent in slicetypeDecide (avg %.3lfms) and prelookahead (avg %.3lfms)\n", + x265_log(m_param, X265_LOG_INFO, "CU: %%%05.2lf time spent in slicetypeDecide (avg %.3lfms), framecost estimation (avg %.3lfms) and prelookahead (avg %.3lfms)\n", 100.0 * lookaheadWorkerTime / totalWorkerTime, ELAPSED_MSEC(m_lookahead->m_slicetypeDecideElapsedTime) / m_lookahead->m_countSlicetypeDecide, + ELAPSED_MSEC(m_lookahead->m_framecostElapsedTime) / m_lookahead->m_countFramecosts, ELAPSED_MSEC(m_lookahead->m_preLookaheadElapsedTime) / m_lookahead->m_countPreLookahead); + if (m_param->bEnableTemporalFilter) + { + x265_log(m_param, X265_LOG_INFO, "CU: %%%05.2lf time spent in temporal filtering (avg %.3lfms per frame)\n", + 100.0 * mcstfBatchElapsedTime / totalWorkerTime, + ELAPSED_MSEC(m_lookahead->m_temporalFilterElapsedTime) / m_lookahead->m_countTemporalFilter); + } x265_log(m_param, X265_LOG_INFO, "CU: %%%05.2lf time spent in other tasks\n", 100.0 * unaccounted / totalWorkerTime); @@ -3320,6 +3326,12 @@ void Encoder::finishFrameStats(Frame* curFrame, FrameEncoder *curEncoder, x265_f frameStats->currTrQP = curFrame->m_targetQp; } + if (m_param->bSelectiveMCSTF && m_param->csvLogLevel >= 2) + { + frameStats->frameNoise = curFrame->m_lowres.noiseScore; + frameStats->isMCSTFEnabled = curFrame->m_lowres.filterThisGOP ? 1 : 0; + } + if (m_param->csvLogLevel >= 1) { frameStats->cuStats.percentIntraNxN = curFrame->m_encData->m_frameStats.percentIntraNxN; @@ -4255,10 +4267,21 @@ void Encoder::configure(x265_param *p) p->limitReferences = 0; } - if ((m_param->bEnableTemporalFilter) && (p->bframes < 5)){ - x265_log(p, X265_LOG_WARNING, "Setting the number of B-frames to 5, as MCSTF filter is enabled.\n"); - p->bframes = 5; + if (p->bSelectiveMCSTF && !p->bEnableTemporalFilter) + p->bEnableTemporalFilter = 1; + + if ((p->bEnableTemporalFilter) && (p->mcstfFrameRange > p->lookaheadDepth)) + { + x265_log(p, X265_LOG_WARNING, "MCSTF frame range is greater than lookahead depth. Disabling MCSTF.\n"); + p->bEnableTemporalFilter = 0; + } + + if (p->bSelectiveMCSTF && !p->bEnableTemporalFilter) + { + x265_log(p, X265_LOG_WARNING, "selective-mcstf requires mcstf to be enabled. Disabling selective-mcstf.\n"); + p->bSelectiveMCSTF = 0; } + if ((p->bEnableTemporalSubLayers > 2) && !p->bframes) { x265_log(p, X265_LOG_WARNING, "B frames not enabled, temporal sublayer disabled\n"); diff --git a/source/encoder/frameencoder.cpp b/source/encoder/frameencoder.cpp index 366c71a87..5b64950b4 100644 --- a/source/encoder/frameencoder.cpp +++ b/source/encoder/frameencoder.cpp @@ -660,10 +660,10 @@ void FrameEncoder::compressFrame(int layer) } } } - if (m_param->bEnableTemporalFilter) + if (m_param->bEnableTemporalFilter && m_top->isFilterThisframe(m_frame[layer]->m_mcstf->m_sliceTypeConfig, m_frame[layer]->m_lowres.sliceType) && m_frame[layer]->m_lowres.filterThisGOP) { m_frame[layer]->m_mcstf->m_QP = qp; - m_frame[layer]->m_mcstf->bilateralFilter(m_frame[layer], m_frame[layer]->m_mcstfRefList, m_param->temporalFilterStrength); + m_frame[layer]->m_mcstf->bilateralFilter(m_frame[layer], m_frame[layer]->m_mcstfRefList, m_pool); } if (m_nr) diff --git a/source/encoder/slicetype.cpp b/source/encoder/slicetype.cpp index 4d362112d..26f62770b 100644 --- a/source/encoder/slicetype.cpp +++ b/source/encoder/slicetype.cpp @@ -36,9 +36,11 @@ #include "ratecontrol.h" #if DETAILED_CU_STATS -#define ProfileLookaheadTime(elapsed, count) ScopedElapsedTime _scope(elapsed); count++ +#define ProfileLookaheadTimeCount(elapsed, count) ScopedElapsedTime _scope(elapsed); count++ +#define ProfileLookaheadTime(elapsed) ScopedElapsedTime _scope(elapsed) #else -#define ProfileLookaheadTime(elapsed, count) +#define ProfileLookaheadTimeCount(elapsed, count) +#define ProfileLookaheadTime(elapsed) #endif using namespace X265_NS; @@ -95,14 +97,14 @@ uint32_t acEnergyVarHist(uint64_t sum_ssd, int shift) return ssd - ((uint64_t)sum * sum >> shift); } -bool computeEdge(pixel* edgePic, pixel* refPic, pixel* edgeTheta, intptr_t stride, int height, int width, bool bcalcTheta, pixel whitePixel) +bool computeEdge(pixel* edgePic, const pixel* refPic, pixel* edgeTheta, intptr_t stride, int height, int width, bool bcalcTheta, pixel whitePixel, int32_t* gradMag) { intptr_t rowOne = 0, rowTwo = 0, rowThree = 0, colOne = 0, colTwo = 0, colThree = 0; intptr_t middle = 0, topLeft = 0, topRight = 0, bottomLeft = 0, bottomRight = 0; const int startIndex = 1; - if (!edgePic || !refPic || (!edgeTheta && bcalcTheta)) + if ((!edgePic && !gradMag) || !refPic || (!edgeTheta && bcalcTheta)) { return false; } @@ -140,7 +142,7 @@ bool computeEdge(pixel* edgePic, pixel* refPic, pixel* edgeTheta, intptr_t strid gradientH = (float)(-3 * refPic[topLeft] + 3 * refPic[topRight] - 10 * refPic[rowTwo + colOne] + 10 * refPic[rowTwo + colThree] - 3 * refPic[bottomLeft] + 3 * refPic[bottomRight]); gradientV = (float)(-3 * refPic[topLeft] - 10 * refPic[rowOne + colTwo] - 3 * refPic[topRight] + 3 * refPic[bottomLeft] + 10 * refPic[rowThree + colTwo] + 3 * refPic[bottomRight]); gradientMagnitude = sqrtf(gradientH * gradientH + gradientV * gradientV); - if(bcalcTheta) + if(bcalcTheta) { edgeTheta[middle] = 0; radians = atan2(gradientV, gradientH); @@ -149,13 +151,43 @@ bool computeEdge(pixel* edgePic, pixel* refPic, pixel* edgeTheta, intptr_t strid theta = 180 + theta; edgeTheta[middle] = (pixel)theta; } - edgePic[middle] = (pixel)(gradientMagnitude >= EDGE_THRESHOLD ? whitePixel : blackPixel); + if (edgePic) + edgePic[middle] = (pixel)(gradientMagnitude >= EDGE_THRESHOLD ? whitePixel : blackPixel); + if (gradMag) + gradMag[middle] = (int32_t)gradientMagnitude; } } return true; } } +static void gaussianBlur5x5(pixel* dst, const pixel* src, intptr_t stride, int height, int width) +{ + /* 5x5 Gaussian filter (sum = 159) + [2 4 5 4 2] + 1 [4 9 12 9 4] + --- [5 12 15 12 5] + 159 [4 9 12 9 4] + [2 4 5 4 2] */ + for (int rowNum = 2; rowNum < height - 2; rowNum++) + { + for (int colNum = 2; colNum < width - 2; colNum++) + { + const intptr_t rowOne = (rowNum - 2) * stride, colOne = colNum - 2; + const intptr_t rowTwo = (rowNum - 1) * stride, colTwo = colNum - 1; + const intptr_t rowThree = rowNum * stride, colThree = colNum; + const intptr_t rowFour = (rowNum + 1) * stride, colFour = colNum + 1; + const intptr_t rowFive = (rowNum + 2) * stride, colFive = colNum + 2; + dst[rowThree + colThree] = (pixel)(( + 2 * src[rowOne + colOne] + 4 * src[rowOne + colTwo] + 5 * src[rowOne + colThree] + 4 * src[rowOne + colFour] + 2 * src[rowOne + colFive] + + 4 * src[rowTwo + colOne] + 9 * src[rowTwo + colTwo] + 12 * src[rowTwo + colThree] + 9 * src[rowTwo + colFour] + 4 * src[rowTwo + colFive] + + 5 * src[rowThree + colOne] + 12 * src[rowThree + colTwo] + 15 * src[rowThree + colThree] + 12 * src[rowThree + colFour] + 5 * src[rowThree + colFive] + + 4 * src[rowFour + colOne] + 9 * src[rowFour + colTwo] + 12 * src[rowFour + colThree] + 9 * src[rowFour + colFour] + 4 * src[rowFour + colFive] + + 2 * src[rowFive + colOne] + 4 * src[rowFive + colTwo] + 5 * src[rowFive + colThree] + 4 * src[rowFive + colFour] + 2 * src[rowFive + colFive]) / 159); + } + } +} + void edgeFilter(Frame *curFrame, x265_param* param) { int height = curFrame->m_fencPic->m_picHeight; @@ -186,37 +218,7 @@ void edgeFilter(Frame *curFrame, x265_param* param) src = (pixel*)curFrame->m_fencPic->m_picOrg[0]; refPic = curFrame->m_gaussianPic + curFrame->m_fencPic->m_lumaMarginY * stride + curFrame->m_fencPic->m_lumaMarginX; edgePic = curFrame->m_edgePic + curFrame->m_fencPic->m_lumaMarginY * stride + curFrame->m_fencPic->m_lumaMarginX; - pixel pixelValue = 0; - - for (int rowNum = 0; rowNum < height; rowNum++) - { - for (int colNum = 0; colNum < width; colNum++) - { - if ((rowNum >= 2) && (colNum >= 2) && (rowNum < height - 2) && (colNum < width - 2)) //Ignoring the border pixels of the picture - { - /* 5x5 Gaussian filter - [2 4 5 4 2] - 1 [4 9 12 9 4] - --- [5 12 15 12 5] - 159 [4 9 12 9 4] - [2 4 5 4 2]*/ - - const intptr_t rowOne = (rowNum - 2)*stride, colOne = colNum - 2; - const intptr_t rowTwo = (rowNum - 1)*stride, colTwo = colNum - 1; - const intptr_t rowThree = rowNum * stride, colThree = colNum; - const intptr_t rowFour = (rowNum + 1)*stride, colFour = colNum + 1; - const intptr_t rowFive = (rowNum + 2)*stride, colFive = colNum + 2; - const intptr_t index = (rowNum*stride) + colNum; - - pixelValue = ((2 * src[rowOne + colOne] + 4 * src[rowOne + colTwo] + 5 * src[rowOne + colThree] + 4 * src[rowOne + colFour] + 2 * src[rowOne + colFive] + - 4 * src[rowTwo + colOne] + 9 * src[rowTwo + colTwo] + 12 * src[rowTwo + colThree] + 9 * src[rowTwo + colFour] + 4 * src[rowTwo + colFive] + - 5 * src[rowThree + colOne] + 12 * src[rowThree + colTwo] + 15 * src[rowThree + colThree] + 12 * src[rowThree + colFour] + 5 * src[rowThree + colFive] + - 4 * src[rowFour + colOne] + 9 * src[rowFour + colTwo] + 12 * src[rowFour + colThree] + 9 * src[rowFour + colFour] + 4 * src[rowFour + colFive] + - 2 * src[rowFive + colOne] + 4 * src[rowFive + colTwo] + 5 * src[rowFive + colThree] + 4 * src[rowFive + colFour] + 2 * src[rowFive + colFive]) / 159); - refPic[index] = pixelValue; - } - } - } + gaussianBlur5x5(refPic, src, stride, height, width); if(!computeEdge(edgePic, refPic, edgeTheta, stride, height, width, true)) x265_log(NULL, X265_LOG_ERROR, "Failed edge computation!"); @@ -968,6 +970,77 @@ void LookaheadTLD::weightsAnalyse(Lowres& fenc, Lowres& ref) } } +int32_t Lookahead::estimateNoise(Frame* curFrame) +{ + int width = curFrame->m_fencPic->m_picWidth; + int height = curFrame->m_fencPic->m_picHeight; + intptr_t stride = curFrame->m_fencPic->m_stride; + const pixel* src = curFrame->m_fencPic->m_picOrg[0]; + + /* Blur source for Sobel: reuse m_gaussianPic (5×5 Gaussian, already computed by + * edgeFilter() during xPreanalyzeQp) when AQ-edge mode ran it; otherwise compute + * a 3×3 box blur inline. */ + const pixel *blurSrc; + if (m_param->rc.aqMode == X265_AQ_EDGE) + { + blurSrc = curFrame->m_gaussianPic + + curFrame->m_fencPic->m_lumaMarginY * stride + + curFrame->m_fencPic->m_lumaMarginX; + } + else + { + /* Lazy-allocate once per encode; stride*height is constant for a given source */ + if (!m_noiseBlurBuf) + { + m_noiseBlurBuf = X265_MALLOC(pixel, stride * height); + if (!m_noiseBlurBuf) return -65536; + } + memcpy(m_noiseBlurBuf, src, stride * height * sizeof(pixel)); + gaussianBlur5x5(m_noiseBlurBuf, src, stride, height, width); + blurSrc = m_noiseBlurBuf; + } + + /* Gradient magnitudes via computeEdge — pass NULL edgePic, receive raw float magnitudes + * (cast to int32_t) in gradMag for the adaptive threshold step. + * Lazy-allocate once per encode; stride*height is constant for a given source. */ + if (!m_gradMagBuf) + { + m_gradMagBuf = X265_MALLOC(int32_t, stride * height); + if (!m_gradMagBuf) return -65536; + } + int32_t* gradMag = m_gradMagBuf; + memset(gradMag, 0, stride * height * sizeof(int32_t)); + + computeEdge(NULL, blurSrc, NULL, stride, height, width, false, (pixel)EDGE_THRESHOLD, gradMag); + + /* Find peak magnitude then apply adaptive threshold at 15% of peak. + * Sits between Canny's low (10%) and high (30%) thresholds. */ + int32_t maxMag = 1; /* seed at 1 to avoid zero-divide on a fully flat frame */ + for (int i = 1; i < height - 1; ++i) + for (int j = 1; j < width - 1; ++j) + { + int32_t m = gradMag[i * (int)stride + j]; + if (m > maxMag) maxMag = m; + } + int32_t threshold = maxMag * 15 / 100; + + int64_t sum = 0, num = 0; + for (int i = 1; i < height - 1; ++i) + for (int j = 1; j < width - 1; ++j) + { + if (gradMag[i * (int)stride + j] >= threshold) continue; + int k = i * (int)stride + j; + /* Weighted Laplacian: centre x4, 4-neighbours x-2, diagonals x+1 */ + int v = 4 * src[k] + - 2 * (src[k-1] + src[k+1] + src[k-(int)stride] + src[k+(int)stride]) + + (src[k-(int)stride-1] + src[k-(int)stride+1] + + src[k+(int)stride-1] + src[k+(int)stride+1]); + sum += abs(v); ++num; + } + if (num < 16) return -65536; + return (int32_t)((sum * 82137) / (6 * num * (1 << (X265_DEPTH - 8)))); +} + Lookahead::Lookahead(x265_param *param, ThreadPool* pool) { m_param = param; @@ -975,8 +1048,11 @@ Lookahead::Lookahead(x265_param *param, ThreadPool* pool) m_lastNonB = NULL; m_isSceneTransition = false; - m_scratch = NULL; - m_tld = NULL; + m_scratch = NULL; + m_tld = NULL; + m_noiseBlurBuf = NULL; + m_gradMagBuf = NULL; + m_filterThisGOP = false; m_filled = false; m_outputSignalRequired = false; m_isActive = true; @@ -1022,6 +1098,14 @@ Lookahead::Lookahead(x265_param *param, ThreadPool* pool) * of work */ m_bBatchFrameCosts = m_bBatchMotionSearch; + /* MCSTF motion search is independent of --b-adapt: it reuses the batched + * CostEstimateGroup plumbing but is functionally decoupled from the frame + * cost DP. finishBatch() runs serially on the lookahead thread when no pool + * is present, so this needs neither a pool nor trellis b-adapt. Unlike + * m_bBatchMotionSearch it must NOT self-disable on small pools (MCSTF has no + * lazy fallback), so keep it a simple enable flag. */ + m_bMcstfMotionSearch = m_param->bEnableTemporalFilter; + if (m_param->lookaheadSlices && !m_pool) { x265_log(param, X265_LOG_WARNING, "No pools found; disabling lookahead-slices\n"); @@ -1055,8 +1139,12 @@ Lookahead::Lookahead(x265_param *param, ThreadPool* pool) #if DETAILED_CU_STATS m_slicetypeDecideElapsedTime = 0; m_preLookaheadElapsedTime = 0; + m_framecostElapsedTime = 0; + m_temporalFilterElapsedTime = 0; m_countSlicetypeDecide = 0; m_countPreLookahead = 0; + m_countFramecosts = 0; + m_countTemporalFilter = 0; #endif m_accHistDiffRunningAvgCb = X265_MALLOC(uint32_t*, NUMBER_OF_SEGMENTS_IN_WIDTH * sizeof(uint32_t*)); @@ -1104,17 +1192,17 @@ Lookahead::Lookahead(x265_param *param, ThreadPool* pool) } #if DETAILED_CU_STATS -void Lookahead::getWorkerStats(int64_t& batchElapsedTime, uint64_t& batchCount, int64_t& coopSliceElapsedTime, uint64_t& coopSliceCount) +void Lookahead::getWorkerStats(int64_t& framecostBatchElapsedTime, int64_t& coopSliceElapsedTime, int64_t& mcstfBatchElapsedTime) { - batchElapsedTime = coopSliceElapsedTime = 0; - coopSliceCount = batchCount = 0; + framecostBatchElapsedTime = 0; + coopSliceElapsedTime = 0; + mcstfBatchElapsedTime = 0; int tldCount = m_pool ? m_pool->m_numWorkers : 1; for (int i = 0; i < tldCount; i++) { - batchElapsedTime += m_tld[i].batchElapsedTime; + framecostBatchElapsedTime += m_tld[i].framecostBatchElapsedTime; coopSliceElapsedTime += m_tld[i].coopSliceElapsedTime; - batchCount += m_tld[i].countBatches; - coopSliceCount += m_tld[i].countCoopSlices; + mcstfBatchElapsedTime += m_tld[i].mcstfBatchElapsedTime; } } #endif @@ -1184,6 +1272,8 @@ void Lookahead::destroy() X265_FREE(m_accHistDiffRunningAvgCr); X265_FREE(m_accHistDiffRunningAvg[0]); X265_FREE(m_accHistDiffRunningAvg); + X265_FREE(m_noiseBlurBuf); + X265_FREE(m_gradMagBuf); X265_FREE(m_scratch); delete [] m_tld; if (m_param->lookaheadThreads > 0) @@ -1283,7 +1373,7 @@ void Lookahead::findJob(int /*workerThreadID*/) if (!doDecide) return; - ProfileLookaheadTime(m_slicetypeDecideElapsedTime, m_countSlicetypeDecide); + ProfileLookaheadTimeCount(m_slicetypeDecideElapsedTime, m_countSlicetypeDecide); ProfileScopeEvent(slicetypeDecideEV); slicetypeDecide(); @@ -1752,10 +1842,9 @@ void PreLookaheadGroup::processTasks(int workerThreadID) while (m_jobAcquired < m_jobTotal) { Frame* preFrame = m_preframes[m_jobAcquired++]; - ProfileLookaheadTime(m_lookahead.m_preLookaheadElapsedTime, m_lookahead.m_countPreLookahead); ProfileScopeEvent(prelookahead); m_lock.release(); - preFrame->m_lowres.init(preFrame->m_fencPic, preFrame->m_poc); + preFrame->m_lowres.init(preFrame->m_fencPic, preFrame->m_poc, m_lookahead.m_param->bEnableTemporalFilter); if (m_lookahead.m_bAdaptiveQuant) tld.calcAdaptiveQuantFrame(preFrame, m_lookahead.m_param); @@ -1817,19 +1906,6 @@ void Lookahead::compCostBref(Lowres **frames, int start, int end, int num) } } -void CostEstimateGroup::estimatelowresmotion(MotionEstimatorTLD& m_metld, Frame* curframe, int refId) -{ - m_metld.m_bitDepth = curframe->m_param->internalBitDepth; - TemporalFilterRefPicInfo* ref = &curframe->m_mcstfRefList[refId]; - - m_metld.motionEstimationLuma(m_metld, ref->mvs0, ref->mvsStride0, curframe->m_lowres.lowerResPlane[0], (int)(curframe->m_lowres.lumaStride / 2), (curframe->m_lowres.lines / 2), (curframe->m_lowres.width / 2), ref->lowerRes, 16, curframe->m_param->searchRangeForLayer2); - m_metld.motionEstimationLuma(m_metld, ref->mvs1, ref->mvsStride1, curframe->m_lowres.lowresPlane[0], (int)(curframe->m_lowres.lumaStride), (curframe->m_lowres.lines), (curframe->m_lowres.width), ref->lowres, 16, curframe->m_param->searchRangeForLayer1, ref->mvs0, ref->mvsStride0, 2); - m_metld.motionEstimationLuma(m_metld, ref->mvs2, ref->mvsStride2, curframe->m_fencPic->m_picOrg[0], (int)curframe->m_fencPic->m_stride, curframe->m_fencPic->m_picHeight, curframe->m_fencPic->m_picWidth, ref->picBuffer->m_picOrg[0], 16, curframe->m_param->searchRangeForLayer0, ref->mvs1, ref->mvsStride1, 2); - m_metld.motionEstimationLumaDoubleRes(m_metld, ref->mvs, ref->mvsStride, curframe->m_fencPic, ref->picBuffer, 8, ref->mvs2, ref->mvsStride2, 1, ref->error); - - curframe->m_lowres.lowresMcstfMvs[0][refId][0].x = 1; -} - inline int enqueueRefFrame(Frame* iterFrame, Frame* curFrame, bool isPreFiltered, int16_t i) { TemporalFilterRefPicInfo * temp = &curFrame->m_mcstfRefList[curFrame->m_mcstf->m_numRef]; @@ -1881,25 +1957,9 @@ bool Lookahead::generatemcstf(Frame * frameEnc, PicList refPic, int poclast) for (int j = 0; j < (2 * frameEnc->m_mcstf->m_range); j++) { if (iterPOC < 0) - continue; + continue; if (iterPOC >= poclast) - { - - TemporalFilter * mcstf = frameEnc->m_mcstf; - while (mcstf->m_numRef) - { - memset(frameEnc->m_mcstfRefList[mcstf->m_numRef].mvs0, 0, sizeof(MV) * ((mcstf->m_sourceWidth / 16) * (mcstf->m_sourceHeight / 16))); - memset(frameEnc->m_mcstfRefList[mcstf->m_numRef].mvs1, 0, sizeof(MV) * ((mcstf->m_sourceWidth / 16) * (mcstf->m_sourceHeight / 16))); - memset(frameEnc->m_mcstfRefList[mcstf->m_numRef].mvs2, 0, sizeof(MV) * ((mcstf->m_sourceWidth / 16) * (mcstf->m_sourceHeight / 16))); - memset(frameEnc->m_mcstfRefList[mcstf->m_numRef].mvs, 0, sizeof(MV) * ((mcstf->m_sourceWidth / 4) * (mcstf->m_sourceHeight / 4))); - memset(frameEnc->m_mcstfRefList[mcstf->m_numRef].noise, 0, sizeof(int) * ((mcstf->m_sourceWidth / 4) * (mcstf->m_sourceHeight / 4))); - memset(frameEnc->m_mcstfRefList[mcstf->m_numRef].error, 0, sizeof(int) * ((mcstf->m_sourceWidth / 4) * (mcstf->m_sourceHeight / 4))); - - mcstf->m_numRef--; - } - - break; - } + break; Frame * iterFrame = refPic.getPOCMCSTF(iterPOC); if (iterFrame->m_poc == iterPOC) { @@ -1971,6 +2031,7 @@ void Lookahead::slicetypeDecide() /* perform pre-analysis on frames which need it, using a bonded task group */ if (pre.m_jobTotal) { + ProfileLookaheadTimeCount(m_preLookaheadElapsedTime, m_countPreLookahead); if (m_pool) pre.tryBondPeers(*m_pool, pre.m_jobTotal); pre.processTasks(-1); @@ -2177,15 +2238,35 @@ void Lookahead::slicetypeDecide() } } - if (m_bBatchMotionSearch && m_param->bEnableTemporalFilter) + if (m_bMcstfMotionSearch) { - /* pre-calculate all motion searches, using many worker threads */ - CostEstimateGroup estGroup(*this, frames); - Frame* frameEnc = m_inputQueue.first(); m_inputLock.acquire(); + Frame* frameEnc = m_inputQueue.first(); for (int b = 0; b < m_inputQueue.size(); b++) { - if (m_param->bEnableTemporalFilter && isFilterThisframe(frameEnc->m_mcstf->m_sliceTypeConfig, frameEnc->m_lowres.sliceType)) + /* Noise gate: re-evaluate at every GOP boundary (IDR/I/scenecut). + * m_filterThisGOP persists across batches so B/P frames that arrive + * before the next I-frame inherit the previous GOP's decision. + * When selective-mcstf is off, always filter (preserve prior behavior). */ + if (m_param->bSelectiveMCSTF) + { + if (frameEnc->m_lowres.sliceType == X265_TYPE_IDR || + frameEnc->m_lowres.sliceType == X265_TYPE_I || + frameEnc->m_lowres.bScenecut) + { + int32_t score = estimateNoise(frameEnc); + frameEnc->m_lowres.noiseScore = score; + m_filterThisGOP = (score >= NOISE_THRESHOLD); + } + } + else + { + m_filterThisGOP = true; + } + /* Stamp the per-frame flag so frameencoder reads a race-free value */ + frameEnc->m_lowres.filterThisGOP = m_filterThisGOP; + + if (frameEnc->m_lowres.filterThisGOP && isFilterThisframe(frameEnc->m_mcstf->m_sliceTypeConfig, frameEnc->m_lowres.sliceType)) { if (!generatemcstf(frameEnc, m_origPicBuf->m_mcstfPicList, m_inputQueue.last()->m_poc)) { @@ -2193,25 +2274,30 @@ void Lookahead::slicetypeDecide() fflush(stderr); } - for (int j = 1; j <= frameEnc->m_mcstf->m_numRef; j++) + ProfileLookaheadTimeCount(m_temporalFilterElapsedTime, m_countTemporalFilter); + + const int rowSize = PARALLEL_ME_ROWSIZE; + const int origHeight = frameEnc->m_fencPic->m_picHeight; + const int levelHeight[4] = {origHeight >> 2, origHeight >> 1, origHeight, origHeight}; + for(int i = 0; i < MOTION_ESTIMATION_LEVELS; i++) { - TemporalFilterRefPicInfo* ref = &frameEnc->m_mcstfRefList[j - 1]; - int i = ref->poc; + const int numBlockRows = (levelHeight[i] + rowSize - 1) / rowSize; + CostEstimateGroup estGroup(*this, frames); - /* Skip search if already done */ - if (frames[b + 1]->lowresMcstfMvs[0][j - 1][0].x != 0x7FFF) - continue; + for (int j = 1; j <= frameEnc->m_mcstf->m_numRef; j++) + { + const TemporalFilterRefPicInfo* ref = &frameEnc->m_mcstfRefList[j - 1]; + int refpoc = ref->poc; - estGroup.add(j - 1, i, frameEnc->m_poc); + for (int row = 0; row < numBlockRows; row++) + estGroup.addRow(j - 1, refpoc, frameEnc->m_poc, row, i, frameEnc); + } + estGroup.finishBatch(); } } frameEnc = frameEnc->m_next; } m_inputLock.release(); - - /* auto-disable after the first batch if pool is small */ - m_bBatchMotionSearch &= m_pool->m_numWorkers >= 4; - estGroup.finishBatch(); } if (m_param->bEnableTemporalSubLayers > 2) @@ -2832,6 +2918,7 @@ void Lookahead::slicetypeAnalyse(Lowres **frames, bool bKeyframe) if (m_bBatchMotionSearch) { + ProfileLookaheadTimeCount(m_framecostElapsedTime, m_countFramecosts); /* pre-calculate all motion searches, using many worker threads */ CostEstimateGroup estGroup(*this, frames); for (int b = 2; b < numFrames; b++) @@ -4069,9 +4156,30 @@ void CostEstimateGroup::add(int p0, int p1, int b) m_batchMode = true; Estimate& e = m_estimates[m_jobTotal++]; - e.p0 = p0; - e.p1 = p1; - e.b = b; + e.p0 = p0; + e.p1 = p1; + e.b = b; + e.blockRow = -1; + e.MElevel = -1; + e.frame = NULL; + + if (m_jobTotal == MAX_BATCH_SIZE) + finishBatch(); +} + +void CostEstimateGroup::addRow(int refIdx, int poc, int curPoc, int blockRow, int level, Frame* frame) +{ + X265_CHECK(m_batchMode || !m_jobTotal, + "single CostEstimateGroup instance cannot mix batch modes\n"); + m_batchMode = true; + + Estimate& e = m_estimates[m_jobTotal++]; + e.p0 = refIdx; + e.p1 = poc; + e.b = curPoc; + e.blockRow = blockRow; + e.MElevel = level; + e.frame = frame; if (m_jobTotal == MAX_BATCH_SIZE) finishBatch(); @@ -4103,23 +4211,35 @@ void CostEstimateGroup::processTasks(int workerThreadID) if (m_batchMode) { - ProfileLookaheadTime(tld.batchElapsedTime, tld.countBatches); ProfileScopeEvent(estCostSingle); - Estimate& e = m_estimates[i]; - m_lookahead.m_inputLock.acquire(); - Frame* curFrame = m_lookahead.m_inputQueue.getPOC(e.b); - m_lookahead.m_inputLock.release(); - if (m_lookahead.m_param->bEnableTemporalFilter && curFrame && (curFrame->m_lowres.sliceType == X265_TYPE_IDR || curFrame->m_lowres.sliceType == X265_TYPE_I || curFrame->m_lowres.sliceType == X265_TYPE_P)) + Frame* curFrame = e.frame; + if (m_lookahead.m_param->bEnableTemporalFilter && curFrame && m_lookahead.isFilterThisframe(curFrame->m_mcstf->m_sliceTypeConfig, curFrame->m_lowres.sliceType)) { - estimatelowresmotion(m_metld, curFrame, e.p0); + ProfileLookaheadTime(tld.mcstfBatchElapsedTime); + + m_metld.m_bitDepth = curFrame->m_param->internalBitDepth; + TemporalFilterRefPicInfo* ref = &curFrame->m_mcstfRefList[e.p0]; + Lowres *lowres = &curFrame->m_lowres; + const int rowSize = PARALLEL_ME_ROWSIZE; + if (e.MElevel == 0) + m_metld.motionEstimationLuma(ref->mvs0, ref->mvsStride0, lowres->lowerResPlane[0], (int)(lowres->lumaStride / 2), (lowres->lines / 2), (lowres->width / 2), ref->lowerRes, e.blockRow, rowSize); + else if (e.MElevel == 1) + m_metld.motionEstimationLuma(ref->mvs1, ref->mvsStride1, lowres->lowresPlane[0], (int)(lowres->lumaStride), (lowres->lines), (lowres->width), ref->lowres, e.blockRow, rowSize, ref->mvs0, ref->mvsStride0, 2); + else if (e.MElevel == 2) + m_metld.motionEstimationLuma(ref->mvs2, ref->mvsStride2, curFrame->m_fencPic->m_picOrg[0], (int)curFrame->m_fencPic->m_stride, curFrame->m_fencPic->m_picHeight, curFrame->m_fencPic->m_picWidth, ref->picBuffer->m_picOrg[0], e.blockRow, rowSize, ref->mvs1, ref->mvsStride1, 2); + else if (e.MElevel == 3) + m_metld.motionEstimationLumaDoubleRes(ref->mvs, ref->mvsStride, curFrame->m_fencPic, ref->picBuffer, ref->mvs2, ref->mvsStride2, 1, ref->error, e.blockRow, rowSize); } else + { + ProfileLookaheadTime(tld.framecostBatchElapsedTime); estimateFrameCost(tld, e.p0, e.p1, e.b, false); + } } else { - ProfileLookaheadTime(tld.coopSliceElapsedTime, tld.countCoopSlices); + ProfileLookaheadTime(tld.coopSliceElapsedTime); ProfileScopeEvent(estCostCoop); X265_CHECK(i < MAX_COOP_SLICES, "impossible number of coop slices\n"); diff --git a/source/encoder/slicetype.h b/source/encoder/slicetype.h index be6ac8112..36fbeb457 100644 --- a/source/encoder/slicetype.h +++ b/source/encoder/slicetype.h @@ -63,6 +63,9 @@ class Lookahead; #define NUM64x64INPIC(w,h) ((w*h)>> (MAX_LOG2_CU_SIZE<<1)) +#define MOTION_ESTIMATION_LEVELS 4 +#define PARALLEL_ME_ROWSIZE 16 + #if HIGH_BIT_DEPTH #define EDGE_THRESHOLD 1023.0 #else @@ -81,10 +84,9 @@ struct LookaheadTLD int paddedLines; #if DETAILED_CU_STATS - int64_t batchElapsedTime; + int64_t framecostBatchElapsedTime; int64_t coopSliceElapsedTime; - uint64_t countBatches; - uint64_t countCoopSlices; + int64_t mcstfBatchElapsedTime; #endif LookaheadTLD() @@ -96,10 +98,9 @@ struct LookaheadTLD widthInCU = heightInCU = ncu = paddedLines = 0; #if DETAILED_CU_STATS - batchElapsedTime = 0; + framecostBatchElapsedTime = 0; coopSliceElapsedTime = 0; - countBatches = 0; - countCoopSlices = 0; + mcstfBatchElapsedTime = 0; #endif } @@ -162,6 +163,9 @@ class Lookahead : public JobProvider x265_param* m_param; Lowres* m_lastNonB; int* m_scratch; // temp buffer for cutree propagate + pixel* m_noiseBlurBuf; // persistent blur buffer for estimateNoise() fallback path + int32_t* m_gradMagBuf; // persistent gradient-magnitude buffer for estimateNoise() + bool m_filterThisGOP; // noise gate decision for the GOP currently being dispatched /* pre-lookahead */ int m_fullQueueSize; @@ -185,6 +189,7 @@ class Lookahead : public JobProvider bool m_outputSignalRequired; bool m_bBatchMotionSearch; bool m_bBatchFrameCosts; + bool m_bMcstfMotionSearch; bool m_filled; bool m_isSceneTransition; int m_numPools; @@ -210,9 +215,13 @@ class Lookahead : public JobProvider #if DETAILED_CU_STATS int64_t m_slicetypeDecideElapsedTime; int64_t m_preLookaheadElapsedTime; + int64_t m_framecostElapsedTime; + int64_t m_temporalFilterElapsedTime; uint64_t m_countSlicetypeDecide; uint64_t m_countPreLookahead; - void getWorkerStats(int64_t& batchElapsedTime, uint64_t& batchCount, int64_t& coopSliceElapsedTime, uint64_t& coopSliceCount); + uint64_t m_countFramecosts; + uint64_t m_countTemporalFilter; + void getWorkerStats(int64_t& framecostBatchElapsedTime, int64_t& coopSliceElapsedTime, int64_t& mcstfBatchElapsedTime); #endif bool create(); @@ -230,6 +239,7 @@ class Lookahead : public JobProvider int findSliceType(int poc); bool generatemcstf(Frame * frame, PicList refPic, int poclast); bool isFilterThisframe(uint8_t sliceTypeConfig, int curSliceType); + int32_t estimateNoise(Frame* curFrame); protected: @@ -312,10 +322,14 @@ class CostEstimateGroup : public BondedTaskGroup enum { MAX_BATCH_SIZE = 512 }; struct Estimate { - int p0, b, p1; + int p0, b, p1; + int blockRow; + int MElevel; + Frame* frame; } m_estimates[MAX_BATCH_SIZE]; void add(int p0, int p1, int b); + void addRow(int refIdx, int poc, int curPoc, int blockRow, int level, Frame* frame); void finishBatch(); protected: @@ -327,11 +341,9 @@ class CostEstimateGroup : public BondedTaskGroup int64_t estimateFrameCost(LookaheadTLD& tld, int p0, int p1, int b, bool intraPenalty); void estimateCUCost(LookaheadTLD& tld, int cux, int cuy, int p0, int p1, int b, bool bDoSearch[2], bool lastRow, int slice, bool hme); - void estimatelowresmotion(MotionEstimatorTLD& m_metld, Frame* curframe, int refId); - CostEstimateGroup& operator=(const CostEstimateGroup&); }; -bool computeEdge(pixel* edgePic, pixel* refPic, pixel* edgeTheta, intptr_t stride, int height, int width, bool bcalcTheta, pixel whitePixel = EDGE_THRESHOLD); +bool computeEdge(pixel* edgePic, const pixel* refPic, pixel* edgeTheta, intptr_t stride, int height, int width, bool bcalcTheta, pixel whitePixel = (pixel)EDGE_THRESHOLD, int32_t* gradMag = NULL); } #endif // ifndef X265_SLICETYPE_H diff --git a/source/test/CMakeLists.txt b/source/test/CMakeLists.txt index 1da47dd67..6cd058313 100644 --- a/source/test/CMakeLists.txt +++ b/source/test/CMakeLists.txt @@ -53,7 +53,8 @@ add_executable(TestBench ${NASM_SRC} pixelharness.cpp pixelharness.h mbdstharness.cpp mbdstharness.h ipfilterharness.cpp ipfilterharness.h - intrapredharness.cpp intrapredharness.h) + intrapredharness.cpp intrapredharness.h + mcstfharness.cpp mcstfharness.h) target_link_libraries(TestBench x265-static ${PLATFORM_LIBS}) if(LINKER_OPTIONS) diff --git a/source/test/mcstfharness.cpp b/source/test/mcstfharness.cpp new file mode 100644 index 000000000..54f176b21 --- /dev/null +++ b/source/test/mcstfharness.cpp @@ -0,0 +1,270 @@ +/***************************************************************************** + * Copyright (C) 2013-2021 MulticoreWare, Inc + * + * Authors: Kirithika Kalirathnam + + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. + * + * This program is also available under a commercial proprietary license. + * For more information, contact us at license @ x265.com. + *****************************************************************************/ + +#include "common.h" +#include "mcstfharness.h" + +using namespace X265_NS; + +/* Fixed interior position/margins used by every check below: the buffers are + * TEST_STRIDE x TEST_STRIDE with CENTER chosen so that the widest kernel + * (bs = 16, +/- MV_PIXEL_RANGE integer MV, +/- 3 tap padding) never reads + * outside the allocated buffers. */ +static const int CENTER = 40; +static const int MV_PIXEL_RANGE = 4; /* max integer part of a test MV, in pixels */ +static const int PAD = 16; /* margin reserved for applyMotion's plane origin */ + +MCSTFHarness::MCSTFHarness() +{ + for (int i = 0; i < TEST_BUF_SIZE; i++) + { + m_origBuf[i] = (pixel)(rand() % (PIXEL_MAX + 1)); + m_refBuf[i] = (pixel)(rand() % (PIXEL_MAX + 1)); + } +} + +bool MCSTFHarness::check_motionErrorLumaFrac(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt) +{ + const int bsValues[] = { 8, 16 }; + const int bestErrorValues[] = { (8 * 8 * 1024 * 1024), 100 }; + + for (size_t bi = 0; bi < sizeof(bsValues) / sizeof(int); bi++) + { + int bs = bsValues[bi]; + + for (int i = 0; i < ITERS; i++) + { + int dx = (rand() % (2 * MV_PIXEL_RANGE * 16 + 1)) - MV_PIXEL_RANGE * 16; + int dy = (rand() % (2 * MV_PIXEL_RANGE * 16 + 1)) - MV_PIXEL_RANGE * 16; + int besterror = bestErrorValues[i % 2]; + + int resRef = ref.motionErrorLumaFrac( + m_origBuf, TEST_STRIDE, m_refBuf, TEST_STRIDE, + CENTER, CENTER, dx, dy, bs, besterror, X265_DEPTH); + int resOpt = opt.motionErrorLumaFrac( + m_origBuf, TEST_STRIDE, m_refBuf, TEST_STRIDE, + CENTER, CENTER, dx, dy, bs, besterror, X265_DEPTH); + + if (resRef != resOpt) + { + printf("motionErrorLumaFrac[bs=%d]: failed (dx=%d dy=%d besterror=%d ref=%d opt=%d)\n", + bs, dx, dy, besterror, resRef, resOpt); + return false; + } + } + } + return true; +} + +bool MCSTFHarness::check_applyMotion(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt) +{ + struct Cfg { int blockSizeX, blockSizeY, csx, csy; }; + const Cfg cfgs[] = { { 8, 8, 0, 0 }, { 4, 4, 1, 1 } }; + + const int width = 64, height = 64; + pixel* planeOrigin = m_origBuf + PAD * TEST_STRIDE + PAD; + + for (size_t ci = 0; ci < sizeof(cfgs) / sizeof(Cfg); ci++) + { + const Cfg& cfg = cfgs[ci]; + const int mvStride = width / cfg.blockSizeX; + const int mvRows = height / cfg.blockSizeY; + + MV mvs[256]; /* worst case: 64/4 x 64/4 chroma grid */ + for (int i = 0; i < mvStride * mvRows; i++) + mvs[i] = MV((rand() % (2 * MV_PIXEL_RANGE * 16 + 1)) - MV_PIXEL_RANGE * 16, + (rand() % (2 * MV_PIXEL_RANGE * 16 + 1)) - MV_PIXEL_RANGE * 16); + + /* whole-frame call (rowSize = 0) and a row-batched call, matching the + * two ways TemporalFilter::applyMotion is actually driven */ + const struct { int blockRow, rowSize, vShift; } rowCfgs[] = + { + { 0, 0, 0 }, + { 1, 16, cfg.csy }, + }; + + for (size_t ri = 0; ri < sizeof(rowCfgs) / sizeof(rowCfgs[0]); ri++) + { + memset(m_dstBufRef, 0, sizeof(m_dstBufRef)); + memset(m_dstBufOpt, 0, sizeof(m_dstBufOpt)); + pixel* dstOriginRef = m_dstBufRef + PAD * TEST_STRIDE + PAD; + pixel* dstOriginOpt = m_dstBufOpt + PAD * TEST_STRIDE + PAD; + + ref.applyMotion(planeOrigin, TEST_STRIDE, dstOriginRef, TEST_STRIDE, + width, height, cfg.blockSizeX, cfg.blockSizeY, mvStride, mvs, + cfg.csx, cfg.csy, rowCfgs[ri].blockRow, rowCfgs[ri].rowSize, rowCfgs[ri].vShift); + opt.applyMotion(planeOrigin, TEST_STRIDE, dstOriginOpt, TEST_STRIDE, + width, height, cfg.blockSizeX, cfg.blockSizeY, mvStride, mvs, + cfg.csx, cfg.csy, rowCfgs[ri].blockRow, rowCfgs[ri].rowSize, rowCfgs[ri].vShift); + + if (memcmp(m_dstBufRef, m_dstBufOpt, sizeof(m_dstBufRef))) + { + printf("applyMotion[bx=%d by=%d rowSize=%d]: failed\n", + cfg.blockSizeX, cfg.blockSizeY, rowCfgs[ri].rowSize); + return false; + } + } + } + return true; +} + +bool MCSTFHarness::check_computeBlockStats(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt) +{ + const int blkSizes[] = { 4, 8 }; + + for (size_t bi = 0; bi < sizeof(blkSizes) / sizeof(int); bi++) + { + int blkSize = blkSizes[bi]; + + for (int i = 0; i < ITERS; i++) + { + int varRef, diffRef, varOpt, diffOpt; + ref.computeBlockStats(m_origBuf + CENTER * TEST_STRIDE + CENTER, TEST_STRIDE, + m_refBuf + CENTER * TEST_STRIDE + CENTER, TEST_STRIDE, blkSize, &varRef, &diffRef); + opt.computeBlockStats(m_origBuf + CENTER * TEST_STRIDE + CENTER, TEST_STRIDE, + m_refBuf + CENTER * TEST_STRIDE + CENTER, TEST_STRIDE, blkSize, &varOpt, &diffOpt); + + if (varRef != varOpt || diffRef != diffOpt) + { + printf("computeBlockStats[bs=%d]: failed (variance %d vs %d, diffsum %d vs %d)\n", + blkSize, varRef, varOpt, diffRef, diffOpt); + return false; + } + } + } + return true; +} + +bool MCSTFHarness::check_bilateralFilter(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt) +{ + const int blkSizes[] = { 4, 8 }; + const int numRefsValues[] = { 1, 2, 4 }; + const double bdw = 1024.0 / (PIXEL_MAX + 1); + const double maxSample = PIXEL_MAX; + + for (size_t bi = 0; bi < sizeof(blkSizes) / sizeof(int); bi++) + { + int blkSize = blkSizes[bi]; + + for (size_t ni = 0; ni < sizeof(numRefsValues) / sizeof(int); ni++) + { + int numRefs = numRefsValues[ni]; + + const pixel* refBlks[4]; + intptr_t refStrides[4]; + double vww[4], vsw[4]; + for (int i = 0; i < numRefs; i++) + { + refBlks[i] = m_refBuf + CENTER * TEST_STRIDE + CENTER + i * 4; + refStrides[i] = TEST_STRIDE; + vww[i] = 0.1 + (rand() % 100) / 100.0; + vsw[i] = 50.0 + (rand() % 500); + } + + const pixel* srcBlk = m_origBuf + CENTER * TEST_STRIDE + CENTER; + + memset(m_dstBufRef, 0, sizeof(m_dstBufRef)); + memset(m_dstBufOpt, 0, sizeof(m_dstBufOpt)); + + ref.bilateralFilter(srcBlk, TEST_STRIDE, numRefs, refBlks, refStrides, + vww, vsw, bdw, maxSample, blkSize, m_dstBufRef, TEST_STRIDE); + opt.bilateralFilter(srcBlk, TEST_STRIDE, numRefs, refBlks, refStrides, + vww, vsw, bdw, maxSample, blkSize, m_dstBufOpt, TEST_STRIDE); + + for (int y = 0; y < blkSize; y++) + { + if (memcmp(m_dstBufRef + y * TEST_STRIDE, m_dstBufOpt + y * TEST_STRIDE, blkSize * sizeof(pixel))) + { + printf("bilateralFilter[bs=%d numRefs=%d]: failed\n", blkSize, numRefs); + return false; + } + } + } + } + return true; +} + +bool MCSTFHarness::testCorrectness(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt) +{ + if (opt.motionErrorLumaFrac && !check_motionErrorLumaFrac(ref, opt)) + return false; + if (opt.applyMotion && !check_applyMotion(ref, opt)) + return false; + if (opt.computeBlockStats && !check_computeBlockStats(ref, opt)) + return false; + if (opt.bilateralFilter && !check_bilateralFilter(ref, opt)) + return false; + return true; +} + +void MCSTFHarness::measureSpeed(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt) +{ + if (opt.motionErrorLumaFrac) + { + printf("motionErrorLumaFrac[bs=16]\t"); + // cppcheck-suppress unsignedPositive + REPORT_SPEEDUP(opt.motionErrorLumaFrac, ref.motionErrorLumaFrac, + m_origBuf, TEST_STRIDE, m_refBuf, TEST_STRIDE, CENTER, CENTER, 20, 5, 16, INT_MAX, X265_DEPTH); + } + + if (opt.applyMotion) + { + static MV mvs[64]; + pixel* planeOrigin = m_origBuf + PAD * TEST_STRIDE + PAD; + printf("applyMotion[8x8]\t"); + // cppcheck-suppress unsignedPositive + REPORT_SPEEDUP(opt.applyMotion, ref.applyMotion, + planeOrigin, TEST_STRIDE, m_dstBufOpt, TEST_STRIDE, 64, 64, 8, 8, 8, mvs, 0, 0, 0, 0, 0); + } + + if (opt.computeBlockStats) + { + int variance, diffsum; + printf("computeBlockStats[bs=8]\t"); + // cppcheck-suppress unsignedPositive + REPORT_SPEEDUP(opt.computeBlockStats, ref.computeBlockStats, + m_origBuf + CENTER * TEST_STRIDE + CENTER, TEST_STRIDE, + m_refBuf + CENTER * TEST_STRIDE + CENTER, TEST_STRIDE, 8, &variance, &diffsum); + } + + if (opt.bilateralFilter) + { + const pixel* refBlks[4]; + intptr_t refStrides[4]; + double vww[4], vsw[4]; + for (int i = 0; i < 4; i++) + { + refBlks[i] = m_refBuf + CENTER * TEST_STRIDE + CENTER + i * 4; + refStrides[i] = TEST_STRIDE; + vww[i] = 1.0; + vsw[i] = 200.0; + } + const pixel* srcBlk = m_origBuf + CENTER * TEST_STRIDE + CENTER; + const double bdw = 1024.0 / (PIXEL_MAX + 1); + + printf("bilateralFilter[bs=8,refs=4]\t"); + // cppcheck-suppress unsignedPositive + REPORT_SPEEDUP(opt.bilateralFilter, ref.bilateralFilter, + srcBlk, TEST_STRIDE, 4, refBlks, refStrides, vww, vsw, bdw, (double)PIXEL_MAX, 8, m_dstBufOpt, TEST_STRIDE); + } +} diff --git a/source/test/mcstfharness.h b/source/test/mcstfharness.h new file mode 100644 index 000000000..a0ee6eae0 --- /dev/null +++ b/source/test/mcstfharness.h @@ -0,0 +1,62 @@ +/***************************************************************************** + * Copyright (C) 2013-2021 MulticoreWare, Inc + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. + * + * This program is also available under a commercial proprietary license. + * For more information, contact us at license @ x265.com. + *****************************************************************************/ + +#ifndef _MCSTFHARNESS_H_1 +#define _MCSTFHARNESS_H_1 1 + +#include "testharness.h" +#include "temporalfilter.h" + +/* MCSTFPrimitives is a standalone function-pointer table (not part of + * EncoderPrimitives), and it only ever has one SIMD tier (AVX2), so this + * harness does not fit the TestHarness(EncoderPrimitives) interface used by + * the rest of the test bench. It is driven directly from testbench.cpp + * instead of through the generic harness[] loop. */ +class MCSTFHarness +{ +protected: + + enum { TEST_STRIDE = 96 }; + enum { TEST_BUF_SIZE = TEST_STRIDE * TEST_STRIDE }; + enum { ITERS = 32 }; + + ALIGN_VAR_32(pixel, m_origBuf[TEST_BUF_SIZE]); + ALIGN_VAR_32(pixel, m_refBuf[TEST_BUF_SIZE]); + ALIGN_VAR_32(pixel, m_dstBufRef[TEST_BUF_SIZE]); + ALIGN_VAR_32(pixel, m_dstBufOpt[TEST_BUF_SIZE]); + + bool check_motionErrorLumaFrac(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt); + bool check_applyMotion(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt); + bool check_computeBlockStats(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt); + bool check_bilateralFilter(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt); + +public: + + MCSTFHarness(); + + const char *getName() const { return "mcstf"; } + + bool testCorrectness(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt); + + void measureSpeed(const MCSTFPrimitives& ref, const MCSTFPrimitives& opt); +}; + +#endif // ifndef _MCSTFHARNESS_H_1 diff --git a/source/test/pixelharness.cpp b/source/test/pixelharness.cpp index bdda926d9..13a20983b 100644 --- a/source/test/pixelharness.cpp +++ b/source/test/pixelharness.cpp @@ -2845,15 +2845,6 @@ bool PixelHarness::testCorrectness(const EncoderPrimitives& ref, const EncoderPr } } - if (opt.frameSubSampleLuma) - { - if (!check_downscaleluma_t(ref.frameSubSampleLuma, opt.frameSubSampleLuma)) - { - printf("SubSample Luma failed!\n"); - return false; - } - } - if (opt.scale1D_128to64[NONALIGNED]) { if (!check_scale1D_pp(ref.scale1D_128to64[NONALIGNED], opt.scale1D_128to64[NONALIGNED])) @@ -3569,12 +3560,6 @@ void PixelHarness::measureSpeed(const EncoderPrimitives& ref, const EncoderPrimi REPORT_SPEEDUP(opt.frameInitLowres, ref.frameInitLowres, pbuf2, pbuf1, pbuf2, pbuf3, pbuf4, 64, 64, 64, 64); } - if (opt.frameSubSampleLuma) - { - HEADER0("downscaleluma"); - REPORT_SPEEDUP(opt.frameSubSampleLuma, ref.frameSubSampleLuma, pbuf2, pbuf1, 64, 64, 64, 64); - } - if (opt.scale1D_128to64[NONALIGNED]) { HEADER0("scale1D_128to64"); diff --git a/source/test/testbench.cpp b/source/test/testbench.cpp index 3a2ac7a58..fb12e340d 100644 --- a/source/test/testbench.cpp +++ b/source/test/testbench.cpp @@ -31,6 +31,7 @@ #include "mbdstharness.h" #include "ipfilterharness.h" #include "intrapredharness.h" +#include "mcstfharness.h" #include "param.h" #include "cpu.h" @@ -131,7 +132,7 @@ void do_help() printf("usage: TestBench [--cpuid CPU] [--testbench BENCH] [--nobench] [--help]\n\n"); printf(" CPU is comma separated SIMD architecture list, for example: SSE4,AVX\n"); printf(" Use `--cpuid list` to print a list of detected SIMD architectures\n\n"); - printf(" BENCH is one of (pixel,transforms,interp,intrapred)\n\n"); + printf(" BENCH is one of (pixel,transforms,interp,intrapred,mcstf)\n\n"); printf(" `--nobench` disables running benchmarks, only run correctness tests\n\n"); printf("By default, the test bench will test all benches on detected CPU architectures\n"); printf("Options and testbench name may be truncated.\n"); @@ -141,6 +142,7 @@ PixelHarness HPixel; MBDstHarness HMBDist; IPFilterHarness HIPFilter; IntraPredHarness HIPred; +MCSTFHarness HMCSTF; int main(int argc, char *argv[]) { @@ -269,6 +271,30 @@ int main(int argc, char *argv[]) } } +#if X265_ARCH_X86 + /* MCSTFPrimitives is a standalone table with a single SIMD tier (AVX2), so + * it is tested once against the effective (possibly --cpuid restricted) + * cpuid rather than through the per-arch EncoderPrimitives loop above. */ + if (!testname || !strncmp(testname, HMCSTF.getName(), strlen(testname))) + { + MCSTFPrimitives mcstfCRef; + memset(&mcstfCRef, 0, sizeof(mcstfCRef)); + setupMCSTFPrimitives_scalar(mcstfCRef); + + MCSTFPrimitives mcstfOpt; + memset(&mcstfOpt, 0, sizeof(mcstfOpt)); + if (cpuid & X265_CPU_AVX2) + setupIntrinsicMCSTF_avx2(mcstfOpt); + + if (!HMCSTF.testCorrectness(mcstfCRef, mcstfOpt)) + { + fflush(stdout); + fprintf(stderr, "\nx265: mcstf primitive has failed. Go and fix that Right Now!\n"); + return -1; + } + } +#endif + /******************* Cycle count for all primitives **********************/ if (run_benchmarks) { @@ -299,6 +325,23 @@ int main(int argc, char *argv[]) harness[h]->measureSpeed(cprim, optprim); } +#if X265_ARCH_X86 + if (!testname || !strncmp(testname, HMCSTF.getName(), strlen(testname))) + { + MCSTFPrimitives mcstfCRef; + memset(&mcstfCRef, 0, sizeof(mcstfCRef)); + setupMCSTFPrimitives_scalar(mcstfCRef); + + MCSTFPrimitives mcstfOpt; + memset(&mcstfOpt, 0, sizeof(mcstfOpt)); + if (cpuid & X265_CPU_AVX2) + setupIntrinsicMCSTF_avx2(mcstfOpt); + + printf("== %s primitives ==\n", HMCSTF.getName()); + HMCSTF.measureSpeed(mcstfCRef, mcstfOpt); + } +#endif + printf("\n"); } return 0; diff --git a/source/test/testharness.h b/source/test/testharness.h index 64aa70733..64828710d 100644 --- a/source/test/testharness.h +++ b/source/test/testharness.h @@ -107,7 +107,7 @@ static inline uint32_t __rdtsc(void) * Then measures the C reference with BENCH_RUNS / 4 runs and reports X factor and average cycles.*/ #define REPORT_SPEEDUP(RUNOPT, RUNREF, ...) \ { \ - uint32_t cycles = 0; int runs = 0; \ + uint32_t cycles = 0; uint32_t runs = 0; \ RUNOPT(__VA_ARGS__); \ for (int ti = 0; ti < BENCH_RUNS; ti++) { \ uint32_t t0 = (uint32_t)__rdtsc(); \ @@ -118,7 +118,7 @@ static inline uint32_t __rdtsc(void) uint32_t t1 = (uint32_t)__rdtsc() - t0; \ if (t1 * runs <= cycles * 4 && ti > 0) { cycles += t1; runs++; } \ } \ - uint32_t refcycles = 0; int refruns = 0; \ + uint32_t refcycles = 0; uint32_t refruns = 0; \ RUNREF(__VA_ARGS__); \ for (int ti = 0; ti < BENCH_RUNS / 4; ti++) { \ uint32_t t0 = (uint32_t)__rdtsc(); \ diff --git a/source/x265.h b/source/x265.h index 5e49645ce..f50cfeb9d 100644 --- a/source/x265.h +++ b/source/x265.h @@ -320,6 +320,8 @@ typedef struct x265_frame_stats int64_t currTrBitrate; double currTrCRF; int currTrQP; + int32_t frameNoise; /* noise score at GOP start (selective-mcstf); -1 for non-GOP-start frames */ + int isMCSTFEnabled; /* 1 if MCSTF bilateral filter will be applied to this frame */ } x265_frame_stats; typedef struct x265_ctu_info_t @@ -650,6 +652,7 @@ typedef enum #define X265_MAX_GOP_CONFIG 3 #define X265_MAX_GOP_LENGTH 16 #define MAX_T_LAYERS 7 +#define NOISE_THRESHOLD 40000 #if ENABLE_MULTIVIEW #define MAX_VIEWS 2 @@ -2338,11 +2341,9 @@ typedef struct x265_param /*Motion compensated temporal filter*/ int bEnableTemporalFilter; - double temporalFilterStrength; - /*Search Range for L0, L1 and L2 in MCTF*/ - int searchRangeForLayer0; - int searchRangeForLayer1; - int searchRangeForLayer2; + int mcstfFrameRange; + /* When enabled, estimate noise at each GOP boundary and skip MCSTF for clean GOPs */ + int bSelectiveMCSTF; /* Threaded ME */ /* Number of CTUs processed at once when a worker thread picks up a task from ThreadedME. */ @@ -2353,7 +2354,6 @@ typedef struct x265_param /*SBRC*/ int bEnableSBRC; - int mcstfFrameRange; /*Alpha channel encoding*/ int bEnableAlpha; diff --git a/source/x265cli.cpp b/source/x265cli.cpp index 9ec74c83a..b6a7a91b6 100755 --- a/source/x265cli.cpp +++ b/source/x265cli.cpp @@ -390,6 +390,8 @@ namespace X265_NS { H0(" --[no-]frame-dup Enable Frame duplication. Default %s\n", OPT(param->bEnableFrameDuplication)); H0(" --dup-threshold PSNR threshold for Frame duplication. Default %d\n", param->dupThreshold); H0(" --[no-]mcstf Enable GOP-based temporal filter. Default %d\n", param->bEnableTemporalFilter); + H0(" --mcstf-ref-range <0..4> Maximum number of range for MCSTF. Default %d\n", param->mcstfFrameRange); + H0(" --[no-]selective-mcstf Skip MCSTF for GOPs estimated as clean (low noise). Default %d\n", param->bSelectiveMCSTF); #if ENABLE_ALPHA H0(" --alpha Enable alpha channel support. Default %d\n", param->bEnableAlpha); #endif diff --git a/source/x265cli.h b/source/x265cli.h index b41d73748..aaf42ba7f 100644 --- a/source/x265cli.h +++ b/source/x265cli.h @@ -358,6 +358,9 @@ static const struct option long_options[] = { "dup-threshold", required_argument, NULL, 0 }, { "mcstf", no_argument, NULL, 0 }, { "no-mcstf", no_argument, NULL, 0 }, + { "mcstf-ref-range", required_argument, NULL, 0 }, + { "selective-mcstf", no_argument, NULL, 0 }, + { "no-selective-mcstf", no_argument, NULL, 0 }, #if ENABLE_ALPHA { "alpha", no_argument, NULL, 0 }, #endif