From 278c9fc1f887e8a042b9f516d0d1cf0e924ff5bf Mon Sep 17 00:00:00 2001 From: Pierre-Anthony Lemieux Date: Thu, 25 Jun 2026 10:40:33 -0700 Subject: [PATCH 1/2] HTJ2KL256: disable lossy coding of non-visual channels Signed-off-by: Pierre-Anthony Lemieux --- src/lib/OpenEXRCore/internal_ht.cpp | 23 ++++++++++++++-------- src/lib/OpenEXRCore/internal_ht_common.cpp | 15 ++++++++++++++ src/lib/OpenEXRCore/internal_ht_common.h | 11 +++++++++-- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/lib/OpenEXRCore/internal_ht.cpp b/src/lib/OpenEXRCore/internal_ht.cpp index c242904a3..1a1fee368 100644 --- a/src/lib/OpenEXRCore/internal_ht.cpp +++ b/src/lib/OpenEXRCore/internal_ht.cpp @@ -386,9 +386,9 @@ ht_apply_impl (exr_encode_pipeline_t* encode) { exr_result_t rv = EXR_ERR_SUCCESS; - std::vector cs_to_file_ch (encode->channel_count); + std::vector cs_channel_info (encode->channel_count); bool isRGB = make_channel_map ( - encode->channel_count, encode->channels, cs_to_file_ch); + encode->channel_count, encode->channels, cs_channel_info); int image_height = encode->chunk.height; int image_width = encode->chunk.width; @@ -403,7 +403,7 @@ ht_apply_impl (exr_encode_pipeline_t* encode) int64_t bpl = 0; for (int16_t c = 0; c < encode->channel_count; c++) { - int file_c = cs_to_file_ch[c].file_index; + int file_c = cs_channel_info[c].file_index; if (encode->channels[file_c].data_type != EXR_PIXEL_UINT) nlt.set_nonlinear_transform ( c, @@ -452,6 +452,13 @@ ht_apply_impl (exr_encode_pipeline_t* encode) ojph::param_qcd qcd = cs.access_qcd (); qcd.set_irrev_quant (lossy_htj2k_quality); + + /* exclude data channels */ + for (int16_t c = 0; c < encode->channel_count; c++) + { + if (cs_channel_info[c].kind != visual) + cod.get_coc(c).set_reversible(true); + } } try @@ -460,7 +467,7 @@ ht_apply_impl (exr_encode_pipeline_t* encode) size_t header_sz = write_header ( (uint8_t*) encode->compressed_buffer, encode->packed_bytes, - cs_to_file_ch); + cs_channel_info); /* write the codestream */ staticmem_outfile output; @@ -479,7 +486,7 @@ ht_apply_impl (exr_encode_pipeline_t* encode) const uint8_t* line_pixels = static_cast (encode->packed_buffer); - int16_t file_c = cs_to_file_ch[c].file_index; + int16_t file_c = cs_channel_info[c].file_index; for (int64_t y = encode->chunk.start_y; y < image_height + encode->chunk.start_y; @@ -536,12 +543,12 @@ ht_apply_impl (exr_encode_pipeline_t* encode) { for (int16_t c = 0; c < encode->channel_count; c++) { - int file_c = cs_to_file_ch[c].file_index; + int file_c = cs_channel_info[c].file_index; if (encode->channels[file_c].data_type == EXR_PIXEL_HALF) { int16_t* channel_pixels = - (int16_t*) (line_pixels + cs_to_file_ch[c].raster_line_offset); + (int16_t*) (line_pixels + cs_channel_info[c].raster_line_offset); for (int32_t p = 0; p < encode->channels[file_c].width; p++) { @@ -551,7 +558,7 @@ ht_apply_impl (exr_encode_pipeline_t* encode) else { int32_t* channel_pixels = - (int32_t*) (line_pixels + cs_to_file_ch[c].raster_line_offset); + (int32_t*) (line_pixels + cs_channel_info[c].raster_line_offset); for (int32_t p = 0; p < encode->channels[file_c].width; p++) { diff --git a/src/lib/OpenEXRCore/internal_ht_common.cpp b/src/lib/OpenEXRCore/internal_ht_common.cpp index c9c32db9f..3780dd085 100644 --- a/src/lib/OpenEXRCore/internal_ht_common.cpp +++ b/src/lib/OpenEXRCore/internal_ht_common.cpp @@ -172,6 +172,21 @@ make_channel_map ( } } + /** Heuristic detection of channels containing visual light samples */ + for (size_t cs_i = 0; cs_i < cs_to_file_ch.size (); cs_i++) + { + const char* name = channels[cs_to_file_ch[cs_i].file_index].channel_name; + const char* suffix = strrchr (name, '.'); + suffix = suffix ? suffix + 1 : name; + + cs_to_file_ch[cs_i].kind = + (areEqual (suffix, "r") || areEqual (suffix, "g") || + areEqual (suffix, "b") || areEqual (suffix, "red") || + areEqual (suffix, "green") || areEqual (suffix, "blue")) + ? visual + : data; + } + return isRGB; } diff --git a/src/lib/OpenEXRCore/internal_ht_common.h b/src/lib/OpenEXRCore/internal_ht_common.h index 8374ad4b4..6ca6288e3 100644 --- a/src/lib/OpenEXRCore/internal_ht_common.h +++ b/src/lib/OpenEXRCore/internal_ht_common.h @@ -10,12 +10,19 @@ #include #include "openexr_coding.h" +/** Indicates the kind of samples carried by a channel */ +enum J2KChannelKind { + visual, + data +}; + /** Maps a JPEG 2000 codestream component index to the corresponding OpenEXR * file channel index and its byte offset within a packed raster line. */ struct CodestreamChannelInfo { - int file_index; - size_t raster_line_offset; + J2KChannelKind kind; + int file_index; + size_t raster_line_offset; }; /** Build a codestream-to-file channel map for @p channel_count OpenEXR From a68415b68444e985242135421078696c3bfaa15d Mon Sep 17 00:00:00 2001 From: Pierre-Anthony Lemieux Date: Thu, 2 Jul 2026 21:43:48 -0700 Subject: [PATCH 2/2] Bump OpenJPH Signed-off-by: Pierre-Anthony Lemieux --- external/OpenJPH/CMakeLists.txt | 69 +- external/OpenJPH/src/core/CMakeLists.txt | 51 +- .../src/core/codestream/ojph_bitbuffer_read.h | 5 +- .../src/core/codestream/ojph_codeblock.cpp | 43 +- .../core/codestream/ojph_codeblock_fun.cpp | 77 +- .../core/codestream/ojph_codestream_avx.cpp | 19 +- .../core/codestream/ojph_codestream_gen.cpp | 43 +- .../core/codestream/ojph_codestream_local.cpp | 57 +- .../core/codestream/ojph_codestream_local.h | 6 +- .../core/codestream/ojph_codestream_sse.cpp | 17 +- .../core/codestream/ojph_codestream_vsx.cpp | 288 +++ .../core/codestream/ojph_codestream_wasm.cpp | 35 +- .../src/core/codestream/ojph_params.cpp | 906 +++---- .../src/core/codestream/ojph_params_local.h | 37 +- .../OpenJPH/src/core/codestream/ojph_tile.cpp | 35 +- .../core/codestream/ojph_visual_weighting.h | 336 +++ .../src/core/coding/ojph_block_common.cpp | 96 +- .../src/core/coding/ojph_block_common.h | 5 +- .../src/core/coding/ojph_block_decoder.h | 6 + .../src/core/coding/ojph_block_decoder32.cpp | 20 +- .../src/core/coding/ojph_block_decoder64.cpp | 18 +- .../core/coding/ojph_block_decoder_avx2.cpp | 1868 +++++++------- .../core/coding/ojph_block_decoder_ssse3.cpp | 256 +- .../core/coding/ojph_block_decoder_vsx.cpp | 2226 +++++++++++++++++ .../core/coding/ojph_block_decoder_wasm.cpp | 320 +-- .../core/coding/ojph_block_encoder_avx2.cpp | 748 +++--- .../coding/ojph_block_encoder_avx2_apple.h | 1230 +++++++++ external/OpenJPH/src/core/openjph/ojph_arch.h | 178 +- external/OpenJPH/src/core/openjph/ojph_base.h | 22 +- .../OpenJPH/src/core/openjph/ojph_params.h | 170 +- .../OpenJPH/src/core/openjph/ojph_simd_vsx.h | 318 +++ .../OpenJPH/src/core/openjph/ojph_version.h | 5 +- .../OpenJPH/src/core/others/ojph_arch.cpp | 44 +- .../OpenJPH/src/core/shared/ojph_simd_vsx.h | 361 +++ .../src/core/transform/ojph_colour.cpp | 19 + .../src/core/transform/ojph_colour_local.h | 102 +- .../src/core/transform/ojph_colour_vsx.cpp | 668 +++++ .../src/core/transform/ojph_transform.cpp | 15 + .../core/transform/ojph_transform_avx2.cpp | 153 +- .../core/transform/ojph_transform_avx512.cpp | 208 +- .../src/core/transform/ojph_transform_local.h | 143 +- .../core/transform/ojph_transform_sse2.cpp | 139 +- .../src/core/transform/ojph_transform_vsx.cpp | 1317 ++++++++++ .../core/transform/ojph_transform_wasm.cpp | 161 +- src/lib/OpenEXRCore/internal_ht.cpp | 2 +- 45 files changed, 10168 insertions(+), 2674 deletions(-) create mode 100644 external/OpenJPH/src/core/codestream/ojph_codestream_vsx.cpp create mode 100644 external/OpenJPH/src/core/codestream/ojph_visual_weighting.h create mode 100644 external/OpenJPH/src/core/coding/ojph_block_decoder_vsx.cpp create mode 100644 external/OpenJPH/src/core/coding/ojph_block_encoder_avx2_apple.h create mode 100644 external/OpenJPH/src/core/openjph/ojph_simd_vsx.h create mode 100644 external/OpenJPH/src/core/shared/ojph_simd_vsx.h create mode 100644 external/OpenJPH/src/core/transform/ojph_colour_vsx.cpp create mode 100644 external/OpenJPH/src/core/transform/ojph_transform_vsx.cpp diff --git a/external/OpenJPH/CMakeLists.txt b/external/OpenJPH/CMakeLists.txt index 88bded5ee..4e39e1737 100644 --- a/external/OpenJPH/CMakeLists.txt +++ b/external/OpenJPH/CMakeLists.txt @@ -40,6 +40,7 @@ option(OJPH_ENABLE_TIFF_SUPPORT "Enables input and output support for TIFF files option(OJPH_BUILD_TESTS "Enables building test code" OFF) option(OJPH_BUILD_EXECUTABLES "Enables building command line executables" ON) option(OJPH_BUILD_STREAM_EXPAND "Enables building ojph_stream_expand executable" OFF) +option(OJPH_INSTALL "Install OpenJPH libraries, headers, and CMake config" ON) option(OJPH_BUILD_FUZZER "Enables building oss-fuzzing target executable" OFF) option(OJPH_DISABLE_SIMD "Disables the use of SIMD instructions -- agnostic to architectures" OFF) @@ -104,7 +105,9 @@ if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24.0") endif() endif() -## Added by Michael Smith +## Set build flags for AddressSanitizer (ASAN) +# these are used when the user sets -DCMAKE_BUILD_TYPE=asan i.e ("cmake .. -DCMAKE_BUILD_TYPE=asan") +# setting flags for specific build types is documented here: https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_FLAGS_CONFIG.html set(CMAKE_CXX_FLAGS_ASAN "-fsanitize=address -fno-optimize-sibling-calls -fsanitize-address-use-after-scope -fno-omit-frame-pointer -g -O1" CACHE STRING "Flags used by the C++ compiler during AddressSanitizer builds." @@ -190,46 +193,46 @@ endif() ################################################################################################ if (OJPH_INSTALL) - install(EXPORT openjph-targets - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/openjph - ) +install(EXPORT openjph-targets + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/openjph +) - include(CMakePackageConfigHelpers) +include(CMakePackageConfigHelpers) - configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/src/openjph-config.cmake.in - "${CMAKE_CURRENT_BINARY_DIR}/openjph-config.cmake" - INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/openjph - ) +configure_package_config_file(${CMAKE_CURRENT_SOURCE_DIR}/src/openjph-config.cmake.in + "${CMAKE_CURRENT_BINARY_DIR}/openjph-config.cmake" + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/openjph +) - write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/openjph-config-version.cmake - COMPATIBILITY SameMinorVersion) +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/openjph-config-version.cmake + COMPATIBILITY SameMinorVersion) - install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openjph-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/openjph-config-version.cmake - DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/openjph - ) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openjph-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/openjph-config-version.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/openjph +) - if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}") - set(PKG_CONFIG_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}") - else() - set(PKG_CONFIG_INCLUDEDIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") - endif() +if(IS_ABSOLUTE "${CMAKE_INSTALL_INCLUDEDIR}") + set(PKG_CONFIG_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}") +else() + set(PKG_CONFIG_INCLUDEDIR "\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}") +endif() - if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}") - set(PKG_CONFIG_LIBDIR "${CMAKE_INSTALL_LIBDIR}") - else() - set(PKG_CONFIG_LIBDIR "\${prefix}/${CMAKE_INSTALL_LIBDIR}") - endif() +if(IS_ABSOLUTE "${CMAKE_INSTALL_LIBDIR}") + set(PKG_CONFIG_LIBDIR "${CMAKE_INSTALL_LIBDIR}") +else() + set(PKG_CONFIG_LIBDIR "\${prefix}/${CMAKE_INSTALL_LIBDIR}") +endif() - configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/src/openjph.pc.in" - "${CMAKE_BINARY_DIR}/${PROJECT_NAME}.pc" - @ONLY - ) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/src/openjph.pc.in" + "${CMAKE_BINARY_DIR}/${PROJECT_NAME}.pc" + @ONLY +) - install(FILES "${CMAKE_BINARY_DIR}/${PROJECT_NAME}.pc" - DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig - ) +install(FILES "${CMAKE_BINARY_DIR}/${PROJECT_NAME}.pc" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig +) endif() ################################################################################################ diff --git a/external/OpenJPH/src/core/CMakeLists.txt b/external/OpenJPH/src/core/CMakeLists.txt index 3541abcda..7cf05f234 100644 --- a/external/OpenJPH/src/core/CMakeLists.txt +++ b/external/OpenJPH/src/core/CMakeLists.txt @@ -5,11 +5,13 @@ file(GLOB CODESTREAM_SSE2 "codestream/*_sse2.cpp") file(GLOB CODESTREAM_AVX "codestream/*_avx.cpp") file(GLOB CODESTREAM_AVX2 "codestream/*_avx2.cpp") file(GLOB CODESTREAM_WASM "codestream/*_wasm.cpp") +file(GLOB CODESTREAM_VSX "codestream/*_vsx.cpp") file(GLOB CODING "coding/*.cpp" "coding/*.h") file(GLOB CODING_SSSE3 "coding/*_ssse3.cpp") file(GLOB CODING_WASM "coding/*_wasm.cpp") file(GLOB CODING_AVX2 "coding/*_avx2.cpp") file(GLOB CODING_AVX512 "coding/*_avx512.cpp") +file(GLOB CODING_VSX "coding/*_vsx.cpp") file(GLOB COMMON "openjph/*.h") file(GLOB OTHERS "others/*.cpp" "others/*.c") file(GLOB TRANSFORM "transform/*.cpp" "transform/*.h") @@ -19,10 +21,11 @@ file(GLOB TRANSFORM_AVX "transform/*_avx.cpp") file(GLOB TRANSFORM_AVX2 "transform/*_avx2.cpp") file(GLOB TRANSFORM_AVX512 "transform/*_avx512.cpp") file(GLOB TRANSFORM_WASM "transform/*_wasm.cpp") +file(GLOB TRANSFORM_VSX "transform/*_vsx.cpp") -list(REMOVE_ITEM CODESTREAM ${CODESTREAM_SSE} ${CODESTREAM_SSE2} ${CODESTREAM_AVX} ${CODESTREAM_AVX2} ${CODESTREAM_WASM}) -list(REMOVE_ITEM CODING ${CODING_SSSE3} ${CODING_WASM} ${CODING_AVX2} ${CODING_AVX512}) -list(REMOVE_ITEM TRANSFORM ${TRANSFORM_SSE} ${TRANSFORM_SSE2} ${TRANSFORM_AVX} ${TRANSFORM_AVX2} ${TRANSFORM_AVX512} ${TRANSFORM_WASM}) +list(REMOVE_ITEM CODESTREAM ${CODESTREAM_SSE} ${CODESTREAM_SSE2} ${CODESTREAM_AVX} ${CODESTREAM_AVX2} ${CODESTREAM_WASM} ${CODESTREAM_VSX}) +list(REMOVE_ITEM CODING ${CODING_SSSE3} ${CODING_WASM} ${CODING_AVX2} ${CODING_AVX512} ${CODING_VSX}) +list(REMOVE_ITEM TRANSFORM ${TRANSFORM_SSE} ${TRANSFORM_SSE2} ${TRANSFORM_AVX} ${TRANSFORM_AVX2} ${TRANSFORM_AVX512} ${TRANSFORM_WASM} ${TRANSFORM_VSX}) list(APPEND SOURCES ${CODESTREAM} ${CODING} ${COMMON} ${OTHERS} ${TRANSFORM}) source_group("codestream" FILES ${CODESTREAM}) @@ -112,6 +115,24 @@ else() endif() + if (("${OJPH_TARGET_ARCH}" MATCHES "OJPH_ARCH_PPC64") + AND (CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le|powerpc64le")) + if (NOT OJPH_DISABLE_SIMD) + # native 128-bit VSX kernels (see ojph_simd_vsx.h). Supported + # targets are POWER9 (ISA 3.0) and newer, little-endian only. + # The block decoder dispatch is selective; see + # ojph_codeblock_fun.cpp. + list(APPEND SOURCES ${CODESTREAM_VSX} ${CODING_VSX} ${TRANSFORM_VSX}) + source_group("codestream" FILES ${CODESTREAM_VSX}) + source_group("coding" FILES ${CODING_VSX}) + source_group("transform" FILES ${TRANSFORM_VSX}) + set_source_files_properties(codestream/ojph_codestream_vsx.cpp PROPERTIES COMPILE_FLAGS "-mcpu=power9") + set_source_files_properties(coding/ojph_block_decoder_vsx.cpp PROPERTIES COMPILE_FLAGS "-mcpu=power9") + set_source_files_properties(transform/ojph_transform_vsx.cpp PROPERTIES COMPILE_FLAGS "-mcpu=power9") + set_source_files_properties(transform/ojph_colour_vsx.cpp PROPERTIES COMPILE_FLAGS "-mcpu=power9") + endif() + endif() + endif() endif() @@ -140,7 +161,11 @@ endif() ## include library version/name set_target_properties(openjph PROPERTIES POSITION_INDEPENDENT_CODE ON) target_compile_definitions(openjph PUBLIC _FILE_OFFSET_BITS=64) -target_include_directories(openjph PUBLIC $ $) +target_include_directories(openjph PUBLIC + $ + $ + $ +) ## This is to check if aligned_alloc or posix_memalign is available # We want the code to compile for C11 and C++11. @@ -173,13 +198,13 @@ else() endif() if (OJPH_INSTALL) - install(TARGETS openjph - EXPORT openjph-targets - ) - - install(DIRECTORY openjph/ - DESTINATION include/openjph - FILES_MATCHING - PATTERN "*.h" - ) +install(TARGETS openjph + EXPORT openjph-targets +) + +install(DIRECTORY openjph/ + DESTINATION include/openjph + FILES_MATCHING + PATTERN "*.h" +) endif() diff --git a/external/OpenJPH/src/core/codestream/ojph_bitbuffer_read.h b/external/OpenJPH/src/core/codestream/ojph_bitbuffer_read.h index 9b01f93d7..cacdfa97f 100644 --- a/external/OpenJPH/src/core/codestream/ojph_bitbuffer_read.h +++ b/external/OpenJPH/src/core/codestream/ojph_bitbuffer_read.h @@ -40,6 +40,7 @@ #define OJPH_BITBUFFER_READ_H #include "ojph_defs.h" +#include "ojph_arch.h" #include "ojph_file.h" namespace ojph { @@ -79,7 +80,7 @@ namespace ojph { { if (bbp->bytes_left > 0) { - ui32 t = 0; + ui8 t = 0; if (bbp->file->read(&t, 1) != 1) throw "error reading from file"; bbp->tmp = t; @@ -195,7 +196,7 @@ namespace ojph { ui16 com_len; if (bbp->file->read(&com_len, 2) != 2) throw "error reading from file"; - com_len = swap_byte(com_len); + com_len = swap_bytes_if_le(com_len); if (com_len != 4) throw "something is wrong with SOP length"; int result = diff --git a/external/OpenJPH/src/core/codestream/ojph_codeblock.cpp b/external/OpenJPH/src/core/codestream/ojph_codeblock.cpp index 88e20ebac..a24be39c4 100644 --- a/external/OpenJPH/src/core/codestream/ojph_codeblock.cpp +++ b/external/OpenJPH/src/core/codestream/ojph_codeblock.cpp @@ -3,21 +3,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2019, Aous Naman +// Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -53,7 +53,7 @@ namespace ojph { { ////////////////////////////////////////////////////////////////////////// - void codeblock::pre_alloc(codestream *codestream, const size& nominal, + void codeblock::pre_alloc(codestream *codestream, const size& nominal, ui32 precision) { mem_fixed_allocator* allocator = codestream->get_allocator(); @@ -61,7 +61,7 @@ namespace ojph { assert(byte_alignment / sizeof(ui32) > 1); const ui32 f = byte_alignment / sizeof(ui32) - 1; ui32 stride = (nominal.w + f) & ~f; // a multiple of 8 - + if (precision <= 32) allocator->pre_alloc_data(nominal.h * (size_t)stride, 0); else @@ -114,23 +114,25 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void codeblock::push(line_buf *line) { - // convert to sign and magnitude and keep max_val + // convert to sign and magnitude and keep max_val if (precision == BUF32) { assert(line->flags & line_buf::LFT_32BIT); - const si32 *sp = line->i32 + line_offset; + const void *sp = (line->flags & line_buf::LFT_INTEGER) + ? (const void*)(line->i32 + line_offset) + : (const void*)(line->f32 + line_offset); ui32 *dp = buf32 + cur_line * stride; - this->codeblock_functions.tx_to_cb32(sp, dp, K_max, delta_inv, + this->codeblock_functions.tx_to_cb32(sp, dp, K_max, delta_inv, cb_size.w, max_val32); ++cur_line; } - else + else { assert(precision == BUF64); assert(line->flags & line_buf::LFT_64BIT); const si64 *sp = line->i64 + line_offset; ui64 *dp = buf64 + cur_line * stride; - this->codeblock_functions.tx_to_cb64(sp, dp, K_max, delta_inv, + this->codeblock_functions.tx_to_cb64(sp, dp, K_max, delta_inv, cb_size.w, max_val64); ++cur_line; } @@ -148,7 +150,7 @@ namespace ojph { assert(coded_cb->missing_msbs > 0); assert(coded_cb->missing_msbs < K_max); coded_cb->num_passes = 1; - + this->codeblock_functions.encode_cb32(buf32, K_max-1, 1, cb_size.w, cb_size.h, stride, coded_cb->pass_length, elastic, coded_cb->next_coded); @@ -164,7 +166,7 @@ namespace ojph { assert(coded_cb->missing_msbs > 0); assert(coded_cb->missing_msbs < K_max); coded_cb->num_passes = 1; - + this->codeblock_functions.encode_cb64(buf64, K_max-1, 1, cb_size.w, cb_size.h, stride, coded_cb->pass_length, elastic, coded_cb->next_coded); @@ -199,7 +201,7 @@ namespace ojph { coded_cb->pass_length[0], coded_cb->pass_length[1], cb_size.w, cb_size.h, stride, stripe_causal); } - else + else { assert(precision == BUF64); result = this->codeblock_functions.decode_cb64( @@ -231,11 +233,13 @@ namespace ojph { if (precision == BUF32) { assert(line->flags & line_buf::LFT_32BIT); - si32 *dp = line->i32 + line_offset; + void *dp = (line->flags & line_buf::LFT_INTEGER) + ? (void*)(line->i32 + line_offset) + : (void*)(line->f32 + line_offset); if (!zero_block) { const ui32 *sp = buf32 + cur_line * stride; - this->codeblock_functions.tx_from_cb32(sp, dp, K_max, delta, + this->codeblock_functions.tx_from_cb32(sp, dp, K_max, delta, cb_size.w); } else @@ -244,12 +248,13 @@ namespace ojph { else { assert(precision == BUF64); - assert(line->flags & line_buf::LFT_64BIT); + assert((reversible && (line->flags & line_buf::LFT_64BIT)) + || (!reversible && (line->flags & line_buf::LFT_32BIT))); si64 *dp = line->i64 + line_offset; if (!zero_block) { const ui64 *sp = buf64 + cur_line * stride; - this->codeblock_functions.tx_from_cb64(sp, dp, K_max, delta, + this->codeblock_functions.tx_from_cb64(sp, dp, K_max, delta, cb_size.w); } else diff --git a/external/OpenJPH/src/core/codestream/ojph_codeblock_fun.cpp b/external/OpenJPH/src/core/codestream/ojph_codeblock_fun.cpp index cad2434ac..27a562366 100644 --- a/external/OpenJPH/src/core/codestream/ojph_codeblock_fun.cpp +++ b/external/OpenJPH/src/core/codestream/ojph_codeblock_fun.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2019, Aous Naman +// Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -61,16 +61,19 @@ namespace ojph { void sse_mem_clear(void* addr, size_t count); void avx_mem_clear(void* addr, size_t count); void wasm_mem_clear(void* addr, size_t count); + void vsx_mem_clear(void* addr, size_t count); ////////////////////////////////////////////////////////////////////////// ui32 gen_find_max_val32(ui32* address); ui32 sse2_find_max_val32(ui32* address); ui32 avx2_find_max_val32(ui32* address); ui32 wasm_find_max_val32(ui32* address); + ui32 vsx_find_max_val32(ui32* address); ui64 gen_find_max_val64(ui64* address); ui64 sse2_find_max_val64(ui64* address); ui64 avx2_find_max_val64(ui64* address); ui64 wasm_find_max_val64(ui64* address); + ui64 vsx_find_max_val64(ui64* address); ////////////////////////////////////////////////////////////////////////// @@ -88,8 +91,12 @@ namespace ojph { float delta_inv, ui32 count, ui32* max_val); void wasm_rev_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, float delta_inv, ui32 count, ui32* max_val); + void vsx_rev_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, + float delta_inv, ui32 count, ui32* max_val); void wasm_irv_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, float delta_inv, ui32 count, ui32* max_val); + void vsx_irv_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, + float delta_inv, ui32 count, ui32* max_val); void gen_rev_tx_to_cb64(const void *sp, ui64 *dp, ui32 K_max, float delta_inv, ui32 count, ui64* max_val); @@ -99,6 +106,8 @@ namespace ojph { float delta_inv, ui32 count, ui64* max_val); void wasm_rev_tx_to_cb64(const void *sp, ui64 *dp, ui32 K_max, float delta_inv, ui32 count, ui64* max_val); + void vsx_rev_tx_to_cb64(const void *sp, ui64 *dp, ui32 K_max, + float delta_inv, ui32 count, ui64* max_val); ////////////////////////////////////////////////////////////////////////// void gen_rev_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, @@ -115,8 +124,12 @@ namespace ojph { float delta, ui32 count); void wasm_rev_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, float delta, ui32 count); + void vsx_rev_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, + float delta, ui32 count); void wasm_irv_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, float delta, ui32 count); + void vsx_irv_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, + float delta, ui32 count); void gen_rev_tx_from_cb64(const ui64 *sp, void *dp, ui32 K_max, float delta, ui32 count); @@ -124,8 +137,12 @@ namespace ojph { float delta, ui32 count); void avx2_rev_tx_from_cb64(const ui64 *sp, void *dp, ui32 K_max, float delta, ui32 count); + void gen_irv_tx_from_cb64(const ui64 *sp, void *dp, ui32 K_max, + float delta, ui32 count); void wasm_rev_tx_from_cb64(const ui64 *sp, void *dp, ui32 K_max, - float delta, ui32 count); + float delta, ui32 count); + void vsx_rev_tx_from_cb64(const ui64 *sp, void *dp, ui32 K_max, + float delta, ui32 count); void codeblock_fun::init(bool reversible) { @@ -155,11 +172,11 @@ namespace ojph { else { tx_to_cb64 = NULL; - tx_from_cb64 = NULL; + tx_from_cb64 = gen_irv_tx_from_cb64; } encode_cb64 = ojph_encode_codeblock64; bool result = initialize_block_encoder_tables(); - assert(result); ojph_unused(result); + assert(result); ojph_unused(result); #ifndef OJPH_DISABLE_SIMD @@ -190,7 +207,7 @@ namespace ojph { else { tx_to_cb64 = NULL; - tx_from_cb64 = NULL; + tx_from_cb64 = gen_irv_tx_from_cb64; } } #endif // !OJPH_DISABLE_SSE2 @@ -229,7 +246,7 @@ namespace ojph { else { tx_to_cb64 = NULL; - tx_from_cb64 = NULL; + tx_from_cb64 = gen_irv_tx_from_cb64; } } #endif // !OJPH_DISABLE_AVX2 @@ -243,7 +260,41 @@ namespace ojph { #endif // !OJPH_DISABLE_AVX512 #elif defined(OJPH_ARCH_ARM) - + + #elif defined(OJPH_ARCH_PPC64LE) + + // 128-bit VSX kernels; see ojph_simd_vsx.h. + // The SIMD block decoder is used everywhere on POWER10 (ISA 3.1), + // where it beats the scalar decoder on all measured content. On + // POWER9 it wins for irreversible content (more magnitude bits + // per sample) but trails the scalar decoder slightly on + // reversible content, so it is dispatched only for the former. + if (get_cpu_ext_level() >= PPC_CPU_EXT_LEVEL_ARCH_3_1 || + (!reversible && + get_cpu_ext_level() >= PPC_CPU_EXT_LEVEL_ARCH_3_00)) + decode_cb32 = ojph_decode_codeblock_vsx; + if (get_cpu_ext_level() >= PPC_CPU_EXT_LEVEL_ARCH_3_00) { + find_max_val32 = vsx_find_max_val32; + mem_clear = vsx_mem_clear; + if (reversible) { + tx_to_cb32 = vsx_rev_tx_to_cb32; + tx_from_cb32 = vsx_rev_tx_from_cb32; + } + else { + tx_to_cb32 = vsx_irv_tx_to_cb32; + tx_from_cb32 = vsx_irv_tx_from_cb32; + } + find_max_val64 = vsx_find_max_val64; + if (reversible) { + tx_to_cb64 = vsx_rev_tx_to_cb64; + tx_from_cb64 = vsx_rev_tx_from_cb64; + } + else { + tx_to_cb64 = NULL; + tx_from_cb64 = gen_irv_tx_from_cb64; + } + } + #endif // !(defined(OJPH_ARCH_X86_64) || defined(OJPH_ARCH_I386)) #endif // !OJPH_DISABLE_SIMD @@ -273,11 +324,11 @@ namespace ojph { else { tx_to_cb64 = NULL; - tx_from_cb64 = NULL; + tx_from_cb64 = gen_irv_tx_from_cb64; } encode_cb64 = ojph_encode_codeblock64; bool result = initialize_block_encoder_tables(); - assert(result); ojph_unused(result); + assert(result); ojph_unused(result); #endif // !OJPH_ENABLE_WASM_SIMD diff --git a/external/OpenJPH/src/core/codestream/ojph_codestream_avx.cpp b/external/OpenJPH/src/core/codestream/ojph_codestream_avx.cpp index 19c5a1882..d07076e12 100644 --- a/external/OpenJPH/src/core/codestream/ojph_codestream_avx.cpp +++ b/external/OpenJPH/src/core/codestream/ojph_codestream_avx.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2022, Aous Naman +// Copyright (c) 2022, Aous Naman // Copyright (c) 2022, Kakadu Software Pty Ltd, Australia // Copyright (c) 2022, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -46,12 +46,11 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void avx_mem_clear(void* addr, size_t count) { - float* p = (float*)addr; - __m256 zero = _mm256_setzero_ps(); - for (size_t i = 0; i < count; i += 32, p += 8) - _mm256_storeu_ps(p, zero); + __m256i zero = _mm256_setzero_si256(); + for (size_t i = 0; i < count; i += 32, addr = (char*)addr + 32) + _mm256_storeu_si256((__m256i*)addr, zero); } - + } } diff --git a/external/OpenJPH/src/core/codestream/ojph_codestream_gen.cpp b/external/OpenJPH/src/core/codestream/ojph_codestream_gen.cpp index cdc72c6ef..2501a789f 100644 --- a/external/OpenJPH/src/core/codestream/ojph_codestream_gen.cpp +++ b/external/OpenJPH/src/core/codestream/ojph_codestream_gen.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2022, Aous Naman +// Copyright (c) 2022, Aous Naman // Copyright (c) 2022, Kakadu Software Pty Ltd, Australia // Copyright (c) 2022, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -35,6 +35,8 @@ // Date: 15 May 2022 //***************************************************************************/ +#include + #include "ojph_defs.h" #include "ojph_arch.h" @@ -44,9 +46,7 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void gen_mem_clear(void* addr, size_t count) { - si64* p = (si64*)addr; - for (size_t i = 0; i < count; i += 8) - *p++ = 0; + memset(addr, 0, count); } ////////////////////////////////////////////////////////////////////////// @@ -56,8 +56,8 @@ namespace ojph { ui64 gen_find_max_val64(ui64* addr) { return addr[0]; } ////////////////////////////////////////////////////////////////////////// - void gen_rev_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, - float delta_inv, ui32 count, + void gen_rev_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, + float delta_inv, ui32 count, ui32* max_val) { ojph_unused(delta_inv); @@ -78,8 +78,8 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - void gen_rev_tx_to_cb64(const void *sp, ui64 *dp, ui32 K_max, - float delta_inv, ui32 count, + void gen_rev_tx_to_cb64(const void *sp, ui64 *dp, ui32 K_max, + float delta_inv, ui32 count, ui64* max_val) { ojph_unused(delta_inv); @@ -101,7 +101,7 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void gen_irv_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, - float delta_inv, ui32 count, + float delta_inv, ui32 count, ui32* max_val) { ojph_unused(K_max); @@ -166,6 +166,21 @@ namespace ojph { *p++ = (v & 0x80000000U) ? -val : val; } } - + + ////////////////////////////////////////////////////////////////////////// + void gen_irv_tx_from_cb64(const ui64 *sp, void *dp, ui32 K_max, + float delta, ui32 count) + { + ojph_unused(K_max); + //convert to sign and magnitude + float *p = (float*)dp; + for (ui32 i = count; i > 0; --i) + { + ui64 v = *sp++; + float val = (float)(v & LLONG_MAX) * delta; + *p++ = (v & (ui64)LLONG_MIN) ? -val : val; + } + } + } } \ No newline at end of file diff --git a/external/OpenJPH/src/core/codestream/ojph_codestream_local.cpp b/external/OpenJPH/src/core/codestream/ojph_codestream_local.cpp index 066d184cc..8e89000ab 100644 --- a/external/OpenJPH/src/core/codestream/ojph_codestream_local.cpp +++ b/external/OpenJPH/src/core/codestream/ojph_codestream_local.cpp @@ -182,15 +182,25 @@ namespace ojph { allocator->pre_alloc_obj(num_tileparts); //precinct scratch buffer - ui32 num_decomps = cod.get_num_decompositions(); - size log_cb = cod.get_log_block_dims(); - + // The precinct scratch is shared by all components, but each component + // may override the codeblock/precinct geometry via a COC marker. The + // per-component tag-tree storage (resolution.cpp) is derived from that + // component's effective params, so size the shared buffer from the + // largest ratio across every component (the main COD and all COC + // overrides). Sizing from the COD alone under-reserves the buffer for + // any component whose COC declares a smaller codeblock than the COD. size ratio; - for (ui32 r = 0; r <= num_decomps; ++r) + for (ui32 c = 0; c < num_comps; ++c) { - size log_PP = cod.get_log_precinct_size(r); - ratio.w = ojph_max(ratio.w, log_PP.w - ojph_min(log_cb.w, log_PP.w)); - ratio.h = ojph_max(ratio.h, log_PP.h - ojph_min(log_cb.h, log_PP.h)); + const param_cod* cdp = cod.get_coc(c); + ui32 num_decomps = cdp->get_num_decompositions(); + size log_cb = cdp->get_log_block_dims(); + for (ui32 r = 0; r <= num_decomps; ++r) + { + size log_PP = cdp->get_log_precinct_size(r); + ratio.w = ojph_max(ratio.w, log_PP.w - ojph_min(log_cb.w, log_PP.w)); + ratio.h = ojph_max(ratio.h, log_PP.h - ojph_min(log_cb.h, log_PP.h)); + } } ui32 max_ratio = ojph_max(ratio.w, ratio.h); max_ratio = 1 << max_ratio; @@ -630,7 +640,7 @@ namespace ojph { this->pre_alloc(); this->finalize_alloc(); - ui16 t = swap_byte(JP2K_MARKER::SOC); + ui16 t = swap_bytes_if_le((ui16)JP2K_MARKER::SOC); if (file->write(&t, 2) != 2) OJPH_ERROR(0x00030022, "Error writing to file"); @@ -655,29 +665,36 @@ namespace ojph { if (!nlt.write(file)) OJPH_ERROR(0x00030027, "Error writing to file"); - char buf[] = " OpenJPH Ver " + const char* version_str = "OpenJPH Ver " OJPH_INT_TO_STRING(OPENJPH_VERSION_MAJOR) "." OJPH_INT_TO_STRING(OPENJPH_VERSION_MINOR) "." OJPH_INT_TO_STRING(OPENJPH_VERSION_PATCH) "."; - size_t len = strlen(buf); - *(ui16*)buf = swap_byte(JP2K_MARKER::COM); - *(ui16*)(buf + 2) = swap_byte((ui16)(len - 2)); - //1 for General use (IS 8859-15:1999 (Latin) values) - *(ui16*)(buf + 4) = swap_byte((ui16)(1)); - if (file->write(buf, len) != len) + size_t data_len = strlen(version_str); + + t = swap_bytes_if_le((ui16)JP2K_MARKER::COM); + if (file->write(&t, sizeof(ui16)) != sizeof(ui16)) OJPH_ERROR(0x00030028, "Error writing to file"); + t = swap_bytes_if_le((ui16)(data_len + 4)); + if (file->write(&t, sizeof(ui16)) != sizeof(ui16)) + OJPH_ERROR(0x0003002D, "Error writing to file"); + //1 for General use (IS 8859-15:1999 (Latin) values) + t = swap_bytes_if_le((ui16)(1)); + if (file->write(&t, sizeof(ui16)) != sizeof(ui16)) + OJPH_ERROR(0x0003002E, "Error writing to file"); + if (file->write(version_str, data_len) != data_len) + OJPH_ERROR(0x0003002F, "Error writing to file"); if (comments != NULL) { for (ui32 i = 0; i < num_comments; ++i) { - t = swap_byte(JP2K_MARKER::COM); + t = swap_bytes_if_le((ui16)JP2K_MARKER::COM); if (file->write(&t, 2) != 2) OJPH_ERROR(0x00030029, "Error writing to file"); - t = swap_byte((ui16)(comments[i].len + 4)); + t = swap_bytes_if_le((ui16)(comments[i].len + 4)); if (file->write(&t, 2) != 2) OJPH_ERROR(0x0003002A, "Error writing to file"); //1 for General use (IS 8859-15:1999 (Latin) values) - t = swap_byte(comments[i].Rcom); + t = swap_bytes_if_le(comments[i].Rcom); if (file->write(&t, 2) != 2) OJPH_ERROR(0x0003002B, "Error writing to file"); if (file->write(comments[i].data, comments[i].len)!=comments[i].len) @@ -726,7 +743,7 @@ namespace ojph { else OJPH_ERROR(0x00030041, "error reading marker"); } - com_len = swap_byte(com_len); + com_len = swap_bytes_if_le(com_len); file->seek(com_len - 2, infile_base::OJPH_SEEK_CUR); if (msg != NULL && msg_level != OJPH_MSG_NO_MSG) { @@ -1141,7 +1158,7 @@ namespace ojph { } for (si32 i = 0; i < repeat; ++i) tiles[i].flush(outfile); - ui16 t = swap_byte(JP2K_MARKER::EOC); + ui16 t = swap_bytes_if_le((ui16)JP2K_MARKER::EOC); if (!outfile->write(&t, 2)) OJPH_ERROR(0x00030071, "Error writing to file"); } diff --git a/external/OpenJPH/src/core/codestream/ojph_codestream_local.h b/external/OpenJPH/src/core/codestream/ojph_codestream_local.h index 2776f602a..8166d59e9 100644 --- a/external/OpenJPH/src/core/codestream/ojph_codestream_local.h +++ b/external/OpenJPH/src/core/codestream/ojph_codestream_local.h @@ -40,6 +40,7 @@ #define OJPH_CODESTREAM_LOCAL_H #include "ojph_defs.h" +#include "ojph_arch.h" #include "ojph_params_local.h" namespace ojph { @@ -54,11 +55,6 @@ namespace ojph { namespace local { ///////////////////////////////////////////////////////////////////////// - static inline - ui16 swap_byte(ui16 t) - { - return (ui16)((t << 8) | (t >> 8)); - } ////////////////////////////////////////////////////////////////////////// //defined elsewhere diff --git a/external/OpenJPH/src/core/codestream/ojph_codestream_sse.cpp b/external/OpenJPH/src/core/codestream/ojph_codestream_sse.cpp index a4b8d783e..f1ea5b903 100644 --- a/external/OpenJPH/src/core/codestream/ojph_codestream_sse.cpp +++ b/external/OpenJPH/src/core/codestream/ojph_codestream_sse.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2022, Aous Naman +// Copyright (c) 2022, Aous Naman // Copyright (c) 2022, Kakadu Software Pty Ltd, Australia // Copyright (c) 2022, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -47,10 +47,9 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void sse_mem_clear(void* addr, size_t count) { - float* p = (float*)addr; - __m128 zero = _mm_setzero_ps(); - for (size_t i = 0; i < count; i += 16, p += 4) - _mm_storeu_ps(p, zero); + __m128i zero = _mm_setzero_si128(); + for (size_t i = 0; i < count; i += 16, addr = (char*)addr + 16) + _mm_storeu_si128((__m128i*)addr, zero); } } } diff --git a/external/OpenJPH/src/core/codestream/ojph_codestream_vsx.cpp b/external/OpenJPH/src/core/codestream/ojph_codestream_vsx.cpp new file mode 100644 index 000000000..7d85e4cc5 --- /dev/null +++ b/external/OpenJPH/src/core/codestream/ojph_codestream_vsx.cpp @@ -0,0 +1,288 @@ +//***************************************************************************/ +// This software is released under the 2-Clause BSD license, included +// below. +// +// Copyright (c) 2022, Aous Naman +// Copyright (c) 2022, Kakadu Software Pty Ltd, Australia +// Copyright (c) 2022, The University of New South Wales, Australia +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************/ +// This file is part of the OpenJPH software implementation. +// File: ojph_codestream_vsx.cpp +// Author: Aous Naman +// Date: 15 May 2022 +//***************************************************************************/ + +#include +#include +#include "ojph_simd_vsx.h" + +#include "ojph_defs.h" + +namespace ojph { + namespace local { + + ////////////////////////////////////////////////////////////////////////// + void vsx_mem_clear(void* addr, size_t count) + { + v128_t zero = vsx_i32x4_splat(0); + for (size_t i = 0; i < count; i += 16, addr = (char*)addr + 16) + vsx_v128_store(addr, zero); + } + + ////////////////////////////////////////////////////////////////////////// + ui32 vsx_find_max_val32(ui32* address) + { + v128_t x1, x0 = vsx_v128_load(address); + x1 = vsx_i32x4_shuffle(x0, x0, 2, 3, 2, 3); // x1 = x0[2,3,2,3] + x0 = vsx_v128_or(x0, x1); + x1 = vsx_i32x4_shuffle(x0, x0, 1, 1, 1, 1); // x1 = x0[1,1,1,1] + x0 = vsx_v128_or(x0, x1); + ui32 t = (ui32)vsx_i32x4_extract_lane(x0, 0); + return t; + } + + ////////////////////////////////////////////////////////////////////////// + ui64 vsx_find_max_val64(ui64* address) + { + v128_t x1, x0 = vsx_v128_load(address); + x1 = vsx_i64x2_shuffle(x0, x0, 1, 1); // x1 = x0[2,3,2,3] + x0 = vsx_v128_or(x0, x1); + ui64 t = (ui64)vsx_i64x2_extract_lane(x0, 0); + return t; + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, + float delta_inv, ui32 count, ui32* max_val) + { + ojph_unused(delta_inv); + + // convert to sign and magnitude and keep max_val + ui32 shift = 31 - K_max; + v128_t m0 = vsx_i32x4_splat(INT_MIN); + v128_t zero = vsx_i32x4_splat(0); + v128_t one = vsx_i32x4_splat(1); + v128_t tmax = vsx_v128_load(max_val); + si32 *p = (si32*)sp; + for ( ; count >= 4; count -= 4, p += 4, dp += 4) + { + v128_t v = vsx_v128_load(p); + v128_t sign = vsx_i32x4_lt(v, zero); + v128_t val = vsx_v128_xor(v, sign); // negate 1's complement + v128_t ones = vsx_v128_and(sign, one); + val = vsx_i32x4_add(val, ones); // 2's complement + sign = vsx_v128_and(sign, m0); + val = vsx_i32x4_shl(val, shift); + tmax = vsx_v128_or(tmax, val); + val = vsx_v128_or(val, sign); + vsx_v128_store(dp, val); + } + if (count) + { + v128_t v = vsx_v128_load(p); + v128_t sign = vsx_i32x4_lt(v, zero); + v128_t val = vsx_v128_xor(v, sign); // negate 1's complement + v128_t ones = vsx_v128_and(sign, one); + val = vsx_i32x4_add(val, ones); // 2's complement + sign = vsx_v128_and(sign, m0); + val = vsx_i32x4_shl(val, shift); + + v128_t c = vsx_i32x4_splat((si32)count); + v128_t idx = vsx_i32x4_make(0, 1, 2, 3); + v128_t mask = vsx_i32x4_gt(c, idx); + c = vsx_v128_and(val, mask); + tmax = vsx_v128_or(tmax, c); + + val = vsx_v128_or(val, sign); + vsx_v128_store(dp, val); + } + vsx_v128_store(max_val, tmax); + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, + float delta_inv, ui32 count, ui32* max_val) + { + ojph_unused(K_max); + + //quantize and convert to sign and magnitude and keep max_val + + v128_t d = vsx_f32x4_splat(delta_inv); + v128_t zero = vsx_i32x4_splat(0); + v128_t one = vsx_i32x4_splat(1); + v128_t tmax = vsx_v128_load(max_val); + float *p = (float*)sp; + for ( ; count >= 4; count -= 4, p += 4, dp += 4) + { + v128_t vf = vsx_v128_load(p); + vf = vsx_f32x4_mul(vf, d); // multiply + v128_t val = vsx_i32x4_trunc_sat_f32x4(vf); // convert to signed int + v128_t sign = vsx_i32x4_lt(val, zero); // get sign + val = vsx_v128_xor(val, sign); // negate 1's complement + v128_t ones = vsx_v128_and(sign, one); + val = vsx_i32x4_add(val, ones); // 2's complement + tmax = vsx_v128_or(tmax, val); + sign = vsx_i32x4_shl(sign, 31); + val = vsx_v128_or(val, sign); + vsx_v128_store(dp, val); + } + if (count) + { + v128_t vf = vsx_v128_load(p); + vf = vsx_f32x4_mul(vf, d); // multiply + v128_t val = vsx_i32x4_trunc_sat_f32x4(vf); // convert to signed int + v128_t sign = vsx_i32x4_lt(val, zero); // get sign + val = vsx_v128_xor(val, sign); // negate 1's complement + v128_t ones = vsx_v128_and(sign, one); + val = vsx_i32x4_add(val, ones); // 2's complement + + v128_t c = vsx_i32x4_splat((si32)count); + v128_t idx = vsx_i32x4_make(0, 1, 2, 3); + v128_t mask = vsx_i32x4_gt(c, idx); + c = vsx_v128_and(val, mask); + tmax = vsx_v128_or(tmax, c); + + sign = vsx_i32x4_shl(sign, 31); + val = vsx_v128_or(val, sign); + vsx_v128_store(dp, val); + } + vsx_v128_store(max_val, tmax); + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, + float delta, ui32 count) + { + ojph_unused(delta); + ui32 shift = 31 - K_max; + v128_t m1 = vsx_i32x4_splat(INT_MAX); + v128_t zero = vsx_i32x4_splat(0); + v128_t one = vsx_i32x4_splat(1); + si32 *p = (si32*)dp; + for (ui32 i = 0; i < count; i += 4, sp += 4, p += 4) + { + v128_t v = vsx_v128_load((v128_t*)sp); + v128_t val = vsx_v128_and(v, m1); + val = vsx_i32x4_shr(val, shift); + v128_t sign = vsx_i32x4_lt(v, zero); + val = vsx_v128_xor(val, sign); // negate 1's complement + v128_t ones = vsx_v128_and(sign, one); + val = vsx_i32x4_add(val, ones); // 2's complement + vsx_v128_store(p, val); + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, + float delta, ui32 count) + { + ojph_unused(K_max); + v128_t m1 = vsx_i32x4_splat(INT_MAX); + v128_t d = vsx_f32x4_splat(delta); + float *p = (float*)dp; + for (ui32 i = 0; i < count; i += 4, sp += 4, p += 4) + { + v128_t v = vsx_v128_load((v128_t*)sp); + v128_t vali = vsx_v128_and(v, m1); + v128_t valf = vsx_f32x4_convert_i32x4(vali); + valf = vsx_f32x4_mul(valf, d); + v128_t sign = vsx_v128_andnot(v, m1); + valf = vsx_v128_or(valf, sign); + vsx_v128_store(p, valf); + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_tx_to_cb64(const void *sp, ui64 *dp, ui32 K_max, + float delta_inv, ui32 count, ui64* max_val) + { + ojph_unused(delta_inv); + + // convert to sign and magnitude and keep max_val + ui32 shift = 63 - K_max; + v128_t m0 = vsx_i64x2_splat(LLONG_MIN); + v128_t zero = vsx_i64x2_splat(0); + v128_t one = vsx_i64x2_splat(1); + v128_t tmax = vsx_v128_load(max_val); + si64 *p = (si64*)sp; + for ( ; count >= 2; count -= 2, p += 2, dp += 2) + { + v128_t v = vsx_v128_load(p); + v128_t sign = vsx_i64x2_lt(v, zero); + v128_t val = vsx_v128_xor(v, sign); // negate 1's complement + v128_t ones = vsx_v128_and(sign, one); + val = vsx_i64x2_add(val, ones); // 2's complement + sign = vsx_v128_and(sign, m0); + val = vsx_i64x2_shl(val, shift); + tmax = vsx_v128_or(tmax, val); + val = vsx_v128_or(val, sign); + vsx_v128_store(dp, val); + } + if (count) + { + v128_t v = vsx_v128_load(p); + v128_t sign = vsx_i64x2_lt(v, zero); + v128_t val = vsx_v128_xor(v, sign); // negate 1's complement + v128_t ones = vsx_v128_and(sign, one); + val = vsx_i64x2_add(val, ones); // 2's complement + sign = vsx_v128_and(sign, m0); + val = vsx_i64x2_shl(val, shift); + + v128_t c = vsx_i32x4_make((si32)0xFFFFFFFF, (si32)0xFFFFFFFF, 0, 0); + c = vsx_v128_and(val, c); + tmax = vsx_v128_or(tmax, c); + + val = vsx_v128_or(val, sign); + vsx_v128_store(dp, val); + } + + vsx_v128_store(max_val, tmax); + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_tx_from_cb64(const ui64 *sp, void *dp, ui32 K_max, + float delta, ui32 count) + { + ojph_unused(delta); + ui32 shift = 63 - K_max; + v128_t m1 = vsx_i64x2_splat(LLONG_MAX); + v128_t zero = vsx_i64x2_splat(0); + v128_t one = vsx_i64x2_splat(1); + si64 *p = (si64*)dp; + for (ui32 i = 0; i < count; i += 2, sp += 2, p += 2) + { + v128_t v = vsx_v128_load((v128_t*)sp); + v128_t val = vsx_v128_and(v, m1); + val = vsx_i64x2_shr(val, shift); + v128_t sign = vsx_i64x2_lt(v, zero); + val = vsx_v128_xor(val, sign); // negate 1's complement + v128_t ones = vsx_v128_and(sign, one); + val = vsx_i64x2_add(val, ones); // 2's complement + vsx_v128_store(p, val); + } + } + } +} diff --git a/external/OpenJPH/src/core/codestream/ojph_codestream_wasm.cpp b/external/OpenJPH/src/core/codestream/ojph_codestream_wasm.cpp index b65e35e99..6558b851c 100644 --- a/external/OpenJPH/src/core/codestream/ojph_codestream_wasm.cpp +++ b/external/OpenJPH/src/core/codestream/ojph_codestream_wasm.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2022, Aous Naman +// Copyright (c) 2022, Aous Naman // Copyright (c) 2022, Kakadu Software Pty Ltd, Australia // Copyright (c) 2022, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -36,7 +36,7 @@ //***************************************************************************/ #include -#include +#include #include #include "ojph_defs.h" @@ -47,10 +47,9 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void wasm_mem_clear(void* addr, size_t count) { - float* p = (float*)addr; v128_t zero = wasm_i32x4_splat(0); - for (size_t i = 0; i < count; i += 16, p += 4) - wasm_v128_store(p, zero); + for (size_t i = 0; i < count; i += 16, addr = (char*)addr + 16) + wasm_v128_store(addr, zero); } ////////////////////////////////////////////////////////////////////////// @@ -76,12 +75,12 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - void wasm_rev_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, + void wasm_rev_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, float delta_inv, ui32 count, ui32* max_val) { ojph_unused(delta_inv); - // convert to sign and magnitude and keep max_val + // convert to sign and magnitude and keep max_val ui32 shift = 31 - K_max; v128_t m0 = wasm_i32x4_splat(INT_MIN); v128_t zero = wasm_i32x4_splat(0); @@ -122,7 +121,7 @@ namespace ojph { } wasm_v128_store(max_val, tmax); } - + ////////////////////////////////////////////////////////////////////////// void wasm_irv_tx_to_cb32(const void *sp, ui32 *dp, ui32 K_max, float delta_inv, ui32 count, ui32* max_val) @@ -174,7 +173,7 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - void wasm_rev_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, + void wasm_rev_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, float delta, ui32 count) { ojph_unused(delta); @@ -197,7 +196,7 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - void wasm_irv_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, + void wasm_irv_tx_from_cb32(const ui32 *sp, void *dp, ui32 K_max, float delta, ui32 count) { ojph_unused(K_max); @@ -217,12 +216,12 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - void wasm_rev_tx_to_cb64(const void *sp, ui64 *dp, ui32 K_max, + void wasm_rev_tx_to_cb64(const void *sp, ui64 *dp, ui32 K_max, float delta_inv, ui32 count, ui64* max_val) { ojph_unused(delta_inv); - // convert to sign and magnitude and keep max_val + // convert to sign and magnitude and keep max_val ui32 shift = 63 - K_max; v128_t m0 = wasm_i64x2_splat(LLONG_MIN); v128_t zero = wasm_i64x2_splat(0); @@ -261,10 +260,10 @@ namespace ojph { } wasm_v128_store(max_val, tmax); - } + } ////////////////////////////////////////////////////////////////////////// - void wasm_rev_tx_from_cb64(const ui64 *sp, void *dp, ui32 K_max, + void wasm_rev_tx_from_cb64(const ui64 *sp, void *dp, ui32 K_max, float delta, ui32 count) { ojph_unused(delta); diff --git a/external/OpenJPH/src/core/codestream/ojph_params.cpp b/external/OpenJPH/src/core/codestream/ojph_params.cpp index 65973efe6..2fa0024b8 100644 --- a/external/OpenJPH/src/core/codestream/ojph_params.cpp +++ b/external/OpenJPH/src/core/codestream/ojph_params.cpp @@ -38,10 +38,12 @@ #define _USE_MATH_DEFINES #include +#include "ojph_arch.h" #include "ojph_base.h" #include "ojph_file.h" #include "ojph_params.h" +#include "ojph_visual_weighting.h" #include "ojph_params_local.h" #include "ojph_message.h" @@ -251,12 +253,32 @@ namespace ojph { } //////////////////////////////////////////////////////////////////////////// - param_coc param_cod::get_coc(ui32 component_idx) + void param_cod::set_num_decomposition(ui32 comp_idx, ui32 num_decompositions) { - local::param_cod *p = state->get_coc(component_idx); - if (p == state) // no COC segment marker for this component - p = state->add_coc_object(component_idx); - return param_coc(p); + local::param_cod* cdp = state->get_or_add_coc(comp_idx); + ojph::param_cod(cdp).set_num_decomposition(num_decompositions); + } + + //////////////////////////////////////////////////////////////////////////// + void param_cod::set_block_dims(ui32 comp_idx, ui32 width, ui32 height) + { + local::param_cod* cdp = state->get_or_add_coc(comp_idx); + ojph::param_cod(cdp).set_block_dims(width, height); + } + + //////////////////////////////////////////////////////////////////////////// + void param_cod::set_precinct_size(ui32 comp_idx, int num_levels, + size* precinct_size) + { + local::param_cod* cdp = state->get_or_add_coc(comp_idx); + ojph::param_cod(cdp).set_precinct_size(num_levels, precinct_size); + } + + //////////////////////////////////////////////////////////////////////////// + void param_cod::set_reversible(ui32 comp_idx, bool reversible) + { + local::param_cod* cdp = state->get_or_add_coc(comp_idx); + ojph::param_cod(cdp).set_reversible(reversible); } //////////////////////////////////////////////////////////////////////////// @@ -350,56 +372,32 @@ namespace ojph { } //////////////////////////////////////////////////////////////////////////// - // - // - // - // - // - //////////////////////////////////////////////////////////////////////////// - - //////////////////////////////////////////////////////////////////////////// - void param_coc::set_num_decomposition(ui32 num_decompositions) - { ojph::param_cod(state).set_num_decomposition(num_decompositions); } - - //////////////////////////////////////////////////////////////////////////// - void param_coc::set_block_dims(ui32 width, ui32 height) - { ojph::param_cod(state).set_block_dims(width, height); } - - //////////////////////////////////////////////////////////////////////////// - void param_coc::set_precinct_size(int num_levels, size* precinct_size) - { ojph::param_cod(state).set_precinct_size(num_levels, precinct_size); } - - //////////////////////////////////////////////////////////////////////////// - void param_coc::set_reversible(bool reversible) - { ojph::param_cod(state).set_reversible(reversible); } - - //////////////////////////////////////////////////////////////////////////// - ui32 param_coc::get_num_decompositions() const - { return ojph::param_cod(state).get_num_decompositions(); } + ui32 param_cod::get_num_decompositions(ui32 comp_idx) const + { return state->get_coc(comp_idx)->get_num_decompositions(); } //////////////////////////////////////////////////////////////////////////// - size param_coc::get_block_dims() const - { return ojph::param_cod(state).get_block_dims(); } + size param_cod::get_block_dims(ui32 comp_idx) const + { return state->get_coc(comp_idx)->get_block_dims(); } //////////////////////////////////////////////////////////////////////////// - size param_coc::get_log_block_dims() const - { return ojph::param_cod(state).get_log_block_dims(); } + size param_cod::get_log_block_dims(ui32 comp_idx) const + { return state->get_coc(comp_idx)->get_log_block_dims(); } //////////////////////////////////////////////////////////////////////////// - bool param_coc::is_reversible() const - { return ojph::param_cod(state).is_reversible(); } + bool param_cod::is_reversible(ui32 comp_idx) const + { return state->get_coc(comp_idx)->is_reversible(); } //////////////////////////////////////////////////////////////////////////// - size param_coc::get_precinct_size(ui32 level_num) const - { return ojph::param_cod(state).get_precinct_size(level_num); } + size param_cod::get_precinct_size(ui32 comp_idx, ui32 level_num) const + { return state->get_coc(comp_idx)->get_precinct_size(level_num); } //////////////////////////////////////////////////////////////////////////// - size param_coc::get_log_precinct_size(ui32 level_num) const - { return ojph::param_cod(state).get_log_precinct_size(level_num); } + size param_cod::get_log_precinct_size(ui32 comp_idx, ui32 level_num) const + { return state->get_coc(comp_idx)->get_log_precinct_size(level_num); } //////////////////////////////////////////////////////////////////////////// - bool param_coc::get_block_vertical_causality() const - { return ojph::param_cod(state).get_block_vertical_causality(); } + bool param_cod::get_block_vertical_causality(ui32 comp_idx) const + { return state->get_coc(comp_idx)->get_block_vertical_causality(); } //////////////////////////////////////////////////////////////////////////// @@ -422,6 +420,11 @@ namespace ojph { state->set_delta(comp_idx, delta); } + ////////////////////////////////////////////////////////////////////////// + void param_qcd::set_qfactor(ui32 comp_idx, comp_type ctype, ui8 qfactor) { + state->set_qfactor(comp_idx, ctype, qfactor); + } + //////////////////////////////////////////////////////////////////////////// // // @@ -485,41 +488,6 @@ namespace ojph { namespace local { - ////////////////////////////////////////////////////////////////////////// - static inline - ui16 swap_byte(ui16 t) - { - return (ui16)((t << 8) | (t >> 8)); - } - - ////////////////////////////////////////////////////////////////////////// - static inline - ui32 swap_byte(ui32 t) - { - ui32 u = swap_byte((ui16)(t & 0xFFFFu)); - u <<= 16; - u |= swap_byte((ui16)(t >> 16)); - return u; - } - - ////////////////////////////////////////////////////////////////////////// - static inline - ui64 swap_byte(ui64 t) - { - ui64 u = swap_byte((ui32)(t & 0xFFFFFFFFu)); - u <<= 32; - u |= swap_byte((ui32)(t >> 32)); - return u; - } - - ////////////////////////////////////////////////////////////////////////// - // - // - // - // - // - ////////////////////////////////////////////////////////////////////////// - ////////////////////////////////////////////////////////////////////////// //static class sqrt_energy_gains @@ -637,40 +605,44 @@ namespace ojph { //marker size excluding header Lsiz = (ui16)(38 + 3 * Csiz); - ui8 buf[4]; + ui8 buf1; + ui16 buf2; + ui32 buf4; bool result = true; - *(ui16*)buf = JP2K_MARKER::SIZ; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Lsiz); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Rsiz); - result &= file->write(&buf, 2) == 2; - *(ui32*)buf = swap_byte(Xsiz); - result &= file->write(&buf, 4) == 4; - *(ui32*)buf = swap_byte(Ysiz); - result &= file->write(&buf, 4) == 4; - *(ui32*)buf = swap_byte(XOsiz); - result &= file->write(&buf, 4) == 4; - *(ui32*)buf = swap_byte(YOsiz); - result &= file->write(&buf, 4) == 4; - *(ui32*)buf = swap_byte(XTsiz); - result &= file->write(&buf, 4) == 4; - *(ui32*)buf = swap_byte(YTsiz); - result &= file->write(&buf, 4) == 4; - *(ui32*)buf = swap_byte(XTOsiz); - result &= file->write(&buf, 4) == 4; - *(ui32*)buf = swap_byte(YTOsiz); - result &= file->write(&buf, 4) == 4; - *(ui16*)buf = swap_byte(Csiz); - result &= file->write(&buf, 2) == 2; + buf2 = JP2K_MARKER::SIZ; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Lsiz); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Rsiz); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf4 = swap_bytes_if_le(Xsiz); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); + buf4 = swap_bytes_if_le(Ysiz); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); + buf4 = swap_bytes_if_le(XOsiz); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); + buf4 = swap_bytes_if_le(YOsiz); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); + buf4 = swap_bytes_if_le(XTsiz); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); + buf4 = swap_bytes_if_le(YTsiz); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); + buf4 = swap_bytes_if_le(XTOsiz); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); + buf4 = swap_bytes_if_le(YTOsiz); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); + buf2 = swap_bytes_if_le(Csiz); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); for (int c = 0; c < Csiz; ++c) { - buf[0] = cptr[c].SSiz; - buf[1] = cptr[c].XRsiz; - buf[2] = cptr[c].YRsiz; - result &= file->write(&buf, 3) == 3; + buf1 = cptr[c].SSiz; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = cptr[c].XRsiz; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = cptr[c].YRsiz; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); } return result; @@ -681,13 +653,13 @@ namespace ojph { { if (file->read(&Lsiz, 2) != 2) OJPH_ERROR(0x00050041, "error reading SIZ marker"); - Lsiz = swap_byte(Lsiz); + Lsiz = swap_bytes_if_le(Lsiz); int num_comps = (Lsiz - 38) / 3; if (Lsiz != 38 + 3 * num_comps) OJPH_ERROR(0x00050042, "error in SIZ marker length"); if (file->read(&Rsiz, 2) != 2) OJPH_ERROR(0x00050043, "error reading SIZ marker"); - Rsiz = swap_byte(Rsiz); + Rsiz = swap_bytes_if_le(Rsiz); if ((Rsiz & 0x4000) == 0) OJPH_ERROR(0x00050044, "Rsiz bit 14 is not set (this is not a JPH file)"); @@ -695,33 +667,41 @@ namespace ojph { OJPH_WARN(0x00050001, "Rsiz in SIZ has unimplemented fields"); if (file->read(&Xsiz, 4) != 4) OJPH_ERROR(0x00050045, "error reading SIZ marker"); - Xsiz = swap_byte(Xsiz); + Xsiz = swap_bytes_if_le(Xsiz); if (file->read(&Ysiz, 4) != 4) OJPH_ERROR(0x00050046, "error reading SIZ marker"); - Ysiz = swap_byte(Ysiz); + Ysiz = swap_bytes_if_le(Ysiz); ui32 t_XOsiz, t_YOsiz; if (file->read(&t_XOsiz, 4) != 4) OJPH_ERROR(0x00050047, "error reading SIZ marker"); if (file->read(&t_YOsiz, 4) != 4) OJPH_ERROR(0x00050048, "error reading SIZ marker"); - set_image_offset(point(swap_byte(t_XOsiz), swap_byte(t_YOsiz))); + set_image_offset(point( + swap_bytes_if_le(t_XOsiz), + swap_bytes_if_le(t_YOsiz))); ui32 t_XTsiz, t_YTsiz; if (file->read(&t_XTsiz, 4) != 4) OJPH_ERROR(0x00050049, "error reading SIZ marker"); if (file->read(&t_YTsiz, 4) != 4) OJPH_ERROR(0x0005004A, "error reading SIZ marker"); - set_tile_size(size(swap_byte(t_XTsiz), swap_byte(t_YTsiz))); + set_tile_size(size( + swap_bytes_if_le(t_XTsiz), + swap_bytes_if_le(t_YTsiz))); ui32 t_XTOsiz, t_YTOsiz; if (file->read(&t_XTOsiz, 4) != 4) OJPH_ERROR(0x0005004B, "error reading SIZ marker"); if (file->read(&t_YTOsiz, 4) != 4) OJPH_ERROR(0x0005004C, "error reading SIZ marker"); - set_tile_offset(point(swap_byte(t_XTOsiz), swap_byte(t_YTOsiz))); + set_tile_offset(point( + swap_bytes_if_le(t_XTOsiz), + swap_bytes_if_le(t_YTOsiz))); if (file->read(&Csiz, 2) != 2) OJPH_ERROR(0x0005004D, "error reading SIZ marker"); - Csiz = swap_byte(Csiz); + Csiz = swap_bytes_if_le(Csiz); if (Csiz != num_comps) OJPH_ERROR(0x0005004E, "Csiz does not match the SIZ marker size"); + if (Csiz == 0) + OJPH_ERROR(0x0005004F, "Wrong Csiz value of 0 in SIZ marker segment"); set_num_components(Csiz); for (int c = 0; c < Csiz; ++c) { @@ -788,19 +768,20 @@ namespace ojph { //marker size excluding header Lcap = 8; - char buf[4]; + ui16 buf2; + ui32 buf4; bool result = true; - *(ui16*)buf = JP2K_MARKER::CAP; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Lcap); - result &= file->write(&buf, 2) == 2; - *(ui32*)buf = swap_byte(Pcap); - result &= file->write(&buf, 4) == 4; + buf2 = JP2K_MARKER::CAP; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Lcap); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf4 = swap_bytes_if_le(Pcap); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); - *(ui16*)buf = swap_byte(Ccap[0]); - result &= file->write(&buf, 2) == 2; + buf2 = swap_bytes_if_le(Ccap[0]); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); return result; } @@ -810,10 +791,10 @@ namespace ojph { { if (file->read(&Lcap, 2) != 2) OJPH_ERROR(0x00050061, "error reading CAP marker"); - Lcap = swap_byte(Lcap); + Lcap = swap_bytes_if_le(Lcap); if (file->read(&Pcap, 4) != 4) OJPH_ERROR(0x00050062, "error reading CAP marker"); - Pcap = swap_byte(Pcap); + Pcap = swap_bytes_if_le(Pcap); ui32 count = population_count(Pcap); if (Pcap & 0xFFFDFFFF) OJPH_ERROR(0x00050063, @@ -857,34 +838,38 @@ namespace ojph { Lcod = 12; Lcod = (ui16)(Lcod + (Scod & 1 ? 1 + SPcod.num_decomp : 0)); - ui8 buf[4]; + ui8 buf1; + ui16 buf2; bool result = true; - *(ui16*)buf = JP2K_MARKER::COD; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Lcod); - result &= file->write(&buf, 2) == 2; - *(ui8*)buf = Scod; - result &= file->write(&buf, 1) == 1; - *(ui8*)buf = SGCod.prog_order; - result &= file->write(&buf, 1) == 1; - *(ui16*)buf = swap_byte(SGCod.num_layers); - result &= file->write(&buf, 2) == 2; - *(ui8*)buf = SGCod.mc_trans; - result &= file->write(&buf, 1) == 1; - buf[0] = SPcod.num_decomp; - buf[1] = SPcod.block_width; - buf[2] = SPcod.block_height; - buf[3] = SPcod.block_style; - result &= file->write(&buf, 4) == 4; - *(ui8*)buf = SPcod.wavelet_trans; - result &= file->write(&buf, 1) == 1; + buf2 = JP2K_MARKER::COD; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Lcod); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf1 = Scod; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SGCod.prog_order; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf2 = swap_bytes_if_le(SGCod.num_layers); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf1 = SGCod.mc_trans; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.num_decomp; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.block_width; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.block_height; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.block_style; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.wavelet_trans; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); if (Scod & 1) for (int i = 0; i <= SPcod.num_decomp; ++i) { - *(ui8*)buf = SPcod.precinct_size[i]; - result &= file->write(&buf, 1) == 1; + buf1 = SPcod.precinct_size[i]; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); } return result; @@ -914,38 +899,42 @@ namespace ojph { Lcod = num_comps < 257 ? 9 : 10; Lcod = (ui16)(Lcod + (Scod & 1 ? 1 + SPcod.num_decomp : 0)); - ui8 buf[4]; + ui8 buf1; + ui16 buf2; bool result = true; - *(ui16*)buf = JP2K_MARKER::COC; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Lcod); - result &= file->write(&buf, 2) == 2; + buf2 = JP2K_MARKER::COC; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Lcod); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); if (num_comps < 257) { - *(ui8*)buf = (ui8)comp_idx; - result &= file->write(&buf, 1) == 1; + buf1 = (ui8)comp_idx; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); } else { - *(ui16*)buf = swap_byte(comp_idx); - result &= file->write(&buf, 2) == 2; - } - *(ui8*)buf = Scod; - result &= file->write(&buf, 1) == 1; - buf[0] = SPcod.num_decomp; - buf[1] = SPcod.block_width; - buf[2] = SPcod.block_height; - buf[3] = SPcod.block_style; - result &= file->write(&buf, 4) == 4; - *(ui8*)buf = SPcod.wavelet_trans; - result &= file->write(&buf, 1) == 1; + buf2 = swap_bytes_if_le(comp_idx); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + } + buf1 = Scod; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.num_decomp; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.block_width; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.block_height; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.block_style; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); + buf1 = SPcod.wavelet_trans; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); if (Scod & 1) for (int i = 0; i <= SPcod.num_decomp; ++i) { - *(ui8*)buf = SPcod.precinct_size[i]; - result &= file->write(&buf, 1) == 1; + buf1 = SPcod.precinct_size[i]; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); } return result; @@ -958,7 +947,7 @@ namespace ojph { if (file->read(&Lcod, 2) != 2) OJPH_ERROR(0x00050071, "error reading COD segment"); - Lcod = swap_byte(Lcod); + Lcod = swap_bytes_if_le(Lcod); if (file->read(&Scod, 1) != 1) OJPH_ERROR(0x00050072, "error reading COD segment"); if (file->read(&SGCod.prog_order, 1) != 1) @@ -966,7 +955,7 @@ namespace ojph { if (file->read(&SGCod.num_layers, 2) != 2) { OJPH_ERROR(0x00050074, "error reading COD segment"); } else - SGCod.num_layers = swap_byte(SGCod.num_layers); + SGCod.num_layers = swap_bytes_if_le(SGCod.num_layers); if (file->read(&SGCod.mc_trans, 1) != 1) OJPH_ERROR(0x00050075, "error reading COD segment"); if (file->read(&SPcod.num_decomp, 1) != 1) @@ -1022,7 +1011,7 @@ namespace ojph { this->top_cod = top_cod; if (file->read(&Lcod, 2) != 2) OJPH_ERROR(0x00050121, "error reading COC segment"); - Lcod = swap_byte(Lcod); + Lcod = swap_bytes_if_le(Lcod); if (num_comps < 257) { ui8 t; if (file->read(&t, 1) != 1) @@ -1032,7 +1021,7 @@ namespace ojph { else { if (file->read(&comp_idx, 2) != 2) OJPH_ERROR(0x00050123, "error reading COC segment"); - comp_idx = swap_byte(comp_idx); + comp_idx = swap_bytes_if_le(comp_idx); } if (file->read(&Scod, 1) != 1) OJPH_ERROR(0x00050124, "error reading COC segment"); @@ -1146,6 +1135,16 @@ namespace ojph { return p->next; } + ////////////////////////////////////////////////////////////////////////// + param_cod* param_cod::get_or_add_coc(ui32 comp_idx) + { + assert(type == COD_MAIN); + local::param_cod *p = get_coc(comp_idx); + if (p == this) + p = add_coc_object(comp_idx); + return p; + } + ////////////////////////////////////////////////////////////////////////// // // @@ -1160,149 +1159,88 @@ namespace ojph { ui32 num_comps = siz.get_num_components(); trim_non_existing_components(num_comps); - // first check that all the component captured by QCD have the same - // bit_depth and signedness - bool all_same = true; - bool other_comps_exist = false; - ui32 first_comp = 0xFFFF; // an impossible component - { - ui32 num_decompositions = 0; - ui32 bit_depth = 0; - bool is_signed = false; - ui32 wavelet_kern = param_cod::DWT_IRV97; + // initialize QCD based on the first component that is (a) associated with + // COD and (b) does not have a COC, or the first component othewise. - for (ui32 c = 0; c < num_comps; ++c) - { - if (get_qcc(c) == this) // no qcc defined for component c - { - const param_cod *p = cod.get_coc(c); - if (bit_depth == 0) // first component captured by QCD - { - num_decompositions = p->get_num_decompositions(); - bit_depth = siz.get_bit_depth(c); - is_signed = siz.is_signed(c); - wavelet_kern = p->get_wavelet_kern(); - first_comp = c; - } - else - { - all_same = all_same - && (num_decompositions == p->get_num_decompositions()) - && (bit_depth == siz.get_bit_depth(c)) - && (is_signed == siz.is_signed(c)) - && (wavelet_kern == p->get_wavelet_kern()); - } - } - else - other_comps_exist = true; - } - } - - // configure QCD according COD - ui32 qcd_num_decompositions; - ui32 qcd_bit_depth; - bool qcd_is_signed; - ui32 qcd_wavelet_kern; + ui32 qcd_comp = 0; + for (ui32 c = 0; c < num_comps; ++c) { - ui32 qcd_component = first_comp != 0xFFFF ? first_comp : 0; - bool employing_color_transform = cod.is_employing_color_transform(); - qcd_num_decompositions = cod.get_num_decompositions(); - qcd_bit_depth = siz.get_bit_depth(qcd_component); - qcd_is_signed = siz.is_signed(qcd_component); - qcd_wavelet_kern = cod.get_wavelet_kern(); - this->num_subbands = 1 + 3 * qcd_num_decompositions; - if (qcd_wavelet_kern == param_cod::DWT_REV53) - set_rev_quant(qcd_num_decompositions, qcd_bit_depth, - qcd_component < 3 ? employing_color_transform : false); - else if (qcd_wavelet_kern == param_cod::DWT_IRV97) + if (cod.get_coc(c) == &cod && get_qcc(c) == this) { - if (this->base_delta == -1.0f) { - ui32 t = ojph_min(16, qcd_bit_depth); - this->base_delta = 1.0f / (float)(1 << t); - } - set_irrev_quant(qcd_num_decompositions); + qcd_comp = c; + break; } - else - assert(0); } - // if not all the same and captured by QCD, then create QCC for them - if (!all_same) + this->make_quant_steps(qcd_comp, cod, siz); + + // initialize every QCC, creating one for every component that (a) cannot + // use QCD and (b) does not already have a QCC + // NOTE: Qfactor always creates a QCC and QCD cannot be reused + for (ui32 c = 0; c < num_comps; ++c) { - bool employing_color_transform = cod.is_employing_color_transform(); - for (ui32 c = 0; c < num_comps; ++c) + param_qcd *qcc = this->get_qcc(c); + const param_cod *coc = cod.get_coc(c); + + // check if a QCC exists for the component + if (qcc == this) { - const param_cod *cp = cod.get_coc(c); - if (qcd_num_decompositions == cp->get_num_decompositions() - && qcd_bit_depth == siz.get_bit_depth(c) - && qcd_is_signed == siz.is_signed(c) - && qcd_wavelet_kern == cp->get_wavelet_kern()) - continue; // captured by QCD - - // Does not match QCD, must have QCC - param_qcd *qp = get_qcc(c); - if (qp == this) // no QCC was defined, create QCC - qp = this->add_qcc_object(c); - - ui32 num_decompositions = cp->get_num_decompositions(); - qp->num_subbands = 1 + 3 * num_decompositions; - ui32 bit_depth = siz.get_bit_depth(c); - if (cp->get_wavelet_kern() == param_cod::DWT_REV53) - qp->set_rev_quant(num_decompositions, bit_depth, - c < 3 ? employing_color_transform : false); - else if (cp->get_wavelet_kern() == param_cod::DWT_IRV97) - { - if (qp->base_delta == -1.0f) { - if (qcd_wavelet_kern == param_cod::DWT_IRV97) { - assert(this->base_delta != -1.0f); - qp->base_delta = this->base_delta; - } - else { - ui32 t = ojph_min(16, qcd_bit_depth); - qp->base_delta = 1.0f / (float)(1 << t); - } - } - qp->set_irrev_quant(num_decompositions); - } - else - assert(0); + // if none exists, do not create one if QCD can be reused + if (!this->is_qcc_needed(c, *coc, siz)) + continue; + + qcc = this->add_qcc_object(c); + qcc->set_delta(this->base_delta); } + + qcc->make_quant_steps(c, *coc, siz); } - else if (other_comps_exist) // Some are captured by QCD + } + + ////////////////////////////////////////////////////////////////////////// + void param_qcd::make_quant_steps(ui32 comp_num, const param_cod &cod, const param_siz &siz) + { + if (this->is_init) + OJPH_ERROR(0x00040001, "Quantization step sizes already initialized."); + + this->is_init = true; + + this->num_decomps = cod.get_num_decompositions(); + this->bit_depth = siz.get_bit_depth(comp_num); + this->is_signed = siz.is_signed(comp_num); + this->is_color_trans = cod.is_employing_color_transform(); + this->wavelet_kern = cod.get_wavelet_kern(); + this->sampling = siz.get_downsampling(comp_num); + this->num_subbands = 1 + 3 * this->num_decomps; + + if (this->wavelet_kern == param_cod::DWT_REV53) + this->set_rev_quant(this->num_decomps, this->bit_depth, + comp_num < 3 ? this->is_color_trans : false); + else if (this->wavelet_kern == param_cod::DWT_IRV97) { - bool employing_color_transform = cod.is_employing_color_transform(); - for (ui32 c = 0; c < num_comps; ++c) + if (this->base_delta == -1.0f) { - param_qcd *qp = get_qcc(c); - if (qp == this) // if captured by QCD continue - continue; - const param_cod *cp = cod.get_coc(c); - ui32 num_decompositions = cp->get_num_decompositions(); - qp->num_subbands = 1 + 3 * num_decompositions; - ui32 bit_depth = siz.get_bit_depth(c); - if (cp->get_wavelet_kern() == param_cod::DWT_REV53) - qp->set_rev_quant(num_decompositions, bit_depth, - c < 3 ? employing_color_transform : false); - else if (cp->get_wavelet_kern() == param_cod::DWT_IRV97) - { - if (qp->base_delta == -1.0f) { - if (qcd_wavelet_kern == param_cod::DWT_IRV97) { - assert(this->base_delta != -1.0f); - qp->base_delta = this->base_delta; - } - else { - ui32 t = ojph_min(16, qcd_bit_depth); - qp->base_delta = 1.0f / (float)(1 << t); - } - } - qp->set_irrev_quant(num_decompositions); - } - else - assert(0); + ui32 t = ojph_min(16, bit_depth); + this->base_delta = 1.0f / (float)(1 << t); } + this->set_irrev_quant(this->num_decomps); } } + ////////////////////////////////////////////////////////////////////////// + bool param_qcd::is_qcc_needed(ui32 comp_num, const param_cod &cod, const param_siz &siz) + { + if (! this->is_init) + OJPH_ERROR(0x00040001, "Quantization step sizes not initialized."); + + return this->num_decomps != cod.get_num_decompositions() || + this->bit_depth != siz.get_bit_depth(comp_num) || + this->is_signed != siz.is_signed(comp_num) || + this->is_color_trans != cod.is_employing_color_transform() || + this->wavelet_kern != cod.get_wavelet_kern(); + + } + ////////////////////////////////////////////////////////////////////////// void param_qcd::set_delta(ui32 comp_idx, float delta) { @@ -1313,6 +1251,22 @@ namespace ojph { p->set_delta(delta); } + ////////////////////////////////////////////////////////////////////////// + void param_qcd::set_qfactor(ui32 comp_idx, ojph::param_qcd::comp_type ctype, ui8 qfactor) { + if (this->top_qcd != NULL) + OJPH_ERROR(0x00040401, "This method is not implemented for QCC."); + + if (qfactor < 1 || qfactor > 100) + OJPH_ERROR(0x00040403, "Qfactor must be between 1 and 100, but was set to %i.", qfactor); + + param_qcd *p = get_qcc(comp_idx); + if (p == this) + p = add_qcc_object(comp_idx); + + p->qfactor = qfactor; + p->ctype = ctype; + } + ////////////////////////////////////////////////////////////////////////// void param_qcd::set_rev_quant(ui32 num_decomps, ui32 bit_depth, bool is_employing_color_transform) @@ -1365,40 +1319,81 @@ namespace ojph { { int guard_bits = 1; Sqcd = (ui8)((guard_bits<<5)|0x2);//one guard bit, scalar quantization - int s = 0; - float gain_l = sqrt_energy_gains::get_gain_l(num_decomps, false); - float delta_b = base_delta / (gain_l * gain_l); - int exp = 0, mantissa; - while (delta_b < 1.0f) - { exp++; delta_b *= 2.0f; } - //with rounding, there is a risk of becoming equal to 1<<12 - // but that should not happen in reality - mantissa = (int)round(delta_b * (float)(1<<11)) - (1<<11); - mantissa = mantissa < (1<<11) ? mantissa : 0x7FF; - SPqcd.u16[s++] = (ui16)((exp << 11) | mantissa); - for (ui32 d = num_decomps; d > 0; --d) + + // the following are used only when Qfactor is set + float qfactor_power = 0; + float delta_ref = 0; + float G_c = 1; + const open_htj2k::visual_weighting_params vp; + std::vector W_b; + if (this->qfactor != QFACTOR_UNSET) { + const open_htj2k::q_scaling qs = open_htj2k::q_to_delta(this->qfactor, (ui8) this->bit_depth); + qfactor_power = (float) qs.qfactor_power; + const open_htj2k::color_transform ct = open_htj2k::resolve_color_transform(vp, this->is_color_trans); + int comp_index = (int)this->ctype; + delta_ref = (float) (qs.delta_Q * open_htj2k::color_gain(ct, 0)); + G_c = (float) open_htj2k::color_gain(ct, comp_index); + + // chroma_format 0 = 4:4:4, 1 = 4:2:0, 2 = 4:2:2 + int sampling = 0; + if (this->sampling.x == 1 && this->sampling.y == 1) + { sampling = 0; } + else if (this->sampling.x == 2 && this->sampling.y == 2) + { sampling = 1; } + else if (this->sampling.x == 2 && this->sampling.y == 1) + { sampling = 2; } + else + { + OJPH_ERROR(0x00050161, "Qfactor can only be used on components with 4:4:4, 4:2:2 or 4:2:0 sampling"); + } + + if (this->ctype == ojph::param_qcd::OJPH_COMP_Y) + W_b = open_htj2k::luma_visual_weights((ui8) num_decomps, vp); + else + W_b = open_htj2k::chroma_visual_weights((ui8) num_decomps, vp, comp_index, sampling, ct); + + } + + // LL, HL, LH, HH, HL, LH, HH... + for (ui32 s = 0; s < (1 + num_decomps * 3); s++) { - float gain_l = sqrt_energy_gains::get_gain_l(d, false); - float gain_h = sqrt_energy_gains::get_gain_h(d - 1, false); + // compute square root of the enery gain factor W_g + float w_g = 1.0; - delta_b = base_delta / (gain_l * gain_h); + if (num_decomps > 0) + { + //In C++, division result truncates towards zero + ui32 d = num_decomps - (ui32)(((int)s - 1) / 3); + float gain_l = sqrt_energy_gains::get_gain_l(d, false); + float gain_h = sqrt_energy_gains::get_gain_h(d - 1, false); + + if (s == 0) + { w_g = gain_l * gain_l; } + else if ((s - 1) % 3 == 2) + { w_g = gain_h * gain_h; } + else + { w_g = gain_l * gain_h; } + } + + float delta_b; + if (this->qfactor == QFACTOR_UNSET) + { + delta_b = base_delta / w_g; + } + else + { + float w_b = (s == 0 || s > W_b.size()) ? (float) 1.0 : (float) pow(W_b[W_b.size() - s], qfactor_power); + delta_b = delta_ref / (w_g * w_b * G_c); + } int exp = 0, mantissa; while (delta_b < 1.0f) { exp++; delta_b *= 2.0f; } mantissa = (int)round(delta_b * (float)(1<<11)) - (1<<11); + // with rounding, there is a risk that the mantissa becomes + // equal to 1<<11 mantissa = mantissa < (1<<11) ? mantissa : 0x7FF; - SPqcd.u16[s++] = (ui16)((exp << 11) | mantissa); - SPqcd.u16[s++] = (ui16)((exp << 11) | mantissa); - - delta_b = base_delta / (gain_h * gain_h); - - exp = 0; - while (delta_b < 1) - { exp++; delta_b *= 2.0f; } - mantissa = (int)round(delta_b * (float)(1<<11)) - (1<<11); - mantissa = mantissa < (1<<11) ? mantissa : 0x7FF; - SPqcd.u16[s++] = (ui16)((exp << 11) | mantissa); + SPqcd.u16[s] = (ui16)((exp << 11) | mantissa); } } @@ -1579,28 +1574,29 @@ namespace ojph { else assert(0); - char buf[4]; + ui8 buf1; + ui16 buf2; bool result = true; - *(ui16*)buf = JP2K_MARKER::QCD; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Lqcd); - result &= file->write(&buf, 2) == 2; - *(ui8*)buf = Sqcd; - result &= file->write(&buf, 1) == 1; + buf2 = JP2K_MARKER::QCD; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Lqcd); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf1 = Sqcd; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); if (irrev == 0) for (ui32 i = 0; i < num_subbands; ++i) { - *(ui8*)buf = SPqcd.u8[i]; - result &= file->write(&buf, 1) == 1; + buf1 = SPqcd.u8[i]; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); } else if (irrev == 2) for (ui32 i = 0; i < num_subbands; ++i) { - *(ui16*)buf = swap_byte(SPqcd.u16[i]); - result &= file->write(&buf, 2) == 2; + buf2 = swap_bytes_if_le(SPqcd.u16[i]); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); } else assert(0); @@ -1637,37 +1633,38 @@ namespace ojph { else assert(0); - char buf[4]; + ui8 buf1; + ui16 buf2; bool result = true; - *(ui16*)buf = JP2K_MARKER::QCC; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Lqcd); - result &= file->write(&buf, 2) == 2; + buf2 = JP2K_MARKER::QCC; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Lqcd); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); if (num_comps < 257) { - *(ui8*)buf = (ui8)comp_idx; - result &= file->write(&buf, 1) == 1; + buf1 = (ui8)comp_idx; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); } else { - *(ui16*)buf = swap_byte(comp_idx); - result &= file->write(&buf, 2) == 2; + buf2 = swap_bytes_if_le(comp_idx); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); } - *(ui8*)buf = Sqcd; - result &= file->write(&buf, 1) == 1; + buf1 = Sqcd; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); if (irrev == 0) for (ui32 i = 0; i < num_subbands; ++i) { - *(ui8*)buf = SPqcd.u8[i]; - result &= file->write(&buf, 1) == 1; + buf1 = SPqcd.u8[i]; + result &= file->write(&buf1, sizeof(ui8)) == sizeof(ui8); } else if (irrev == 2) for (ui32 i = 0; i < num_subbands; ++i) { - *(ui16*)buf = swap_byte(SPqcd.u16[i]); - result &= file->write(&buf, 2) == 2; + buf2 = swap_bytes_if_le(SPqcd.u16[i]); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); } else assert(0); @@ -1693,7 +1690,7 @@ namespace ojph { { if (file->read(&Lqcd, 2) != 2) OJPH_ERROR(0x00050081, "error reading QCD marker"); - Lqcd = swap_byte(Lqcd); + Lqcd = swap_bytes_if_le(Lqcd); if (file->read(&Sqcd, 1) != 1) OJPH_ERROR(0x00050082, "error reading QCD marker"); if ((Sqcd & 0x1F) == 0) @@ -1728,7 +1725,7 @@ namespace ojph { { if (file->read(&SPqcd.u16[i], 2) != 2) OJPH_ERROR(0x00050087, "error reading QCD marker"); - SPqcd.u16[i] = swap_byte(SPqcd.u16[i]); + SPqcd.u16[i] = swap_bytes_if_le(SPqcd.u16[i]); } } else @@ -1740,7 +1737,7 @@ namespace ojph { { if (file->read(&Lqcd, 2) != 2) OJPH_ERROR(0x000500A1, "error reading QCC marker"); - Lqcd = swap_byte(Lqcd); + Lqcd = swap_bytes_if_le(Lqcd); if (num_comps < 257) { ui8 v; @@ -1752,7 +1749,7 @@ namespace ojph { { if (file->read(&comp_idx, 2) != 2) OJPH_ERROR(0x000500A3, "error reading QCC marker"); - comp_idx = swap_byte(comp_idx); + comp_idx = swap_bytes_if_le(comp_idx); } if (file->read(&Sqcd, 1) != 1) OJPH_ERROR(0x000500A4, "error reading QCC marker"); @@ -1789,7 +1786,7 @@ namespace ojph { { if (file->read(&SPqcd.u16[i], 2) != 2) OJPH_ERROR(0x000500A9, "error reading QCC marker"); - SPqcd.u16[i] = swap_byte(SPqcd.u16[i]); + SPqcd.u16[i] = swap_bytes_if_le(SPqcd.u16[i]); } } else @@ -1974,20 +1971,20 @@ namespace ojph { if (is_any_enabled() == false) return true; - char buf[2]; + ui16 buf2; bool result = true; const param_nlt* p = this; while (p) { if (p->enabled) { - *(ui16*)buf = JP2K_MARKER::NLT; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(p->Lnlt); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(p->Cnlt); - result &= file->write(&buf, 2) == 2; + buf2 = JP2K_MARKER::NLT; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(p->Lnlt); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(p->Cnlt); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); result &= file->write(&p->BDnlt, 1) == 1; result &= file->write(&p->Tnlt, 1) == 1; } @@ -1999,23 +1996,32 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void param_nlt::read(infile_base* file) { - ui8 buf[6]; + ui16 buf2_len; + ui16 buf2_comp; + ui8 buf1_BDnlt; + ui8 buf1_Tnlt; - if (file->read(buf, 6) != 6) + if (file->read(&buf2_len, sizeof(ui16)) != sizeof(ui16)) OJPH_ERROR(0x00050141, "error reading NLT marker segment"); - - ui16 length = swap_byte(*(ui16*)buf); - if (length != 6 || (buf[5] != 3 && buf[5] != 0)) // wrong length or type - OJPH_ERROR(0x00050142, "Unsupported NLT type %d\n", buf[5]); - - ui16 comp = swap_byte(*(ui16*)(buf + 2)); + if (file->read(&buf2_comp, sizeof(ui16)) != sizeof(ui16)) + OJPH_ERROR(0x00050142, "error reading NLT marker segment"); + if (file->read(&buf1_BDnlt, sizeof(ui8)) != sizeof(ui8)) + OJPH_ERROR(0x00050143, "error reading NLT marker segment"); + if (file->read(&buf1_Tnlt, sizeof(ui8)) != sizeof(ui8)) + OJPH_ERROR(0x00050144, "error reading NLT marker segment"); + + ui16 length = swap_bytes_if_le(buf2_len); + if (length != 6 || (buf1_Tnlt != 3 && buf1_Tnlt != 0)) + OJPH_ERROR(0x00050145, "Unsupported NLT type %d\n", buf1_Tnlt); + + ui16 comp = swap_bytes_if_le(buf2_comp); param_nlt* p = get_nlt_object(comp); if (p == NULL) p = add_object(comp); p->enabled = true; p->Cnlt = comp; - p->BDnlt = buf[4]; - p->Tnlt = buf[5]; + p->BDnlt = buf1_BDnlt; + p->Tnlt = buf1_Tnlt; } ////////////////////////////////////////////////////////////////////////// @@ -2095,20 +2101,21 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// bool param_sot::write(outfile_base *file, ui32 payload_len) { - char buf[4]; + ui16 buf2; + ui32 buf4; bool result = true; this->Psot = payload_len + 14; //inc. SOT marker, field & SOD - *(ui16*)buf = JP2K_MARKER::SOT; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Lsot); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Isot); - result &= file->write(&buf, 2) == 2; - *(ui32*)buf = swap_byte(Psot); - result &= file->write(&buf, 4) == 4; + buf2 = JP2K_MARKER::SOT; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Lsot); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Isot); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf4 = swap_bytes_if_le(Psot); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); result &= file->write(&TPsot, 1) == 1; result &= file->write(&TNsot, 1) == 1; @@ -2119,18 +2126,19 @@ namespace ojph { bool param_sot::write(outfile_base *file, ui32 payload_len, ui8 TPsot, ui8 TNsot) { - char buf[4]; + ui32 buf4; + ui16 buf2; bool result = true; - *(ui16*)buf = JP2K_MARKER::SOT; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Lsot); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Isot); - result &= file->write(&buf, 2) == 2; - *(ui32*)buf = swap_byte(payload_len + 14); - result &= file->write(&buf, 4) == 4; + buf2 = JP2K_MARKER::SOT; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Lsot); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Isot); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf4 = swap_bytes_if_le(payload_len + 14); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); result &= file->write(&TPsot, 1) == 1; result &= file->write(&TNsot, 1) == 1; @@ -2148,7 +2156,7 @@ namespace ojph { Lsot = 0; Isot = 0; Psot = 0; TPsot = 0; TNsot = 0; return false; } - Lsot = swap_byte(Lsot); + Lsot = swap_bytes_if_le(Lsot); if (Lsot != 10) { OJPH_INFO(0x00050092, "error in SOT length"); @@ -2161,7 +2169,7 @@ namespace ojph { Lsot = 0; Isot = 0; Psot = 0; TPsot = 0; TNsot = 0; return false; } - Isot = swap_byte(Isot); + Isot = swap_bytes_if_le(Isot); if (Isot == 0xFFFF) { OJPH_INFO(0x00050094, "tile index in SOT marker cannot be 0xFFFF"); @@ -2174,7 +2182,7 @@ namespace ojph { Lsot = 0; Isot = 0; Psot = 0; TPsot = 0; TNsot = 0; return false; } - Psot = swap_byte(Psot); + Psot = swap_bytes_if_le(Psot); if (file->read(&TPsot, 1) != 1) { OJPH_INFO(0x00050096, "error reading SOT marker"); @@ -2192,17 +2200,17 @@ namespace ojph { { if (file->read(&Lsot, 2) != 2) OJPH_ERROR(0x00050091, "error reading SOT marker"); - Lsot = swap_byte(Lsot); + Lsot = swap_bytes_if_le(Lsot); if (Lsot != 10) OJPH_ERROR(0x00050092, "error in SOT length"); if (file->read(&Isot, 2) != 2) OJPH_ERROR(0x00050093, "error reading SOT tile index"); - Isot = swap_byte(Isot); + Isot = swap_bytes_if_le(Isot); if (Isot == 0xFFFF) OJPH_ERROR(0x00050094, "tile index in SOT marker cannot be 0xFFFF"); if (file->read(&Psot, 4) != 4) OJPH_ERROR(0x00050095, "error reading SOT marker"); - Psot = swap_byte(Psot); + Psot = swap_bytes_if_le(Psot); if (file->read(&TPsot, 1) != 1) OJPH_ERROR(0x00050096, "error reading SOT marker"); if (file->read(&TNsot, 1) != 1) @@ -2248,22 +2256,23 @@ namespace ojph { bool param_tlm::write(outfile_base *file) { assert(next_pair_index == num_pairs); - char buf[4]; + ui16 buf2; + ui32 buf4; bool result = true; - *(ui16*)buf = JP2K_MARKER::TLM; - *(ui16*)buf = swap_byte(*(ui16*)buf); - result &= file->write(&buf, 2) == 2; - *(ui16*)buf = swap_byte(Ltlm); - result &= file->write(&buf, 2) == 2; + buf2 = JP2K_MARKER::TLM; + buf2 = swap_bytes_if_le(buf2); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf2 = swap_bytes_if_le(Ltlm); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); result &= file->write(&Ztlm, 1) == 1; result &= file->write(&Stlm, 1) == 1; for (ui32 i = 0; i < num_pairs; ++i) { - *(ui16*)buf = swap_byte(pairs[i].Ttlm); - result &= file->write(&buf, 2) == 2; - *(ui32*)buf = swap_byte(pairs[i].Ptlm); - result &= file->write(&buf, 4) == 4; + buf2 = swap_bytes_if_le(pairs[i].Ttlm); + result &= file->write(&buf2, sizeof(ui16)) == sizeof(ui16); + buf4 = swap_bytes_if_le(pairs[i].Ptlm); + result &= file->write(&buf4, sizeof(ui32)) == sizeof(ui32); } return result; } @@ -2363,16 +2372,19 @@ namespace ojph { if (file->read(&Ldfs, 2) != 2) OJPH_ERROR(0x000500D1, "error reading DFS-Ldfs parameter"); - Ldfs = swap_byte(Ldfs); + Ldfs = swap_bytes_if_le(Ldfs); if (file->read(&Sdfs, 2) != 2) OJPH_ERROR(0x000500D2, "error reading DFS-Sdfs parameter"); - Sdfs = swap_byte(Sdfs); + Sdfs = swap_bytes_if_le(Sdfs); if (Sdfs > 15) OJPH_ERROR(0x000500D3, "The DFS-Sdfs parameter is %d, which is " "larger than the permissible 15", Sdfs); ui8 t, l_Ids = 0; if (file->read(&l_Ids, 1) != 1) OJPH_ERROR(0x000500D4, "error reading DFS-Ids parameter"); + if (l_Ids == 0) + OJPH_ERROR(0x000500D8, + "The value of the Ids member in the DFS marker segment cannot be 0"); constexpr int max_Ddfs = sizeof(Ddfs) * 4; if (l_Ids > max_Ddfs) OJPH_INFO(0x000500D5, "The DFS-Ids parameter is %d; while this is " @@ -2444,27 +2456,25 @@ namespace ojph { ui16 v; if (file->read(&v, 2) != 2) return false; bytes -= 2; - K = swap_byte(v); + K = swap_bytes_if_le(v); } else if (coeff_type == 2) { // float - union { - float f; - ui32 i; - } v; - if (file->read(&v.i, 4) != 4) return false; + ui32 i; + if (file->read(&i, sizeof(ui32)) != sizeof(ui32)) return false; bytes -= 4; - v.i = swap_byte(v.i); - K = v.f; + i = swap_bytes_if_le(i); + float f; + memcpy(&f, &i, sizeof(float)); + K = f; } else if (coeff_type == 3) { // double - union { - double d; - ui64 i; - } v; - if (file->read(&v.i, 8) != 8) return false; + ui64 i; + if (file->read(&i, sizeof(ui64)) != sizeof(ui64)) return false; bytes -= 8; - v.i = swap_byte(v.i); - K = (float)v.d; + i = swap_bytes_if_le(i); + double d; + memcpy(&d, &i, sizeof(double)); + K = (float)d; } else if (coeff_type == 4) { // 128 bit float ui64 v, v1; @@ -2472,12 +2482,8 @@ namespace ojph { bytes -= 8; if (file->read(&v1, 8) != 8) return false; // v1 not needed bytes -= 8; - v = swap_byte(v); + v = swap_bytes_if_le(v); - union { - float f; - ui32 i; - } s; // convert the MSB of 128b float to 32b float // 32b float has 1 sign bit, 8 exponent (offset 127), 23 mantissa // 128b float has 1 sign bit, 15 exponent (offset 16383), 112 mantissa @@ -2486,11 +2492,13 @@ namespace ojph { e += 127; e = e & 0xFF; // removes MSBs if negative e <<= 23; // move bits to their location - s.i = 0; - s.i |= ((ui32)(v >> 32) & 0x80000000); // copy sign bit - s.i |= (ui32)e; // copy exponent - s.i |= (ui32)((v >> 25) & 0x007FFFFF); // copy 23 mantissa - K = s.f; + ui32 i = 0; + i |= ((ui32)(v >> 32) & 0x80000000); // copy sign bit + i |= (ui32)e; // copy exponent + i |= (ui32)((v >> 25) & 0x007FFFFF); // copy 23 mantissa + float f; + memcpy(&f, &i, sizeof(float)); + K = f; } return true; } @@ -2510,7 +2518,7 @@ namespace ojph { si16 v; if (file->read(&v, 2) != 2) return false; bytes -= 2; - K = (si16)swap_byte((ui16)v); + K = (si16)swap_bytes_if_le((ui16)v); } else return false; @@ -2525,13 +2533,13 @@ namespace ojph { if (file->read(&Latk, 2) != 2) OJPH_ERROR(0x000500E1, "error reading ATK-Latk parameter"); - Latk = swap_byte(Latk); + Latk = swap_bytes_if_le(Latk); si32 bytes = Latk - 2; ojph::ui16 temp_Satk; if (file->read(&temp_Satk, 2) != 2) OJPH_ERROR(0x000500E2, "error reading ATK-Satk parameter"); bytes -= 2; - temp_Satk = swap_byte(temp_Satk); + temp_Satk = swap_bytes_if_le(temp_Satk); int tmp_idx = temp_Satk & 0xFF; if ((top_atk && top_atk->get_atk(tmp_idx) != NULL) || tmp_idx == 0 || tmp_idx == 1) @@ -2577,7 +2585,7 @@ namespace ojph { if (file->read(&d[s].rev.Batk, 2) != 2) OJPH_ERROR(0x000500EA, "error reading ATK-Batk parameter"); bytes -= 2; - d[s].rev.Batk = (si16)swap_byte((ui16)d[s].rev.Batk); + d[s].rev.Batk = (si16)swap_bytes_if_le((ui16)d[s].rev.Batk); ui8 LCatk; if (file->read(&LCatk, 1) != 1) OJPH_ERROR(0x000500EB, "error reading ATK-LCatk parameter"); @@ -2649,7 +2657,7 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// param_atk* param_atk::add_object() { - assert(top_atk = NULL); + assert(top_atk == NULL); param_atk *p = this; while (p->next != NULL) p = p->next; diff --git a/external/OpenJPH/src/core/codestream/ojph_params_local.h b/external/OpenJPH/src/core/codestream/ojph_params_local.h index fd5e22563..40f411731 100644 --- a/external/OpenJPH/src/core/codestream/ojph_params_local.h +++ b/external/OpenJPH/src/core/codestream/ojph_params_local.h @@ -602,6 +602,9 @@ namespace ojph { //////////////////////////////////////// param_cod* add_coc_object(ui32 comp_idx); + /////////////////////////////////////// + param_cod* get_or_add_coc(ui32 comp_idx); + //////////////////////////////////////// const param_atk* access_atk() const { return atk; } @@ -629,8 +632,15 @@ namespace ojph { type = top_cod ? COC_MAIN : COD_MAIN; Lcod = 0; Scod = 0; + SPcod = cod_SPcod(); // SPcod is initialized to default values next = NULL; atk = NULL; + // For COC marker segment: + // Lcod will be initialized on writing the marker segment to disk + // Scod is initialized to 0 + // SGcod does not exist in COC + // SPcoc is initialized to default values + // atk is initialized to NULL this->top_cod = top_cod; this->comp_idx = comp_idx; } @@ -691,6 +701,11 @@ namespace ojph { QCC_TILE = 4 // not implemented }; + //////////////////////////////////////// + enum qfactor_const : ui8 { + QFACTOR_UNSET = 0 + }; + public: param_qcd(param_qcd* top_qcd = NULL, ui16 comp_idx = OJPH_QCD_DEFAULT) { avail = NULL; init(top_qcd, comp_idx); } @@ -707,8 +722,11 @@ namespace ojph { } void check_validity(const param_siz& siz, const param_cod& cod); + void make_quant_steps(ui32 comp_num, const param_cod &cod, const param_siz &siz); + bool is_qcc_needed(ui32 comp_num, const param_cod &cod, const param_siz &siz); void set_delta(float delta) { base_delta = delta; } void set_delta(ui32 comp_idx, float delta); + void set_qfactor(ui32 comp_idx, ojph::param_qcd::comp_type ctype, ui8 qfactor); ui32 get_num_guard_bits() const; ui32 get_MAGB() const; ui32 get_Kmax(const param_dfs* dfs, ui32 num_decompositions, @@ -731,12 +749,16 @@ namespace ojph { //////////////////////////////////////// void init(param_qcd* top_qcd, ui16 comp_idx) { + is_init = false; type = top_qcd ? QCC_MAIN : QCD_MAIN; Lqcd = 0; Sqcd = 0; memset(&SPqcd, 0, sizeof(SPqcd)); num_subbands = 0; base_delta = -1.0f; + qfactor = QFACTOR_UNSET; + ctype = ojph::param_qcd::OJPH_COMP_Y; + sampling = ojph::point(1, 1); enabled = true; next = NULL; this->top_qcd = top_qcd; @@ -769,6 +791,7 @@ namespace ojph { private: // QCD variables qcd_type type; + bool is_init; // have the quantization steps been generated ui16 Lqcd; ui8 Sqcd; union @@ -777,12 +800,22 @@ namespace ojph { ui16 u16[97]; } SPqcd; ui32 num_subbands; // number of subbands - float base_delta; // base quantization step size -- all other - // step sizes are derived from it. bool enabled; // enabled if two, and ignored if false param_qcd *next; // pointer to create chains of qcc marker segments param_qcd *top_qcd; // pointer to the top QCD (this is the default) + // variables used to generate the quantization step sizes + float base_delta; // base quantization step size -- all other + // step sizes are derived from it. + ui8 qfactor; + ojph::param_qcd::comp_type ctype; + bool is_color_trans; + ui32 num_decomps; + ui32 bit_depth; + bool is_signed; + ui32 wavelet_kern; + ojph::point sampling; + private: // QCC only variables ui16 comp_idx; diff --git a/external/OpenJPH/src/core/codestream/ojph_tile.cpp b/external/OpenJPH/src/core/codestream/ojph_tile.cpp index df95d274f..b866fd4c9 100644 --- a/external/OpenJPH/src/core/codestream/ojph_tile.cpp +++ b/external/OpenJPH/src/core/codestream/ojph_tile.cpp @@ -110,6 +110,7 @@ namespace ojph { ui32 recon_ty1 = recon_tile_rect.org.y + recon_tile_rect.siz.h; ui32 width = 0; + rect colour_comp_rect[3]; for (ui32 i = 0; i < num_comps; ++i) { point downsamp = szp->get_downsampling(i); @@ -129,6 +130,9 @@ namespace ojph { comp_rect.siz.w = tcx1 - tcx0; comp_rect.siz.h = tcy1 - tcy0; + if (i < 3) + colour_comp_rect[i] = comp_rect; + rect recon_comp_rect; recon_comp_rect.org.x = recon_tcx0; recon_comp_rect.org.y = recon_tcy0; @@ -147,7 +151,7 @@ namespace ojph { for (ui32 i = 0; i < 3; ++i) reversible[i] = codestream->get_coc(i)->is_reversible(); if (reversible[0] != reversible[1] || reversible[1] != reversible[2]) - OJPH_ERROR(0x000300A2, "When the colour transform is employed. " + OJPH_ERROR(0x000300A2, "When the colour transform is employed, " "all colour components must undergo either reversible or " "irreversible wavelet transform; if not, then it is not clear " "what colour transform should be used (reversible or " @@ -157,6 +161,21 @@ namespace ojph { reversible[1] ? "reversible" : "irreversible", reversible[2] ? "reversible" : "irreversible"); + if (colour_comp_rect[0] != colour_comp_rect[1] || + colour_comp_rect[1] != colour_comp_rect[2]) + OJPH_ERROR(0x000300A3, "When the colour transform is employed, " + "the first three colour components must have the same rectangle; " + "i.e., the same origin on the canvas and the same width and " + "height. The first three components have the following " + "origin-size (x,y)-(w,h) values. Component 0 (%d,%d)-(%d,%d), " + "Component 1 (%d,%d)-(%d,%d), Component 2 (%d,%d)-(%d,%d)", + colour_comp_rect[0].org.x, colour_comp_rect[0].org.y, + colour_comp_rect[0].siz.w, colour_comp_rect[0].siz.h, + colour_comp_rect[1].org.x, colour_comp_rect[1].org.y, + colour_comp_rect[1].siz.w, colour_comp_rect[1].siz.h, + colour_comp_rect[2].org.x, colour_comp_rect[2].org.y, + colour_comp_rect[2].siz.w, colour_comp_rect[2].siz.h); + allocator->pre_alloc_obj(3); if (reversible[0]) for (int i = 0; i < 3; ++i) @@ -274,8 +293,8 @@ namespace ojph { OJPH_ERROR(0x000300A1, "Mismatch between Ssiz (bit_depth = %d, " "is_signed = %s) from SIZ marker segment, and BDnlt " "(bit_depth = %d, is_signed = %s) from NLT marker segment, " - "for component %d", i, num_bits[i], - is_signed[i] ? "True" : "False", bd, is ? "True" : "False"); + "for component %d", num_bits[i], + is_signed[i] ? "True" : "False", bd, is ? "True" : "False", i); if (result == false) nlt_type3[i] = param_nlt::nonlinearity::OJPH_NLT_NO_NLT; cur_line[i] = 0; @@ -575,7 +594,7 @@ namespace ojph { OJPH_ERROR(0x00030081, "Error writing to file"); //write start of data - ui16 t = swap_byte(JP2K_MARKER::SOD); + ui16 t = swap_bytes_if_le((ui16)JP2K_MARKER::SOD); if (!file->write(&t, 2)) OJPH_ERROR(0x00030082, "Error writing to file"); } @@ -603,7 +622,7 @@ namespace ojph { OJPH_ERROR(0x00030083, "Error writing to file"); //write start of data - ui16 t = swap_byte(JP2K_MARKER::SOD); + ui16 t = swap_bytes_if_le((ui16)JP2K_MARKER::SOD); if (!file->write(&t, 2)) OJPH_ERROR(0x00030084, "Error writing to file"); @@ -623,7 +642,7 @@ namespace ojph { (ui8)(c + r * num_comps), (ui8)num_tileparts)) OJPH_ERROR(0x00030085, "Error writing to file"); //write start of data - ui16 t = swap_byte(JP2K_MARKER::SOD); + ui16 t = swap_bytes_if_le((ui16)JP2K_MARKER::SOD); if (!file->write(&t, 2)) OJPH_ERROR(0x00030086, "Error writing to file"); comps[c].write_precincts(r, file); @@ -644,7 +663,7 @@ namespace ojph { OJPH_ERROR(0x00030087, "Error writing to file"); //write start of data - ui16 t = swap_byte(JP2K_MARKER::SOD); + ui16 t = swap_bytes_if_le((ui16)JP2K_MARKER::SOD); if (!file->write(&t, 2)) OJPH_ERROR(0x00030088, "Error writing to file"); } @@ -719,7 +738,7 @@ namespace ojph { OJPH_ERROR(0x0003008A, "Error writing to file"); //write start of data - ui16 t = swap_byte(JP2K_MARKER::SOD); + ui16 t = swap_bytes_if_le((ui16)JP2K_MARKER::SOD); if (!file->write(&t, 2)) OJPH_ERROR(0x0003008B, "Error writing to file"); } diff --git a/external/OpenJPH/src/core/codestream/ojph_visual_weighting.h b/external/OpenJPH/src/core/codestream/ojph_visual_weighting.h new file mode 100644 index 000000000..8533b5b8d --- /dev/null +++ b/external/OpenJPH/src/core/codestream/ojph_visual_weighting.h @@ -0,0 +1,336 @@ +// Copyright (c) 2019 - 2026, Osamu Watanabe +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +// --------------------------------------------------------------------------- +// Analytic per-subband visual (CSF) weighting for Qfactor quantization. +// +// PROTOTYPE: the default model is `legacy_table`, which returns the exact +// hard-coded Zeng et al. (Signal Processing: Image Communication 17, 2002, +// Table 2) weights that the Qfactor path has always used -- so the emitted +// QCD/QCC bytes are bit-identical unless an analytic model is explicitly +// requested. The analytic models replace that single-viewing-condition table +// with a contrast-sensitivity-function (CSF) evaluated at each subband's +// radial frequency, so the weighting follows an arbitrary viewing distance / +// zoom. See scripts/csf_fit_luma.py for the fit that calibrates `ref_ppd`: +// * mannos_sakrison reproduces W_b_Y to RMS ~0.029 at ref_ppd ~= 72 +// * daly reproduces W_b_Y near the paper's ~1700 px viewing +// distance but with a looser shape (RMS ~0.056) +// Header-only and dependency-free (only //) so it can +// be exercised by a standalone demo as well as by the marker code. +// --------------------------------------------------------------------------- + +#include +#include +#include + +namespace open_htj2k { + +enum class csf_model { + legacy_table, // exact Zeng Table 2 weights (default; bit-identical output) + mannos_sakrison, // analytic Mannos-Sakrison CSF + daly // analytic Daly CSF (light-adaptation form) +}; + +// Color decorrelation actually in force, which drives BOTH the per-component +// synthesis gain and whether a QCC component is treated as chroma (opponent) or +// luma (e.g. undecorrelated RGB). Quantization error in component c is amplified +// by ||column_c(MCT^-1)||_2 in reconstructed (R,G,B) space, so the step is scaled +// by 1/G_c. With no MCT every component is independent (unit gain, luma role). +enum class color_transform { + ict, // irreversible 9/7 MCT (YCbCr): ICT inverse column-norm gains, chroma roles + none // no MCT: unit gains, luma role for every component (RGB / generic) +}; + +// Viewing condition for analytic weighting. Defaults select the legacy table, +// so a default-constructed value reproduces the historical Qfactor output. +struct visual_weighting_params { + csf_model model = csf_model::legacy_table; + // Reference pixels-per-degree at zoom 1.0. Calibrated so that + // mannos_sakrison reproduces the legacy W_b_Y table (see header note). + double ref_ppd = 72.0; + // Display magnification. zoom > 1 (zoom-in) lowers the effective ppd, shifts + // every subband to a lower cycles/degree, and flattens the weighting toward + // 1.0 (i.e. toward flat MSE-optimal quantization) -- the correct limit when + // the viewer is close / heavily magnified. + double zoom = 1.0; + // Radial-frequency multiplier for the diagonal (HH) subband relative to the + // horizontal/vertical (LH/HL) center. Geometric value is sqrt(2); the legacy + // table behaves closer to ~1.25 (no oblique-effect penalty). + double hh_factor = 1.4142135623730951; +}; + +// --- color synthesis gain -------------------------------------------------- +// +// G_c = ||column c of the inverse decorrelating transform||_2. Quantization +// error in component c is amplified by this in reconstructed (R,G,B) space, so +// the step is scaled by 1/G_c. For a future Part-2 custom MCT this would be a +// column norm of the actual inverse matrix; the built-in ICT path uses the +// historical 4-dp literals so the emitted QCD/QCC stays bit-identical. + +// Per-component synthesis gain. `none` (no MCT) -> unit gain (independent +// components). `ict` -> the inverse-ICT column norms {sqrt(3), 1.80511, 1.57340} +// rounded to 4 dp exactly as the legacy encoder stored them. +inline double color_gain(color_transform ct, int comp_index) { + if (ct == color_transform::none) return 1.0; + static const double g[3] = {1.7321, 1.8051, 1.5734}; + return g[(comp_index >= 0 && comp_index < 3) ? comp_index : 0]; +} + +// Resolve the effective transform for color scaling. Legacy mode reproduces the +// historical encoder, which assumed ICT regardless of the real MCT; analytic +// modes honor the transform actually applied (identity when the MCT is off). +inline color_transform resolve_color_transform(const visual_weighting_params &vp, bool mct_on) { + if (vp.model == csf_model::legacy_table) return color_transform::ict; + return mct_on ? color_transform::ict : color_transform::none; +} + +// --- Qfactor -> base quantization scale ------------------------------------ +// +// Maps the 1..100 Qfactor to the reference step delta and the exponent applied +// to the per-subband visual weights. Shared verbatim by QCD, QCC, and +// estimate_qfactor so all three invert to identical steps (guarded by the +// qfest_* round-trip tests). RI is the component dynamic range in bits. +struct q_scaling { + double delta_Q; // reference step before basis / visual / color normalization + double qfactor_power; // exponent applied to each subband's visual weight +}; +inline q_scaling q_to_delta(uint8_t qfactor, uint8_t RI) { + const uint8_t t0 = 65, t1 = 97; + const double alpha_T0 = 0.04, alpha_T1 = 0.10; + const double M_T0 = 2.0 * (1.0 - t0 / 100.0); + const double M_T1 = 2.0 * (1.0 - t1 / 100.0); + const double M_Q = (qfactor < 50) ? 50.0 / qfactor : 2.0 * (1.0 - qfactor / 100.0); + double alpha_Q = alpha_T0; + double qfactor_power = 1.0; + if (qfactor >= t1) { + qfactor_power = 0.0; + alpha_Q = alpha_T1; + } else if (qfactor > t0) { + qfactor_power = (std::log(M_T1) - std::log(M_Q)) / (std::log(M_T1) - std::log(M_T0)); + alpha_Q = alpha_T1 * std::pow(alpha_T0 / alpha_T1, qfactor_power); + } + // eps0 = sqrt(1/2) / 2^RI. Use ldexp rather than `1 << RI`: bit-identical for + // the encoder's RI (<= 16 bpp) but well-defined for any RI, so a crafted + // high-bit-depth file fed to estimate_qfactor cannot trigger signed-shift UB. + const double eps0 = std::sqrt(0.5) * std::ldexp(1.0, -static_cast(RI)); + return {alpha_Q * M_Q + eps0, qfactor_power}; +} + +// --- CSF models (spatial frequency f in cycles/degree) --------------------- + +inline double mannos_sakrison_csf(double f) { + return 2.6 * (0.0192 + 0.114 * f) * std::exp(-std::pow(0.114 * f, 1.1)); +} + +inline double daly_csf(double f) { + // Daly 1993 CSF core, light-adaptation form at L = 100 cd/m^2. The low- + // frequency image-area term is omitted because the low side is clamped flat. + constexpr double L = 100.0, eps = 0.9; + const double A = 0.801 * std::pow(1.0 + 0.7 / L, -0.2); + const double B = 0.3 * std::pow(1.0 + 100.0 / L, 0.15); + const double x = B * eps * f; + return A * eps * f * std::exp(-x) * std::sqrt(1.0 + 0.06 * std::exp(x)); +} + +inline double csf_value(double f, csf_model m) { + return (m == csf_model::daly) ? daly_csf(f) : mannos_sakrison_csf(f); +} + +// Guard a visual weight against extreme viewing conditions (very large effective +// ppd) where the CSF underflows to 0 or Daly's exp() overflows to NaN. Either +// would make the step `delta/(basis*w*G)` divide by zero / propagate NaN and +// emit a corrupt QCD/QCC. Replaces only non-finite or non-positive values, so it +// is a no-op for every realistic weight (bit-identical normal output). +inline double finite_weight(double w) { + return (std::isfinite(w) && w > 0.0) ? w : 1e-300; +} + +// Peak (frequency, amplitude) of a CSF, found once by a coarse scan. Used to +// normalize the weight to <= 1 and to clamp the band-pass low side to flat. +struct csf_peak_t { + double f_peak; + double h_peak; +}; + +inline csf_peak_t csf_peak(csf_model m) { + auto scan = [](csf_model mm) { + csf_peak_t pk{0.0, 0.0}; + for (int i = 1; i < 100000; ++i) { + const double f = i * 0.002; // 0.002 .. 200 cpd + const double v = csf_value(f, mm); + if (v > pk.h_peak) { + pk.h_peak = v; + pk.f_peak = f; + } + } + return pk; + }; + // Peak depends only on the model -> compute each once (thread-safe magic statics). + static const csf_peak_t mannos_pk = scan(csf_model::mannos_sakrison); + static const csf_peak_t daly_pk = scan(csf_model::daly); + return (m == csf_model::daly) ? daly_pk : mannos_pk; +} + +// Normalized CSF weight: flat (= 1) at/below the peak, falling CSF above it. +inline double csf_weight(double f, csf_model m, const csf_peak_t &pk) { + if (f <= pk.f_peak) return 1.0; + return finite_weight(csf_value(f, m) / pk.h_peak); +} + +// Square-root-domain luminance visual weights, one per detail subband, in the +// QCD/wmse build order (per level, finest first): [HH_l, LH_l, HL_l]. Length +// is 3 * dwt_levels for analytic models; the LL band is handled by the caller +// (weight 1.0). In legacy mode the historical 15-entry table is returned +// verbatim, so the caller's existing out-of-range guard still applies. +inline std::vector luma_visual_weights(uint8_t dwt_levels, + const visual_weighting_params &vp) { + if (vp.model == csf_model::legacy_table) { + // Zeng et al., Table 2, Y column (= sqrt of the MSE-domain weights). + return {0.0901, 0.2758, 0.2758, 0.7018, 0.8378, 0.8378, 1.0000, 1.0000, + 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000}; + } + + const csf_peak_t pk = csf_peak(vp.model); + const double zoom = (vp.zoom > 0.0) ? vp.zoom : 1.0; + const double ppd = vp.ref_ppd / zoom; // zoom-in lowers effective ppd + const double f_N = ppd / 2.0; // Nyquist in cycles/degree + + std::vector w; + w.reserve(static_cast(3) * dwt_levels); + for (uint8_t lvl = 1; lvl <= dwt_levels; ++lvl) { + // Geometric-mean radial center of octave band [f_N/2^lvl, f_N/2^(lvl-1)]. + const double f_r = f_N * std::pow(2.0, -static_cast(lvl)) * std::sqrt(2.0); + const double f_hh = f_r * vp.hh_factor; + w.push_back(csf_weight(f_hh, vp.model, pk)); // HH + w.push_back(csf_weight(f_r, vp.model, pk)); // LH + w.push_back(csf_weight(f_r, vp.model, pk)); // HL + } + return w; +} + +// --- chroma (chrominance) analytic weighting ------------------------------- +// +// Chrominance CSF is low-pass (not band-pass like luminance) and rolls off well +// below the luminance CSF. A single per-channel CSF, evaluated at each subband's +// per-axis radial frequency, reproduces ALL of the historical 4:4:4 / 4:2:0 / +// 4:2:2 QCC tables: chroma subsampling is just a frequency shift (sx, sy) +// applied to the same CSF, which also yields the 4:2:2 LH != HL anisotropy for +// free. Parameters below were fit to the 4:4:4 Cb/Cr tables at ref_ppd = 72 +// (see scripts/csf_fit_chroma.py): Cb RMS 0.010, Cr RMS 0.021; predicted 4:2:0 / 4:2:2 +// RMS <= 0.064. The chroma CSF is independent of vp.model's luminance choice. + +// Stretched-exponential low-pass chroma CSF (= 1 at DC). f in cycles/degree. +inline double chroma_csf(double f, double a, double b) { return std::exp(-std::pow(a * f, b)); } + +// Calibrated chroma CSF parameters per opponent channel (at ref_ppd = 72). +struct chroma_csf_params { + double a, b; +}; +inline chroma_csf_params chroma_params_for(int comp_index) { + // comp_index 1 = Cb (blue-yellow, steeper roll-off); otherwise Cr (red-green). + return (comp_index == 1) ? chroma_csf_params{0.1173, 0.840} : chroma_csf_params{0.0699, 1.050}; +} + +// Exact historical QCC table row (sqrt-domain). comp_index 1 = Cb, else Cr; +// chroma_format 0 = 4:4:4, 1 = 4:2:0, 2 = 4:2:2 (matches j2kmarkers YCC*). +inline std::vector legacy_chroma_row(int comp_index, int chroma_format) { + static const double Cb444[15] = {0.0263, 0.0863, 0.0863, 0.1362, 0.2564, 0.2564, 0.3346, 0.4691, + 0.4691, 0.5444, 0.6523, 0.6523, 0.7078, 0.7797, 0.7797}; + static const double Cr444[15] = {0.0773, 0.1835, 0.1835, 0.2598, 0.4130, 0.4130, 0.5040, 0.6464, + 0.6464, 0.7220, 0.8254, 0.8254, 0.8769, 0.9424, 0.9424}; + static const double Cb420[15] = {0.1362, 0.2564, 0.2564, 0.3346, 0.4691, 0.4691, 0.5444, 0.6523, + 0.6523, 0.7078, 0.7797, 0.7797, 1.0000, 1.0000, 1.0000}; + static const double Cr420[15] = {0.2598, 0.4130, 0.4130, 0.5040, 0.6464, 0.6464, 0.7220, 0.8254, + 0.8254, 0.8769, 0.9424, 0.9424, 1.0000, 1.0000, 1.0000}; + static const double Cb422[15] = {0.0863, 0.0863, 0.2564, 0.2564, 0.2564, 0.4691, 0.4691, 0.4691, + 0.6523, 0.6523, 0.6523, 0.7797, 0.7797, 0.7797, 1.0000}; + static const double Cr422[15] = {0.1835, 0.1835, 0.4130, 0.4130, 0.4130, 0.6464, 0.6464, 0.6464, + 0.8254, 0.8254, 0.8254, 0.9424, 0.9424, 0.9424, 1.0000}; + const double *r; + switch (chroma_format) { + case 1: r = (comp_index == 1) ? Cb420 : Cr420; break; // 4:2:0 + case 2: r = (comp_index == 1) ? Cb422 : Cr422; break; // 4:2:2 + default: r = (comp_index == 1) ? Cb444 : Cr444; break; // 4:4:4 + } + return std::vector(r, r + 15); +} + +// Square-root-domain chroma visual weights, one per detail subband, in QCC order +// (per level, finest first): [HH_l, LH_l, HL_l]. legacy_table returns the +// historical row verbatim (bit-identical). Analytic models fold chroma +// subsampling into the effective horizontal/vertical ppd, so LH (vertical +// detail) and HL (horizontal detail) diverge under 4:2:2 as they should. +inline std::vector chroma_visual_weights(uint8_t dwt_levels, + const visual_weighting_params &vp, int comp_index, + int chroma_format, + color_transform ct = color_transform::ict) { + if (vp.model == csf_model::legacy_table) { + return legacy_chroma_row(comp_index, chroma_format); + } + // Without a luma/chroma decorrelating transform this component carries + // luminance (e.g. a raw RGB channel), so it takes the luminance CSF, never + // the chroma roll-off. Such components are not subsampled (full resolution). + if (ct == color_transform::none) { + return luma_visual_weights(dwt_levels, vp); + } + + double sx = 1.0, sy = 1.0; // horizontal/vertical chroma subsampling factors + if (chroma_format == 1) { // 4:2:0 + sx = 2.0; + sy = 2.0; + } else if (chroma_format == 2) { // 4:2:2 + sx = 2.0; + sy = 1.0; + } + + const chroma_csf_params cp = chroma_params_for(comp_index); + const double zoom = (vp.zoom > 0.0) ? vp.zoom : 1.0; + const double f_N = (vp.ref_ppd / zoom) / 2.0; // luma Nyquist (cpd) + const double f_Nx = f_N / sx; // chroma horizontal Nyquist + const double f_Ny = f_N / sy; // chroma vertical Nyquist + + std::vector w; + w.reserve(static_cast(3) * dwt_levels); + for (uint8_t lvl = 1; lvl <= dwt_levels; ++lvl) { + const double dx = f_Nx * std::pow(2.0, -static_cast(lvl)) * std::sqrt(2.0); + const double dy = f_Ny * std::pow(2.0, -static_cast(lvl)) * std::sqrt(2.0); + // HH radial = geometric per-axis diagonal, scaled by hh_factor relative to the + // isotropic sqrt(2) so the same knob tunes luma and chroma (default sqrt(2) = no change). + const double f_hh = (vp.hh_factor / std::sqrt(2.0)) * std::sqrt(dx * dx + dy * dy); + // chroma_csf = exp(-(a*f)^b) is <= 1 by construction; finite_weight only + // guards the underflow-to-0 case at extreme effective ppd. + w.push_back(finite_weight(chroma_csf(f_hh, cp.a, cp.b))); // HH + w.push_back(finite_weight(chroma_csf(dy, cp.a, cp.b))); // LH (vertical detail) + w.push_back(finite_weight(chroma_csf(dx, cp.a, cp.b))); // HL (horizontal detail) + } + return w; +} + +} // namespace open_htj2k diff --git a/external/OpenJPH/src/core/coding/ojph_block_common.cpp b/external/OpenJPH/src/core/coding/ojph_block_common.cpp index 2ba138a98..a3dd93f32 100644 --- a/external/OpenJPH/src/core/coding/ojph_block_common.cpp +++ b/external/OpenJPH/src/core/coding/ojph_block_common.cpp @@ -5,7 +5,8 @@ // Copyright (c) 2022, Aous Naman // Copyright (c) 2022, Kakadu Software Pty Ltd, Australia // Copyright (c) 2022, The University of New South Wales, Australia -// +// Copyright (c) 2026, Osamu Watanabe +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -32,6 +33,7 @@ // This file is part of the OpenJPH software implementation. // File: ojph_block_common.cpp // Author: Aous Naman +// Author: Osamu Watanabe // Date: 13 May 2022 //***************************************************************************/ @@ -102,9 +104,14 @@ namespace ojph { /// @brief uvlc_tbl0 contains decoding information for initial row of quads ui16 uvlc_tbl0[256+64] = { 0 }; - /// @brief uvlc_tbl1 contains decoding information for non-initial row of + /// @brief uvlc_tbl1 contains decoding information for non-initial row of /// quads ui16 uvlc_tbl1[256] = { 0 }; + /// @brief uvlc_tbl1_wide: wider UVLC table for non-initial rows. + /// Index = mode(2 bits) * 1024 + vlc_data(10 bits) = 12 bits. + /// Entry bits: [4:0]=total_bits, [12:5]=u_q0, [20:13]=u_q1. + /// total_bits == 0x1F means fallback to original decode path. + ui32 uvlc_tbl1_wide[4096] = { 0 }; /// @brief uvlc_bias contains decoding info. for initial row of quads ui8 uvlc_bias[256+64] = { 0 }; /// @} @@ -329,6 +336,85 @@ namespace ojph { return true; } + //************************************************************************/ + /** @ingroup uvlc_decoding_tables_grp + * @brief Initializes uvlc_tbl1_wide: wider UVLC table for non-initial + * rows. Index = mode(2b) * 1024 + vlc(10b). Entry packs + * total_bits[4:0], u_q0[12:5], u_q1[20:13]. + * total_bits == 0x1F signals fallback to original decode. + */ + static bool uvlc_init_wide_table() + { + static const ui8 dec[8] = { + 3 | (5 << 2) | (5 << 5), //000 + 1 | (0 << 2) | (1 << 5), //xx1 + 2 | (0 << 2) | (2 << 5), //x10 + 1 | (0 << 2) | (1 << 5), //xx1 + 3 | (1 << 2) | (3 << 5), //100 + 1 | (0 << 2) | (1 << 5), //xx1 + 2 | (0 << 2) | (2 << 5), //x10 + 1 | (0 << 2) | (1 << 5) //xx1 + }; + + for (ui32 idx = 0; idx < 4096; ++idx) + { + ui32 mode = idx >> 10; // 2 bits + ui32 vlc = idx & 0x3FF; // 10 bits + + if (mode == 0) { + uvlc_tbl1_wide[idx] = 0; + continue; + } + + if (mode <= 2) // single UVLC (one u_off set) + { + ui32 d = dec[vlc & 0x7]; + ui32 prefix_len = d & 0x3; + ui32 suffix_len = (d >> 2) & 0x7; + ui32 u_pfx = d >> 5; + ui32 suffix_val = (vlc >> prefix_len) & ((1u << suffix_len) - 1); + ui32 u_val = u_pfx + suffix_val; + ui32 total = prefix_len + suffix_len; + ui32 u_q0 = (mode == 1) ? u_val : 0; + ui32 u_q1 = (mode == 2) ? u_val : 0; + uvlc_tbl1_wide[idx] = total | (u_q0 << 5) | (u_q1 << 13); + continue; + } + + // mode == 3: both u_off set + // Bitstream layout: [prefix0][prefix1][suffix0][suffix1] + ui32 d0 = dec[vlc & 0x7]; + ui32 p0_len = d0 & 0x3; + ui32 s0_len = (d0 >> 2) & 0x7; + ui32 u0_pfx = d0 >> 5; + + ui32 vlc1 = vlc >> p0_len; // consume prefix0 + ui32 d1 = dec[vlc1 & 0x7]; + ui32 p1_len = d1 & 0x3; + ui32 s1_len = (d1 >> 2) & 0x7; + ui32 u1_pfx = d1 >> 5; + + ui32 total_prefix = p0_len + p1_len; + ui32 total_suffix = s0_len + s1_len; + ui32 total = total_prefix + total_suffix; + + if (total > 10) { + uvlc_tbl1_wide[idx] = 0x1F; // fallback sentinel + continue; + } + + // suffixes follow both prefixes in the bitstream + ui32 suffix_bits = vlc >> total_prefix; + ui32 s0_val = suffix_bits & ((1u << s0_len) - 1); + ui32 s1_val = (suffix_bits >> s0_len) & ((1u << s1_len) - 1); + + ui32 u_q0 = u0_pfx + s0_val; + ui32 u_q1 = u1_pfx + s1_val; + uvlc_tbl1_wide[idx] = total | (u_q0 << 5) | (u_q1 << 13); + } + return true; + } + //************************************************************************/ /** @ingroup vlc_decoding_tables_grp * @brief Initializes VLC tables vlc_tbl0 and vlc_tbl1 @@ -341,5 +427,11 @@ namespace ojph { */ static bool uvlc_tables_initialized = uvlc_init_tables(); + //************************************************************************/ + /** @ingroup uvlc_decoding_tables_grp + * @brief Initializes wide UVLC table uvlc_tbl1_wide + */ + static bool uvlc_wide_initialized = uvlc_init_wide_table(); + } // !namespace local } // !namespace ojph diff --git a/external/OpenJPH/src/core/coding/ojph_block_common.h b/external/OpenJPH/src/core/coding/ojph_block_common.h index f8d650326..4b3e18b99 100644 --- a/external/OpenJPH/src/core/coding/ojph_block_common.h +++ b/external/OpenJPH/src/core/coding/ojph_block_common.h @@ -5,7 +5,8 @@ // Copyright (c) 2022, Aous Naman // Copyright (c) 2022, Kakadu Software Pty Ltd, Australia // Copyright (c) 2022, The University of New South Wales, Australia -// +// Copyright (c) 2026, Osamu Watanabe +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -32,6 +33,7 @@ // This file is part of the OpenJPH software implementation. // File: ojph_block_common.h // Author: Aous Naman +// Author: Osamu Watanabe // Date: 13 May 2022 //***************************************************************************/ @@ -44,6 +46,7 @@ namespace ojph{ extern ui16 vlc_tbl1[1024]; extern ui16 uvlc_tbl0[256+64]; extern ui16 uvlc_tbl1[256]; + extern ui32 uvlc_tbl1_wide[4096]; extern ui8 uvlc_bias[256+64]; } // !namespace local } // !namespace ojph diff --git a/external/OpenJPH/src/core/coding/ojph_block_decoder.h b/external/OpenJPH/src/core/coding/ojph_block_decoder.h index a1970174e..9b7bb19e2 100644 --- a/external/OpenJPH/src/core/coding/ojph_block_decoder.h +++ b/external/OpenJPH/src/core/coding/ojph_block_decoder.h @@ -77,6 +77,12 @@ namespace ojph { ui32 missing_msbs, ui32 num_passes, ui32 lengths1, ui32 lengths2, ui32 width, ui32 height, ui32 stride, bool stripe_causal); + // POWER VSX-accelerated decoder + bool + ojph_decode_codeblock_vsx(ui8* coded_data, ui32* decoded_data, + ui32 missing_msbs, ui32 num_passes, ui32 lengths1, ui32 lengths2, + ui32 width, ui32 height, ui32 stride, bool stripe_causal); + } } diff --git a/external/OpenJPH/src/core/coding/ojph_block_decoder32.cpp b/external/OpenJPH/src/core/coding/ojph_block_decoder32.cpp index daf2312dc..3b3e7d8c0 100644 --- a/external/OpenJPH/src/core/coding/ojph_block_decoder32.cpp +++ b/external/OpenJPH/src/core/coding/ojph_block_decoder32.cpp @@ -97,7 +97,7 @@ namespace ojph { ui32 val = 0xFFFFFFFF; // feed in 0xFF if buffer is exhausted if (melp->size > 4) { // if there is data in the MEL segment - val = *(ui32*)melp->data; // read 32 bits from MEL data + val = load_le_ui32(melp->data); // read 32 bits from MEL data melp->data += 4; // advance pointer melp->size -= 4; // reduce counter } @@ -315,7 +315,7 @@ namespace ojph { if (vlcp->size > 3) // if there are more than 3 bytes left in VLC { // (vlcp->data - 3) move pointer back to read 32 bits at once - val = *(ui32*)(vlcp->data - 3); // then read 32 bits + val = load_le_ui32(vlcp->data - 3); // then read 32 bits vlcp->data -= 4; // move data pointer back by 4 vlcp->size -= 4; // reduce available byte by 4 } @@ -458,7 +458,7 @@ namespace ojph { ui32 val = 0; if (mrp->size > 3) // If there are 3 byte or more { // (mrp->data - 3) move pointer back to read 32 bits at once - val = *(ui32*)(mrp->data - 3); // read 32 bits + val = load_le_ui32(mrp->data - 3); // read 32 bits mrp->data -= 4; // move back pointer mrp->size -= 4; // reduce count } @@ -612,7 +612,7 @@ namespace ojph { ui32 val = 0; if (msp->size > 3) { - val = *(ui32*)msp->data; // read 32 bits + val = load_le_ui32(msp->data); // read 32 bits msp->data += 4; // increment pointer msp->size -= 4; // reduce size } @@ -1408,13 +1408,13 @@ namespace ojph { // We need data for at least 5 columns out of 8. // Therefore loading 32 bits is easier than loading 16 bits // twice. - ui32 ps = *(ui32*)prev_sig; - ui32 ns = *(ui32*)(cur_sig + mstr); + ui32 ps = load_le_ui16x2(prev_sig); + ui32 ns = load_le_ui16x2(cur_sig + mstr); ui32 u = (ps & 0x88888888) >> 3; // the row on top if (!stripe_causal) u |= (ns & 0x11111111) << 3; // the row below - ui32 cs = *(ui32*)cur_sig; + ui32 cs = load_le_ui16x2(cur_sig); // vertical integration ui32 mbr = cs; // this sig. info. mbr |= (cs & 0x77777777) << 1; //above neighbors @@ -1566,16 +1566,16 @@ namespace ojph { for (ui32 y = 0; y < height; y += 4) { - ui32 *cur_sig = (ui32*)(sigma + (y >> 2) * mstr); + ui16 *cur_sig = sigma + (y >> 2) * mstr; ui32 *dpp = decoded_data + y * stride; ui32 half = 1 << (p - 2); - for (ui32 i = 0; i < width; i += 8) + for (ui32 i = 0; i < width; i += 8, cur_sig += 2) { //Process one entry from sigma array at a time // Each nibble (4 bits) in the sigma array represents 4 rows, // and the 32 bits contain 8 columns ui32 cwd = rev_fetch_mrp(&magref); // get 32 bit data - ui32 sig = *cur_sig++; // 32 bit that will be processed now + ui32 sig = load_le_ui16x2(cur_sig); // 32 bits processed now ui32 col_mask = 0xFu; // a mask for a column in sig if (sig) // if any of the 32 bits are set { diff --git a/external/OpenJPH/src/core/coding/ojph_block_decoder64.cpp b/external/OpenJPH/src/core/coding/ojph_block_decoder64.cpp index bce5b9ec4..1c940fb81 100644 --- a/external/OpenJPH/src/core/coding/ojph_block_decoder64.cpp +++ b/external/OpenJPH/src/core/coding/ojph_block_decoder64.cpp @@ -97,7 +97,7 @@ namespace ojph { ui32 val = 0xFFFFFFFF; // feed in 0xFF if buffer is exhausted if (melp->size > 4) { // if there is data in the MEL segment - val = *(ui32*)melp->data; // read 32 bits from MEL data + val = load_le_ui32(melp->data); // read 32 bits from MEL data melp->data += 4; // advance pointer melp->size -= 4; // reduce counter } @@ -410,7 +410,7 @@ namespace ojph { ui32 val = 0; if (mrp->size > 3) // If there are 3 byte or more { // (mrp->data - 3) move pointer back to read 32 bits at once - val = *(ui32*)(mrp->data - 3); // read 32 bits + val = load_le_ui32(mrp->data - 3); // read 32 bits mrp->data -= 4; // move back pointer mrp->size -= 4; // reduce count } @@ -564,7 +564,7 @@ namespace ojph { ui32 val = 0; if (msp->size > 3) { - val = *(ui32*)msp->data; // read 32 bits + val = load_le_ui32(msp->data); // read 32 bits msp->data += 4; // increment pointer msp->size -= 4; // reduce size } @@ -1455,13 +1455,13 @@ namespace ojph { // We need data for at least 5 columns out of 8. // Therefore loading 32 bits is easier than loading 16 bits // twice. - ui32 ps = *(ui32*)prev_sig; - ui32 ns = *(ui32*)(cur_sig + mstr); + ui32 ps = load_le_ui16x2(prev_sig); + ui32 ns = load_le_ui16x2(cur_sig + mstr); ui32 u = (ps & 0x88888888) >> 3; // the row on top if (!stripe_causal) u |= (ns & 0x11111111) << 3; // the row below - ui32 cs = *(ui32*)cur_sig; + ui32 cs = load_le_ui16x2(cur_sig); // vertical integration ui32 mbr = cs; // this sig. info. mbr |= (cs & 0x77777777) << 1; //above neighbors @@ -1613,16 +1613,16 @@ namespace ojph { for (ui32 y = 0; y < height; y += 4) { - ui32 *cur_sig = (ui32*)(sigma + (y >> 2) * mstr); + ui16 *cur_sig = sigma + (y >> 2) * mstr; ui64 *dpp = decoded_data + y * stride; ui64 half = 1ULL << (p - 2); - for (ui32 i = 0; i < width; i += 8) + for (ui32 i = 0; i < width; i += 8, cur_sig += 2) { //Process one entry from sigma array at a time // Each nibble (4 bits) in the sigma array represents 4 rows, // and the 32 bits contain 8 columns ui32 cwd = rev_fetch_mrp(&magref); // get 32 bit data - ui32 sig = *cur_sig++; // 32 bit that will be processed now + ui32 sig = load_le_ui16x2(cur_sig); // 32 bits processed now ui32 col_mask = 0xFu; // a mask for a column in sig if (sig) // if any of the 32 bits are set { diff --git a/external/OpenJPH/src/core/coding/ojph_block_decoder_avx2.cpp b/external/OpenJPH/src/core/coding/ojph_block_decoder_avx2.cpp index cccb4fd81..814a82b23 100644 --- a/external/OpenJPH/src/core/coding/ojph_block_decoder_avx2.cpp +++ b/external/OpenJPH/src/core/coding/ojph_block_decoder_avx2.cpp @@ -6,6 +6,7 @@ // Copyright (c) 2022, Kakadu Software Pty Ltd, Australia // Copyright (c) 2022, The University of New South Wales, Australia // Copyright (c) 2024, Intel Corporation +// Copyright (c) 2026, Osamu Watanabe // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -100,7 +101,7 @@ namespace ojph { ui32 val = 0xFFFFFFFF; // feed in 0xFF if buffer is exhausted if (melp->size > 4) { // if there is data in the MEL segment - val = *(ui32*)melp->data; // read 32 bits from MEL data + memcpy(&val, melp->data, 4); // read 32 bits from MEL data melp->data += 4; // advance pointer melp->size -= 4; // reduce counter } @@ -271,507 +272,349 @@ namespace ojph { return t; // return run } - //************************************************************************/ - /** @brief A structure for reading and unstuffing a segment that grows - * backward, such as VLC and MRP - */ - struct rev_struct { - rev_struct() : data(NULL), tmp(0), bits(0), size(0), unstuff(false) - {} - //storage - ui8* data; //! static inline - void rev_read(rev_struct *vlcp) + ui32 destuff_frwd(const ui8* src, int size, ui8* dst, ui32 cap) { - //process 4 bytes at a time - if (vlcp->bits > 32) // if there are more than 32 bits in tmp, then - return; // reading 32 bits can overflow vlcp->tmp - ui32 val = 0; - //the next line (the if statement) needs to be tested first - if (vlcp->size > 3) // if there are more than 3 bytes left in VLC + if (size < 0) + size = 0; + ui8* o = dst; + ui8* o_end = dst + cap; + const ui8* s = src; + const ui8* s_end = src + size; + ui64 acc = 0; // partial output byte, low nb bits are valid + ui32 nb = 0; // number of valid bits in acc; always < 8 + bool prev_ff = false; + + // fast path; 16 source bytes at a time when they contain no 0xFF + while (s + 16 <= s_end && o + 24 <= o_end) { - // (vlcp->data - 3) move pointer back to read 32 bits at once - val = *(ui32*)(vlcp->data - 3); // then read 32 bits - vlcp->data -= 4; // move data pointer back by 4 - vlcp->size -= 4; // reduce available byte by 4 - } - else if (vlcp->size > 0) - { // 4 or less - int i = 24; - while (vlcp->size > 0) { - ui32 v = *vlcp->data--; // read one byte at a time - val |= (v << i); // put byte in its correct location - --vlcp->size; - i -= 8; + __m128i v = _mm_loadu_si128((const __m128i*)s); + int ff = _mm_movemask_epi8(_mm_cmpeq_epi8(v, _mm_set1_epi8(-1))); + if (ff != 0 || prev_ff) + { // process these 16 bytes one at a time + for (int i = 0; i < 16; ++i) { + ui8 b = *s++; + acc |= (ui64)b << nb; + nb += prev_ff ? 7u : 8u; + prev_ff = (b == 0xFFu); + if (nb >= 8) { *o++ = (ui8)acc; acc >>= 8; nb -= 8; } + } + continue; } + ui64 v0, v1; + memcpy(&v0, s, 8); + memcpy(&v1, s + 8, 8); + ui64 w0 = acc | (v0 << nb); + ui64 w1 = (v1 << nb) | (nb ? (v0 >> (64 - nb)) : 0); + memcpy(o, &w0, 8); + memcpy(o + 8, &w1, 8); + acc = nb ? (v1 >> (64 - nb)) : 0; + o += 16; + s += 16; } - - __m128i tmp_vec = _mm_set1_epi32((int32_t)val); - tmp_vec = _mm_srlv_epi32(tmp_vec, _mm_setr_epi32(24, 16, 8, 0)); - tmp_vec = _mm_and_si128(tmp_vec, _mm_set1_epi32(0xff)); - - __m128i unstuff_vec = _mm_cmpgt_epi32(tmp_vec, _mm_set1_epi32(0x8F)); - bool unstuff_next = _mm_extract_epi32(unstuff_vec, 3); - unstuff_vec = _mm_slli_si128(unstuff_vec, 4); - unstuff_vec = _mm_insert_epi32(unstuff_vec, vlcp->unstuff * 0xffffffff, 0); - - __m128i val_7f = _mm_set1_epi32(0x7F); - __m128i this_byte_7f = _mm_cmpeq_epi32(_mm_and_si128(tmp_vec, val_7f), val_7f); - unstuff_vec = _mm_and_si128(unstuff_vec, this_byte_7f); - unstuff_vec = _mm_srli_epi32(unstuff_vec, 31); - - __m128i inc_sum = _mm_sub_epi32(_mm_set1_epi32(8), unstuff_vec); - inc_sum = _mm_add_epi32(inc_sum, _mm_bslli_si128(inc_sum, 4)); - inc_sum = _mm_add_epi32(inc_sum, _mm_bslli_si128(inc_sum, 8)); - ui32 total_bits = (ui32)_mm_extract_epi32(inc_sum, 3); - - __m128i final_shift = _mm_slli_si128(inc_sum, 4); - tmp_vec = _mm_sllv_epi32(tmp_vec, final_shift); - tmp_vec = _mm_or_si128(tmp_vec, _mm_bsrli_si128(tmp_vec, 8)); - - ui64 tmp = (ui32)_mm_cvtsi128_si32(tmp_vec) | (ui32)_mm_extract_epi32(tmp_vec, 1); - - vlcp->unstuff = unstuff_next; - vlcp->tmp |= tmp << vlcp->bits; - vlcp->bits += total_bits; - } - - //************************************************************************/ - /** @brief Initiates the rev_struct structure and reads a few bytes to - * move the read address to multiple of 4 - * - * There is another similar rev_init_mrp subroutine. The difference is - * that this one, rev_init, discards the first 12 bits (they have the - * sum of the lengths of VLC and MEL segments), and first unstuff depends - * on first 4 bits. - * - * @param [in] vlcp is a pointer to rev_struct structure - * @param [in] data is a pointer to byte at the start of the cleanup pass - * @param [in] lcup is the length of MagSgn+MEL+VLC segments - * @param [in] scup is the length of MEL+VLC segments - */ - static inline - void rev_init(rev_struct *vlcp, ui8* data, int lcup, int scup) - { - //first byte has only the upper 4 bits - vlcp->data = data + lcup - 2; - - //size can not be larger than this, in fact it should be smaller - vlcp->size = scup - 2; - - ui32 d = *vlcp->data--; // read one byte (this is a half byte) - vlcp->tmp = d >> 4; // both initialize and set - vlcp->bits = 4 - ((vlcp->tmp & 7) == 7); //check standard - vlcp->unstuff = (d | 0xF) > 0x8F; //this is useful for the next byte - - //This code is designed for an architecture that read address should - // align to the read size (address multiple of 4 if read size is 4) - //These few lines take care of the case where data is not at a multiple - // of 4 boundary. It reads 1,2,3 up to 4 bytes from the VLC bitstream. - // To read 32 bits, read from (vlcp->data - 3) - int num = 1 + (int)(intptr_t(vlcp->data) & 0x3); - int tnum = num < vlcp->size ? num : vlcp->size; - for (int i = 0; i < tnum; ++i) { - ui64 d; - d = *vlcp->data--; // read one byte and move read pointer - //check if the last byte was >0x8F (unstuff == true) and this is 0x7F - ui32 d_bits = 8 - ((vlcp->unstuff && ((d & 0x7F) == 0x7F)) ? 1 : 0); - vlcp->tmp |= d << vlcp->bits; // move data to vlcp->tmp - vlcp->bits += d_bits; - vlcp->unstuff = d > 0x8F; // for next byte - } - vlcp->size -= tnum; - rev_read(vlcp); // read another 32 buts - } - - //************************************************************************/ - /** @brief Retrieves 32 bits from the head of a rev_struct structure - * - * By the end of this call, vlcp->tmp must have no less than 33 bits - * - * @param [in] vlcp is a pointer to rev_struct structure - */ - static inline - ui32 rev_fetch(rev_struct *vlcp) - { - if (vlcp->bits < 32) // if there are less then 32 bits, read more + // tail; one byte at a time + while (s < s_end && o < o_end) { - rev_read(vlcp); // read 32 bits, but unstuffing might reduce this - if (vlcp->bits < 32)// if there is still space in vlcp->tmp for 32 bits - rev_read(vlcp); // read another 32 + ui8 b = *s++; + acc |= (ui64)b << nb; + nb += prev_ff ? 7u : 8u; + prev_ff = (b == 0xFFu); + if (nb >= 8) { *o++ = (ui8)acc; acc >>= 8; nb -= 8; } } - return (ui32)vlcp->tmp; // return the head (bottom-most) of vlcp->tmp - } - - //************************************************************************/ - /** @brief Consumes num_bits from a rev_struct structure - * - * @param [in] vlcp is a pointer to rev_struct structure - * @param [in] num_bits is the number of bits to be removed - */ - static inline - ui32 rev_advance(rev_struct *vlcp, ui32 num_bits) - { - assert(num_bits <= vlcp->bits); // vlcp->tmp must have more than num_bits - vlcp->tmp >>= num_bits; // remove bits - vlcp->bits -= num_bits; // decrement the number of bits - return (ui32)vlcp->tmp; + // fill the bits above nb with X, and pad with X bytes + ui32 fill = (X == 0xFF) ? (0xFFu << nb) : 0; + *o = (ui8)((ui32)acc | fill); + __m256i pad = _mm256_set1_epi8((char)X); + _mm256_storeu_si256((__m256i*)(o + 1), pad); + _mm256_storeu_si256((__m256i*)(o + 33), pad); + return (ui32)(o - dst) + 1; } //************************************************************************/ - /** @brief Reads and unstuffs from rev_struct + /** @brief Fetches 128 bits from a destuffed MagSgn buffer * - * This is different than rev_read in that this fills in zeros when the - * the available data is consumed. The other does not care about the - * values when all data is consumed. + * Returns the 128 bits starting at bit position pos of the buffer + * produced by destuff_frwd. Unlike a stateful bitstream reader, + * this carries no + * serial state; fetches at independent positions can execute out of + * order. The byte offset is clamped to limit so that positions past + * the end of the stream read as 1s without leaving the buffer. * - * See rev_read for more information about unstuffing - * - * @param [in] mrp is a pointer to rev_struct structure + * @param [in] dbuf is the destuffed MagSgn buffer + * @param [in] limit is the clamp offset returned by destuff_frwd + * @param [in] pos is the absolute bit position to fetch from */ - static inline - void rev_read_mrp(rev_struct *mrp) + OJPH_FORCE_INLINE + __m128i dfetch(const ui8* dbuf, ui32 limit, ui32 pos) { - //process 4 bytes at a time - if (mrp->bits > 32) - return; - ui32 val = 0; - if (mrp->size > 3) // If there are 3 byte or more - { // (mrp->data - 3) move pointer back to read 32 bits at once - val = *(ui32*)(mrp->data - 3); // read 32 bits - mrp->data -= 4; // move back pointer - mrp->size -= 4; // reduce count - } - else if (mrp->size > 0) - { - int i = 24; - while (mrp->size > 0) { - ui32 v = *mrp->data--; // read one byte at a time - val |= (v << i); // put byte in its correct location - --mrp->size; - i -= 8; - } - } - - //accumulate in tmp, and keep count in bits - ui32 bits, tmp = val >> 24; - - //test if the last byte > 0x8F (unstuff must be true) and this is 0x7F - bits = 8 - ((mrp->unstuff && (((val >> 24) & 0x7F) == 0x7F)) ? 1 : 0); - bool unstuff = (val >> 24) > 0x8F; - - //process the next byte - tmp |= ((val >> 16) & 0xFF) << bits; - bits += 8 - ((unstuff && (((val >> 16) & 0x7F) == 0x7F)) ? 1 : 0); - unstuff = ((val >> 16) & 0xFF) > 0x8F; - - tmp |= ((val >> 8) & 0xFF) << bits; - bits += 8 - ((unstuff && (((val >> 8) & 0x7F) == 0x7F)) ? 1 : 0); - unstuff = ((val >> 8) & 0xFF) > 0x8F; - - tmp |= (val & 0xFF) << bits; - bits += 8 - ((unstuff && ((val & 0x7F) == 0x7F)) ? 1 : 0); - unstuff = (val & 0xFF) > 0x8F; - - mrp->tmp |= (ui64)tmp << mrp->bits; // move data to mrp pointer - mrp->bits += bits; - mrp->unstuff = unstuff; // next byte + ui32 off = pos >> 3; + off = off < limit ? off : limit; + const ui8* p = dbuf + off; + __m128i v = _mm_loadu_si128((const __m128i*)p); + __m128i w = _mm_loadu_si128((const __m128i*)(p + 8)); + int k = (int)(pos & 7); + __m128i r = _mm_srl_epi64(v, _mm_cvtsi32_si128(k)); + __m128i c = _mm_sll_epi64(w, _mm_cvtsi32_si128(64 - k)); + return _mm_or_si128(r, c); } //************************************************************************/ - /** @brief Initialized rev_struct structure for MRP segment, and reads - * a number of bytes such that the next 32 bits read are from - * an address that is a multiple of 4. Note this is designed for - * an architecture that read size must be compatible with the - * alignment of the read address + /** @brief Fetches at least 57 bits from a destuffed bitstream * - * There is another similar subroutine rev_init. This subroutine does - * NOT skip the first 12 bits, and starts with unstuff set to true. + * Scalar counterpart of dfetch; returns the bits starting at bit + * position pos of a buffer produced by destuff_frwd, in the lower + * bits of the result. Bits above 64 - (pos & 7) are garbage. * - * @param [in] mrp is a pointer to rev_struct structure - * @param [in] data is a pointer to byte at the start of the cleanup pass - * @param [in] lcup is the length of MagSgn+MEL+VLC segments - * @param [in] len2 is the length of SPP+MRP segments + * @param [in] dbuf is the destuffed bitstream buffer + * @param [in] limit is the clamp offset returned by destuff_frwd + * @param [in] pos is the absolute bit position to fetch from */ - static inline - void rev_init_mrp(rev_struct *mrp, ui8* data, int lcup, int len2) + OJPH_FORCE_INLINE + ui64 dfetch64(const ui8* dbuf, ui32 limit, ui32 pos) { - mrp->data = data + lcup + len2 - 1; - mrp->size = len2; - mrp->unstuff = true; - mrp->bits = 0; - mrp->tmp = 0; - - //This code is designed for an architecture that read address should - // align to the read size (address multiple of 4 if read size is 4) - //These few lines take care of the case where data is not at a multiple - // of 4 boundary. It reads 1,2,3 up to 4 bytes from the MRP stream - int num = 1 + (int)(intptr_t(mrp->data) & 0x3); - for (int i = 0; i < num; ++i) { - ui64 d; - //read a byte, 0 if no more data - d = (mrp->size-- > 0) ? *mrp->data-- : 0; - //check if unstuffing is needed - ui32 d_bits = 8 - ((mrp->unstuff && ((d & 0x7F) == 0x7F)) ? 1 : 0); - mrp->tmp |= d << mrp->bits; // move data to vlcp->tmp - mrp->bits += d_bits; - mrp->unstuff = d > 0x8F; // for next byte - } - rev_read_mrp(mrp); + ui32 off = pos >> 3; + off = off < limit ? off : limit; + ui64 v; + memcpy(&v, dbuf + off, 8); + return v >> (pos & 7); } //************************************************************************/ - /** @brief Retrieves 32 bits from the head of a rev_struct structure + /** @brief Branchless refill of a register-resident bit window * - * By the end of this call, mrp->tmp must have no less than 33 bits + * Loads 8 bytes from a destuffed buffer and inserts whole bytes + * above the bits remaining in the window, leaving 56 to 63 valid + * bits. Because the inserted bits land above the remaining ones, + * consumers of the low bits need not wait for the load, keeping it + * off the critical dependency chain. The read offset is clamped to + * limit so that, when consumption overruns the stream (truncated + * codeblocks), reads come from the zero padding -- the bytes there + * are exactly the fill the stream would produce -- instead of from + * uninitialized buffer memory. * - * @param [in] mrp is a pointer to rev_struct structure + * @param [in,out] val is the bit window; its low bits are valid + * @param [in,out] bits is the number of valid bits in val + * @param [in,out] off is the read offset in the destuffed buffer + * @param [in] dbuf is the destuffed bitstream buffer + * @param [in] limit is the clamp offset returned by destuff_vlc */ - static inline - ui32 rev_fetch_mrp(rev_struct *mrp) + OJPH_FORCE_INLINE + void drefill(ui64& val, ui32& bits, ui32& off, + const ui8* dbuf, ui32 limit) { - if (mrp->bits < 32) // if there are less than 32 bits in mrp->tmp - { - rev_read_mrp(mrp); // read 30-32 bits from mrp - if (mrp->bits < 32) // if there is a space of 32 bits - rev_read_mrp(mrp); // read more - } - return (ui32)mrp->tmp; // return the head of mrp->tmp + ui64 v; + ui32 o = off < limit ? off : limit; + memcpy(&v, dbuf + o, 8); + val |= v << bits; + off += (63 - bits) >> 3; + bits |= 56; } //************************************************************************/ - /** @brief Consumes num_bits from a rev_struct structure + /** @brief Consumes bits from a window refilled by drefill * - * @param [in] mrp is a pointer to rev_struct structure - * @param [in] num_bits is the number of bits to be removed + * @param [in,out] val is the bit window + * @param [in,out] bits is the number of valid bits in val + * @param [in] num_bits is the number of bits to consume */ - inline ui32 rev_advance_mrp(rev_struct *mrp, ui32 num_bits) + OJPH_FORCE_INLINE + void dconsume(ui64& val, ui32& bits, ui32 num_bits) { - assert(num_bits <= mrp->bits); // we must not consume more than mrp->bits - mrp->tmp >>= num_bits; // discard the lowest num_bits bits - mrp->bits -= num_bits; - return (ui32)mrp->tmp; // return data after consumption + val >>= num_bits; + bits -= num_bits; } //************************************************************************/ - /** @brief State structure for reading and unstuffing of forward-growing - * bitstreams; these are: MagSgn and SPP bitstreams - */ - struct frwd_struct_avx2 { - const ui8* data; //! 0x8F), so the fast path can detect + * stuffing with a pairwise byte comparison within the segment. * - * @tparam X is the value fed in when the bitstream is exhausted - * @param [in] msp is a pointer to frwd_struct_avx2 structure + * Writes at most cap + 1 destuffed bytes followed by 64 bytes of + * zero padding; dst must have room for cap + 65 bytes. cap must be + * no smaller than the maximum number of bits the decoder can consume + * divided by 8, so clipping the data at cap loses no decodable bits. * + * @param [in] p is a pointer to the last byte of the segment, where + * backward reading starts + * @param [in] size is the number of bytes in the segment + * @param [in] unstuff is the initial unstuffing state + * @param [in] acc holds bits produced by initialization, low nb bits + * @param [in] nb is the number of valid bits in acc; less than 8 + * @param [in] dst is the output buffer, of at least cap + 65 bytes + * @param [in] cap is the maximum number of destuffed bytes to write + * @return ui32 clamp offset for dfetch/dfetch64; bytes at or beyond + * this offset hold no stream bits (they read as 0s) */ - template static inline - void frwd_read(frwd_struct_avx2 *msp) + ui32 destuff_rev(const ui8* p, int size, bool unstuff, + ui64 acc, ui32 nb, ui8* dst, ui32 cap) { - assert(msp->bits <= 128); - - __m128i offset, val, validity, all_xff; - val = _mm_loadu_si128((__m128i*)msp->data); - int bytes = msp->size >= 16 ? 16 : msp->size; - validity = _mm_set1_epi8((char)bytes); - msp->data += bytes; - msp->size -= bytes; - int bits = 128; - offset = _mm_set_epi64x(0x0F0E0D0C0B0A0908,0x0706050403020100); - validity = _mm_cmpgt_epi8(validity, offset); - all_xff = _mm_set1_epi8(-1); - if (X == 0xFF) // the compiler should remove this if statement + ui8* o = dst; + ui8* o_end = dst + cap; + + // process the first byte with the caller-provided unstuff state; + // afterwards the state is implied by the byte at p[1], which the + // fast path checks vectorially (and which stays inside the segment) + if (size > 0 && o < o_end) { - __m128i t = _mm_xor_si128(validity, all_xff); // complement - val = _mm_or_si128(t, val); // fill with 0xFF - } - else if (X == 0) - val = _mm_and_si128(validity, val); // fill with zeros - else - assert(0); - - __m128i ff_bytes; - ff_bytes = _mm_cmpeq_epi8(val, all_xff); - ff_bytes = _mm_and_si128(ff_bytes, validity); - ui32 flags = (ui32)_mm_movemask_epi8(ff_bytes); - flags <<= 1; // unstuff following byte - ui32 next_unstuff = flags >> 16; - flags |= msp->unstuff; - flags &= 0xFFFF; - while (flags) - { // bit unstuffing occurs on average once every 256 bytes - // therefore it is not an issue if it is a bit slow - // here we process 16 bytes - --bits; // consuming one stuffing bit - - ui32 loc = 31 - count_leading_zeros(flags); - flags ^= 1 << loc; - - __m128i m, t, c; - t = _mm_set1_epi8((char)loc); - m = _mm_cmpgt_epi8(offset, t); - - t = _mm_and_si128(m, val); // keep bits at locations larger than loc - c = _mm_srli_epi64(t, 1); // 1 bits left - t = _mm_srli_si128(t, 8); // 8 bytes left - t = _mm_slli_epi64(t, 63); // keep the MSB only - t = _mm_or_si128(t, c); // combine the above 3 steps - - val = _mm_or_si128(t, _mm_andnot_si128(m, val)); + ui32 d = *p--; + --size; + acc |= (ui64)d << nb; + nb += 8 - ((unstuff && ((d & 0x7F) == 0x7F)) ? 1u : 0u); + unstuff = d > 0x8F; + if (nb >= 8) { *o++ = (ui8)acc; acc >>= 8; nb -= 8; } } - // combine with earlier data - assert(msp->bits >= 0 && msp->bits <= 128); - int cur_bytes = msp->bits >> 3; - int cur_bits = msp->bits & 7; - __m128i b1, b2; - b1 = _mm_sll_epi64(val, _mm_set1_epi64x(cur_bits)); - b2 = _mm_slli_si128(val, 8); // 8 bytes right - b2 = _mm_srl_epi64(b2, _mm_set1_epi64x(64-cur_bits)); - b1 = _mm_or_si128(b1, b2); - b2 = _mm_loadu_si128((__m128i*)(msp->tmp + cur_bytes)); - b2 = _mm_or_si128(b1, b2); - _mm_storeu_si128((__m128i*)(msp->tmp + cur_bytes), b2); - - int consumed_bits = bits < 128 - cur_bits ? bits : 128 - cur_bits; - cur_bytes = (msp->bits + (ui32)consumed_bits + 7) >> 3; // round up - int upper = _mm_extract_epi16(val, 7); - upper >>= consumed_bits - 128 + 16; - msp->tmp[cur_bytes] = (ui8)upper; // copy byte - - msp->bits += (ui32)bits; - msp->unstuff = next_unstuff; // next unstuff - assert(msp->unstuff == 0 || msp->unstuff == 1); + // fast path; 16 source bytes at a time when none needs unstuffing + while (size >= 16 && o + 24 <= o_end) + { + __m128i v = _mm_loadu_si128((const __m128i*)(p - 15)); + __m128i nx = _mm_loadu_si128((const __m128i*)(p - 14)); + __m128i is7f = _mm_cmpeq_epi8( + _mm_and_si128(v, _mm_set1_epi8(0x7F)), _mm_set1_epi8(0x7F)); + // le8f is 0xFF where the byte after (in memory) is <= 0x8F + __m128i le8f = _mm_cmpeq_epi8( + _mm_subs_epu8(nx, _mm_set1_epi8((char)0x8F)), + _mm_setzero_si128()); + __m128i stuff = _mm_andnot_si128(le8f, is7f); + if (!_mm_testz_si128(stuff, stuff)) + { // process these 16 bytes one at a time + for (int i = 0; i < 16; ++i) { + ui32 d = *p--; + acc |= (ui64)d << nb; + nb += 8 - ((unstuff && ((d & 0x7F) == 0x7F)) ? 1u : 0u); + unstuff = d > 0x8F; + if (nb >= 8) { *o++ = (ui8)acc; acc >>= 8; nb -= 8; } + } + size -= 16; + continue; + } + __m128i r = _mm_shuffle_epi8(v, + _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15)); // reverse bytes +#ifdef OJPH_ARCH_X86_64 + ui64 v0 = (ui64)_mm_cvtsi128_si64(r); + ui64 v1 = (ui64)_mm_extract_epi64(r, 1); +#else // 32-bit x86 lacks the 64-bit extract intrinsics + ui64 v0 = (ui32)_mm_cvtsi128_si32(r) + | ((ui64)(ui32)_mm_extract_epi32(r, 1) << 32); + ui64 v1 = (ui32)_mm_extract_epi32(r, 2) + | ((ui64)(ui32)_mm_extract_epi32(r, 3) << 32); +#endif + ui64 w0 = acc | (v0 << nb); + ui64 w1 = (v1 << nb) | (nb ? (v0 >> (64 - nb)) : 0); + memcpy(o, &w0, 8); + memcpy(o + 8, &w1, 8); + acc = nb ? (v1 >> (64 - nb)) : 0; + o += 16; + p -= 16; + size -= 16; + unstuff = p[1] > 0x8F; + } + // tail; one byte at a time + while (size > 0 && o < o_end) + { + ui32 d = *p--; + --size; + acc |= (ui64)d << nb; + nb += 8 - ((unstuff && ((d & 0x7F) == 0x7F)) ? 1u : 0u); + unstuff = d > 0x8F; + if (nb >= 8) { *o++ = (ui8)acc; acc >>= 8; nb -= 8; } + } + // the bits above nb are already 0 = fill; pad with zero bytes + *o = (ui8)acc; + __m256i z = _mm256_setzero_si256(); + _mm256_storeu_si256((__m256i*)(o + 1), z); + _mm256_storeu_si256((__m256i*)(o + 33), z); + return (ui32)(o - dst) + 1; } //************************************************************************/ - /** @brief Initialize frwd_struct_avx2 struct and reads some bytes + /** @brief De-stuffs the VLC segment into a flat buffer * - * @tparam X is the value fed in when the bitstream is exhausted. - * See frwd_read regarding the template - * @param [in] msp is a pointer to frwd_struct_avx2 - * @param [in] data is a pointer to the start of data - * @param [in] size is the number of byte in the bitstream - */ - template - static inline - void frwd_init(frwd_struct_avx2 *msp, const ui8* data, int size) - { - msp->data = data; - _mm_storeu_si128((__m128i *)msp->tmp, _mm_setzero_si128()); - _mm_storeu_si128((__m128i *)msp->tmp + 1, _mm_setzero_si128()); - _mm_storeu_si128((__m128i *)msp->tmp + 2, _mm_setzero_si128()); - - msp->bits = 0; - msp->unstuff = 0; - msp->size = size; - - frwd_read(msp); // read 128 bits more - } - - //************************************************************************/ - /** @brief Consume num_bits bits from the bitstream of frwd_struct_avx2 + * Performs the same initialization rev_init does -- the first byte + * contributes only its upper 4 bits (3 bits if its lower 3 bits are + * all 1s) -- then de-stuffs the remaining scup - 2 bytes backward; + * see destuff_rev. * - * @param [in] msp is a pointer to frwd_struct_avx2 - * @param [in] num_bits is the number of bit to consume + * @param [in] data is a pointer to byte at the start of the cleanup + * pass + * @param [in] lcup is the length of MagSgn+MEL+VLC segments + * @param [in] scup is the length of MEL+VLC segments + * @param [in] dst is the output buffer, of at least cap + 65 bytes + * @param [in] cap is the maximum number of destuffed bytes to write + * @return ui32 clamp offset for dfetch64 */ static inline - void frwd_advance(frwd_struct_avx2 *msp, ui32 num_bits) + ui32 destuff_vlc(const ui8* data, int lcup, int scup, + ui8* dst, ui32 cap) { - assert(num_bits > 0 && num_bits <= msp->bits && num_bits < 128); - msp->bits -= num_bits; - - __m128i *p = (__m128i*)(msp->tmp + ((num_bits >> 3) & 0x18)); - num_bits &= 63; - - __m128i v0, v1, c0, c1, t; - v0 = _mm_loadu_si128(p); - v1 = _mm_loadu_si128(p + 1); - - // shift right by num_bits - c0 = _mm_srl_epi64(v0, _mm_set1_epi64x(num_bits)); - t = _mm_srli_si128(v0, 8); - t = _mm_sll_epi64(t, _mm_set1_epi64x(64 - num_bits)); - c0 = _mm_or_si128(c0, t); - t = _mm_slli_si128(v1, 8); - t = _mm_sll_epi64(t, _mm_set1_epi64x(64 - num_bits)); - c0 = _mm_or_si128(c0, t); - - _mm_storeu_si128((__m128i*)msp->tmp, c0); - - c1 = _mm_srl_epi64(v1, _mm_set1_epi64x(num_bits)); - t = _mm_srli_si128(v1, 8); - t = _mm_sll_epi64(t, _mm_set1_epi64x(64 - num_bits)); - c1 = _mm_or_si128(c1, t); - - _mm_storeu_si128((__m128i*)msp->tmp + 1, c1); + const ui8* p = data + lcup - 2; + ui32 d = *p; // first byte, only the upper 4 bits are used + ui64 acc = d >> 4; + ui32 nb = 4 - ((acc & 7) == 7); // check standard + bool unstuff = (d | 0xF) > 0x8F; + return destuff_rev(p - 1, scup - 2, unstuff, acc, nb, dst, cap); } //************************************************************************/ - /** @brief Fetches 32 bits from the frwd_struct_avx2 bitstream + /** @brief De-stuffs the MRP segment into a flat buffer * - * @tparam X is the value fed in when the bitstream is exhausted. - * See frwd_read regarding the template - * @param [in] msp is a pointer to frwd_struct_avx2 + * Performs the same initialization rev_init_mrp does -- reading + * starts at the last byte of the SPP+MRP segments with the unstuff + * state set -- then de-stuffs backward; see destuff_rev. + * + * @param [in] data is a pointer to byte at the start of the cleanup + * pass + * @param [in] lcup is the length of MagSgn+MEL+VLC segments + * @param [in] len2 is the length of SPP+MRP segments + * @param [in] dst is the output buffer, of at least cap + 65 bytes + * @param [in] cap is the maximum number of destuffed bytes to write + * @return ui32 clamp offset for dfetch64 */ - template static inline - __m128i frwd_fetch(frwd_struct_avx2 *msp) + ui32 destuff_mrp(const ui8* data, int lcup, int len2, + ui8* dst, ui32 cap) { - if (msp->bits <= 128) - { - frwd_read(msp); - if (msp->bits <= 128) //need to test - frwd_read(msp); - } - __m128i t = _mm_loadu_si128((__m128i*)msp->tmp); - return t; + return destuff_rev(data + lcup + len2 - 1, len2, true, 0, 0, + dst, cap); } //************************************************************************/ @@ -784,7 +627,10 @@ namespace ojph { * @param vn used for handling E values (stores v_n values) * @return __m256i decoded two quads */ - static inline __m256i decode_two_quad32_avx2(__m256i inf_u_q, __m256i U_q, frwd_struct_avx2* magsgn, ui32 p, __m128i& vn) { + OJPH_FORCE_INLINE + __m256i decode_two_quad32_avx2(__m256i inf_u_q, __m256i U_q, + const ui8* dbuf, ui32 limit, ui32& pos, + ui32 p, __m128i& vn) { __m256i row = _mm256_setzero_si256(); // we keeps e_k, e_1, and rho in w2 @@ -810,19 +656,12 @@ namespace ojph { __m256i inc_sum = m_n; // inclusive scan inc_sum = _mm256_add_epi32(inc_sum, _mm256_bslli_epi128(inc_sum, 4)); inc_sum = _mm256_add_epi32(inc_sum, _mm256_bslli_epi128(inc_sum, 8)); - int total_mn1 = _mm256_extract_epi16(inc_sum, 6); - int total_mn2 = _mm256_extract_epi16(inc_sum, 14); - - __m128i ms_vec0 = _mm_setzero_si128(); - __m128i ms_vec1 = _mm_setzero_si128(); - if (total_mn1) { - ms_vec0 = frwd_fetch<0xFF>(magsgn); - frwd_advance(magsgn, (ui32)total_mn1); - } - if (total_mn2) { - ms_vec1 = frwd_fetch<0xFF>(magsgn); - frwd_advance(magsgn, (ui32)total_mn2); - } + ui32 total_mn1 = (ui32)_mm256_extract_epi16(inc_sum, 6); + ui32 total_mn2 = (ui32)_mm256_extract_epi16(inc_sum, 14); + + __m128i ms_vec0 = dfetch(dbuf, limit, pos); + __m128i ms_vec1 = dfetch(dbuf, limit, pos + total_mn1); + pos += total_mn1 + total_mn2; __m256i ms_vec = _mm256_inserti128_si256(_mm256_castsi128_si256(ms_vec0), ms_vec1, 0x1); @@ -898,7 +737,10 @@ namespace ojph { * @return __m128i decoded quad */ - static inline __m256i decode_four_quad16(const __m128i inf_u_q, __m128i U_q, frwd_struct_avx2* magsgn, ui32 p, __m128i& vn) { + OJPH_FORCE_INLINE + __m256i decode_four_quad16(const __m128i inf_u_q, __m128i U_q, + const ui8* dbuf, ui32 limit, ui32& pos, + ui32 p, __m128i& vn) { __m256i w0; // workers __m256i insig; // lanes hold FF's if samples are insignificant @@ -939,20 +781,13 @@ namespace ojph { inc_sum = _mm256_add_epi16(inc_sum, _mm256_bslli_epi128(inc_sum, 2)); inc_sum = _mm256_add_epi16(inc_sum, _mm256_bslli_epi128(inc_sum, 4)); inc_sum = _mm256_add_epi16(inc_sum, _mm256_bslli_epi128(inc_sum, 8)); - int total_mn1 = _mm256_extract_epi16(inc_sum, 7); - int total_mn2 = _mm256_extract_epi16(inc_sum, 15); + ui32 total_mn1 = (ui32)_mm256_extract_epi16(inc_sum, 7); + ui32 total_mn2 = (ui32)_mm256_extract_epi16(inc_sum, 15); __m256i ex_sum = _mm256_bslli_epi128(inc_sum, 2); // exclusive scan - __m128i ms_vec0 = _mm_setzero_si128(); - __m128i ms_vec1 = _mm_setzero_si128(); - if (total_mn1) { - ms_vec0 = frwd_fetch<0xFF>(magsgn); - frwd_advance(magsgn, (ui32)total_mn1); - } - if (total_mn2) { - ms_vec1 = frwd_fetch<0xFF>(magsgn); - frwd_advance(magsgn, (ui32)total_mn2); - } + __m128i ms_vec0 = dfetch(dbuf, limit, pos); + __m128i ms_vec1 = dfetch(dbuf, limit, pos + total_mn1); + pos += total_mn1 + total_mn2; __m256i ms_vec = _mm256_inserti128_si256(_mm256_castsi128_si256(ms_vec0), ms_vec1, 0x1); @@ -981,29 +816,25 @@ namespace ojph { d0 = _mm256_or_si256(d0, d1); // find location of e_k and mask - __m256i shift, t0, t1, Uq0, Uq1; + __m256i shift; __m256i ones = _mm256_set1_epi16(1); __m256i twos = _mm256_set1_epi16(2); - __m256i U_q_m1 = _mm256_sub_epi32(U_q_avx, ones); - Uq0 = _mm256_and_si256(U_q_m1, _mm256_set_epi32(0, 0, 0, 0x1F, 0, 0, 0, 0x1F)); - Uq1 = _mm256_bsrli_epi128(U_q_m1, 14); + // shift = (2 - e_k) << (U_q - 1); AVX2 has no _mm256_sllv_epi16, + // so the variable shift is emulated with a pshufb power-of-two + // lookup and a 16-bit multiply. U_q - 1 <= 14 in this path; + // for U_q == 0 the lookup indices have their MSB set, so pshufb + // returns 0, the same shift value the former uniform 16-bit + // shift emulation produced (it shifted by 31) + __m256i kq = _mm256_sub_epi16(U_q_avx, ones); + __m256i idx = _mm256_or_si256(kq, + _mm256_slli_epi16(_mm256_sub_epi16(kq, + _mm256_set1_epi16(8)), 8)); + const __m256i pow2_tbl = _mm256_setr_epi8( + 1, 2, 4, 8, 16, 32, 64, (char)128, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 2, 4, 8, 16, 32, 64, (char)128, 0, 0, 0, 0, 0, 0, 0, 0); + __m256i pow2 = _mm256_shuffle_epi8(pow2_tbl, idx); w0 = _mm256_sub_epi16(twos, w0); - t0 = _mm256_and_si256(w0, _mm256_set_epi64x(0, -1, 0, -1)); - t1 = _mm256_and_si256(w0, _mm256_set_epi64x(-1, 0, -1, 0)); - {//no _mm256_sllv_epi16 in avx2 - __m128i t_0_sse = _mm256_castsi256_si128(t0); - t_0_sse = _mm_sll_epi16(t_0_sse, _mm256_castsi256_si128(Uq0)); - __m128i t_1_sse = _mm256_extracti128_si256(t0 , 0x1); - t_1_sse = _mm_sll_epi16(t_1_sse, _mm256_extracti128_si256(Uq0, 0x1)); - t0 = _mm256_inserti128_si256(_mm256_castsi128_si256(t_0_sse), t_1_sse, 0x1); - - t_0_sse = _mm256_castsi256_si128(t1); - t_0_sse = _mm_sll_epi16(t_0_sse, _mm256_castsi256_si128(Uq1)); - t_1_sse = _mm256_extracti128_si256(t1, 0x1); - t_1_sse = _mm_sll_epi16(t_1_sse, _mm256_extracti128_si256(Uq1, 0x1)); - t1 = _mm256_inserti128_si256(_mm256_castsi128_si256(t_0_sse), t_1_sse, 0x1); - } - shift = _mm256_or_si256(t0, t1); + shift = _mm256_mullo_epi16(w0, pow2); ms_vec = _mm256_and_si256(d0, _mm256_sub_epi16(shift, ones)); // next e_1 @@ -1065,380 +896,166 @@ namespace ojph { * @param [in] stride is the decoded codeblock buffer stride * @param [in] stripe_causal is true for stripe causal mode */ - bool ojph_decode_codeblock_avx2(ui8* coded_data, ui32* decoded_data, - ui32 missing_msbs, ui32 num_passes, - ui32 lengths1, ui32 lengths2, - ui32 width, ui32 height, ui32 stride, - bool stripe_causal) + //************************************************************************/ + /** @brief Step-2 MagSgn decode for the 16-bit (4-quad) path. + * + * Outlined into its own function so that decode_four_quad16 can be + * force-inlined here without inflating register pressure in the + * much larger ojph_decode_codeblock_avx2 (which also hosts the + * mutually-exclusive 32-bit step-2 path). Returns false on a + * precision-overflow error, true otherwise. + */ + OJPH_NO_INLINE + bool decode_cb_step2_16bit(ui16* scratch, ui32* decoded_data, + ui8* coded_data, ui32 width, ui32 height, + ui32 stride, ui32 sstr, ui32 p, ui32 mmsbp2, + int lcup, int scup) { - static bool insufficient_precision = false; - static bool modify_code = false; - static bool truncate_spp_mrp = false; + // reduce bitplane by 16 because we now have 16 bits instead of 32 + p -= 16; - if (num_passes > 1 && lengths2 == 0) - { - OJPH_WARN(0x00010001, "A malformed codeblock that has more than " - "one coding pass, but zero length for " - "2nd and potential 3rd pass."); - num_passes = 1; - } + const int v_n_size = 512 + 16; +#ifdef __MINGW64__ + ui16 v_n_scratch[v_n_size] = {0}; + ui32 v_n_scratch_32[v_n_size] = {0}; +#else + ui16 v_n_scratch[v_n_size]; + memset(v_n_scratch + (width >> 1) + 4, 0, 8 * sizeof(ui16)); + ui32 v_n_scratch_32[v_n_size]; +#endif - if (num_passes > 3) - { - OJPH_WARN(0x00010002, "We do not support more than 3 coding passes; " - "This codeblocks has %d passes.", - num_passes); - return false; - } + // maximum consumable MagSgn bits: 4096 samples x (mmsbp2 < 16) bits + const ui32 dbuf_cap = 4096 * 15 / 8; + ui8 dbuf[dbuf_cap + 72]; + ui32 limit = destuff_frwd<0xFF>(coded_data, lcup - scup, dbuf, dbuf_cap); + ui32 pos = 0; - if (missing_msbs > 30) // p < 0 - { - if (insufficient_precision == false) { - insufficient_precision = true; - OJPH_WARN(0x00010003, "32 bits are not enough to decode this " - "codeblock. This message will not be " - "displayed again."); - } - return false; - } - else if (missing_msbs == 30) // p == 0 - { // not enough precision to decode and set the bin center to 1 - if (modify_code == false) { - modify_code = true; - OJPH_WARN(0x00010004, "Not enough precision to decode the cleanup " - "pass. The code can be modified to support " - "this case. This message will not be " - "displayed again."); - } - return false; // 32 bits are not enough to decode this - } - else if (missing_msbs == 29) // if p is 1, then num_passes must be 1 - { - if (num_passes > 1) { - num_passes = 1; - if (truncate_spp_mrp == false) { - truncate_spp_mrp = true; - OJPH_WARN(0x00010005, "Not enough precision to decode the SgnProp " - "nor MagRef passes; both will be skipped. " - "This message will not be displayed " - "again."); - } - } - } - ui32 p = 30 - missing_msbs; // The least significant bitplane for CUP - // There is a way to handle the case of p == 0, but a different path - // is required + ui16 *sp = scratch; + ui16 *vp = v_n_scratch; + ui32 *dp = decoded_data; + vp[0] = 2; // for easy calculation of emax - if (lengths1 < 2) - { - OJPH_WARN(0x00010006, "Wrong codeblock length."); - return false; - } + for (ui32 x = 0; x < width; x += 8, sp += 8, vp += 4, dp += 8) { + ////process four quads + __m128i inf_u_q = _mm_loadu_si128((__m128i*)sp); + __m128i U_q = _mm_srli_epi32(inf_u_q, 16); + __m128i w = _mm_cmpgt_epi32(U_q, _mm_set1_epi32((int)mmsbp2)); + if (!_mm_testz_si128(w, w)) { + return false; + } - // read scup and fix the bytes there - int lcup, scup; - lcup = (int)lengths1; // length of CUP - //scup is the length of MEL + VLC - scup = (((int)coded_data[lcup-1]) << 4) + (coded_data[lcup-2] & 0xF); - if (scup < 2 || scup > lcup || scup > 4079) //something is wrong - return false; + __m128i vn = _mm_set1_epi16(2); + __m256i row = decode_four_quad16(inf_u_q, U_q, dbuf, limit, pos, p, vn); - // The temporary storage scratch holds two types of data in an - // interleaved fashion. The interleaving allows us to use one - // memory pointer. - // We have one entry for a decoded VLC code, and one entry for UVLC. - // Entries are 16 bits each, corresponding to one quad, - // but since we want to use XMM registers of the SSE family - // of SIMD; we allocated 16 bytes or more per quad row; that is, - // the width is no smaller than 16 bytes (or 8 entries), and the - // height is 512 quads - // Each VLC entry contains, in the following order, starting - // from MSB - // e_k (4bits), e_1 (4bits), rho (4bits), useless for step 2 (4bits) - // Each entry in UVLC contains u_q - // One extra row to handle the case of SPP propagating downwards - // when codeblock width is 4 - ui16 scratch[8 * 513] = {0}; // 8+ kB + w = _mm_cvtsi32_si128(*(unsigned short const*)(vp)); + _mm_storeu_si128((__m128i*)vp, _mm_or_si128(w, vn)); - // We need an extra two entries (one inf and one u_q) beyond - // the last column. - // If the block width is 4 (2 quads), then we use sstr of 8 - // (enough for 4 quads). If width is 8 (4 quads) we use - // sstr is 16 (enough for 8 quads). For a width of 16 (8 - // quads), we use 24 (enough for 12 quads). - ui32 sstr = ((width + 2u) + 7u) & ~7u; // multiples of 8 + __m256i w0 = _mm256_shuffle_epi8(row, _mm256_set_epi16(0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1, 0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1)); + __m256i w1 = _mm256_shuffle_epi8(row, _mm256_set_epi16(0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1, 0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1)); - assert((stride & 0x3) == 0); + _mm256_storeu_si256((__m256i*)dp, w0); + _mm256_storeu_si256((__m256i*)(dp + stride), w1); + } + } - ui32 mmsbp2 = missing_msbs + 2; + for (ui32 y = 2; y < height; y += 2) { + { + // perform 15 - count_leading_zeros(*vp) here + ui16 *vp = v_n_scratch; + ui32 *vp_32 = v_n_scratch_32; - // The cleanup pass is decoded in two steps; in step one, - // the VLC and MEL segments are decoded, generating a record that - // has 2 bytes per quad. The 2 bytes contain, u, rho, e^1 & e^k. - // This information should be sufficient for the next step. - // In step 2, we decode the MagSgn segment. + ui16* sp = scratch + (y >> 1) * sstr; + const __m256i avx_mmsbp2 = _mm256_set1_epi32((int)mmsbp2); + const __m256i avx_31 = _mm256_set1_epi32(31); + const __m256i avx_f0 = _mm256_set1_epi32(0xF0); + const __m256i avx_1 = _mm256_set1_epi32(1); + const __m256i avx_0 = _mm256_setzero_si256(); - // step 1 decoding VLC and MEL segments - { - // init structures - dec_mel_st mel; - mel_init(&mel, coded_data, lcup, scup); - rev_struct vlc; - rev_init(&vlc, coded_data, lcup, scup); - - int run = mel_get_run(&mel); // decode runs of events from MEL bitstrm - // data represented as runs of 0 events - // See mel_decode description - - ui32 vlc_val; - ui32 c_q = 0; - ui16 *sp = scratch; - //initial quad row - for (ui32 x = 0; x < width; sp += 4) - { - // decode VLC - ///////////// - - // first quad - vlc_val = rev_fetch(&vlc); - - //decode VLC using the context c_q and the head of VLC bitstream - ui16 t0 = vlc_tbl0[ c_q + (vlc_val & 0x7F) ]; - - // if context is zero, use one MEL event - if (c_q == 0) //zero context - { - run -= 2; //subtract 2, since events number if multiplied by 2 - - // Is the run terminated in 1? if so, use decoded VLC code, - // otherwise, discard decoded data, since we will decoded again - // using a different context - t0 = (run == -1) ? t0 : 0; - - // is run -1 or -2? this means a run has been consumed - if (run < 0) - run = mel_get_run(&mel); // get another run - } - //run -= (c_q == 0) ? 2 : 0; - //t0 = (c_q != 0 || run == -1) ? t0 : 0; - //if (run < 0) - // run = mel_get_run(&mel); // get another run - sp[0] = t0; - x += 2; - - // prepare context for the next quad; eqn. 1 in ITU T.814 - c_q = ((t0 & 0x10U) << 3) | ((t0 & 0xE0U) << 2); - - //remove data from vlc stream (0 bits are removed if vlc is not used) - vlc_val = rev_advance(&vlc, t0 & 0x7); - - //second quad - ui16 t1 = 0; - - //decode VLC using the context c_q and the head of VLC bitstream - t1 = vlc_tbl0[c_q + (vlc_val & 0x7F)]; - - // if context is zero, use one MEL event - if (c_q == 0 && x < width) //zero context - { - run -= 2; //subtract 2, since events number if multiplied by 2 - - // if event is 0, discard decoded t1 - t1 = (run == -1) ? t1 : 0; - - if (run < 0) // have we consumed all events in a run - run = mel_get_run(&mel); // if yes, then get another run - } - t1 = x < width ? t1 : 0; - //run -= (c_q == 0 && x < width) ? 2 : 0; - //t1 = (c_q != 0 || run == -1) ? t1 : 0; - //if (run < 0) - // run = mel_get_run(&mel); // get another run - sp[2] = t1; - x += 2; - - //prepare context for the next quad, eqn. 1 in ITU T.814 - c_q = ((t1 & 0x10U) << 3) | ((t1 & 0xE0U) << 2); - - //remove data from vlc stream, if qinf is not used, cwdlen is 0 - vlc_val = rev_advance(&vlc, t1 & 0x7); - - // decode u - ///////////// - // uvlc_mode is made up of u_offset bits from the quad pair - ui32 uvlc_mode = ((t0 & 0x8U) << 3) | ((t1 & 0x8U) << 4); - if (uvlc_mode == 0xc0)// if both u_offset are set, get an event from - { // the MEL run of events - run -= 2; //subtract 2, since events number if multiplied by 2 - - uvlc_mode += (run == -1) ? 0x40 : 0; // increment uvlc_mode by - // is 0x40 - - if (run < 0)//if run is consumed (run is -1 or -2), get another run - run = mel_get_run(&mel); - } - //run -= (uvlc_mode == 0xc0) ? 2 : 0; - //uvlc_mode += (uvlc_mode == 0xc0 && run == -1) ? 0x40 : 0; - //if (run < 0) - // run = mel_get_run(&mel); // get another run - - //decode uvlc_mode to get u for both quads - ui32 uvlc_entry = uvlc_tbl0[uvlc_mode + (vlc_val & 0x3F)]; - //remove total prefix length - vlc_val = rev_advance(&vlc, uvlc_entry & 0x7); - uvlc_entry >>= 3; - //extract suffixes for quad 0 and 1 - ui32 len = uvlc_entry & 0xF; //suffix length for 2 quads - ui32 tmp = vlc_val & ((1 << len) - 1); //suffix value for 2 quads - vlc_val = rev_advance(&vlc, len); - ojph_unused(vlc_val); //static code analysis: unused value - uvlc_entry >>= 4; - // quad 0 length - len = uvlc_entry & 0x7; // quad 0 suffix length - uvlc_entry >>= 3; - ui16 u_q = (ui16)(1 + (uvlc_entry&7) + (tmp&~(0xFFU<> 3) + (tmp >> len)); //kappa == 1 - sp[3] = u_q; - } - sp[0] = sp[1] = 0; - - //non initial quad rows - for (ui32 y = 2; y < height; y += 2) - { - c_q = 0; // context - ui16 *sp = scratch + (y >> 1) * sstr; // this row of quads - - for (ui32 x = 0; x < width; sp += 4) - { - // decode VLC - ///////////// + for (ui32 x = 0; x <= width; x += 16, vp += 8, sp += 16, vp_32 += 8) { + __m128i v = _mm_loadu_si128((__m128i*)vp); + __m128i v_p1 = _mm_loadu_si128((__m128i*)(vp + 1)); + v = _mm_or_si128(v, v_p1); - // sigma_q (n, ne, nf) - c_q |= ((sp[0 - (si32)sstr] & 0xA0U) << 2); - c_q |= ((sp[2 - (si32)sstr] & 0x20U) << 4); + __m256i v_avx = _mm256_cvtepu16_epi32(v); + v_avx = avx2_lzcnt_epi32(v_avx); + v_avx = _mm256_sub_epi32(avx_31, v_avx); - // first quad - vlc_val = rev_fetch(&vlc); + __m256i inf_u_q = _mm256_loadu_si256((__m256i*)sp); + __m256i gamma = _mm256_and_si256(inf_u_q, avx_f0); + __m256i w0 = _mm256_sub_epi32(gamma, avx_1); + gamma = _mm256_and_si256(gamma, w0); + gamma = _mm256_cmpeq_epi32(gamma, avx_0); - //decode VLC using the context c_q and the head of VLC bitstream - ui16 t0 = vlc_tbl1[ c_q + (vlc_val & 0x7F) ]; + v_avx = _mm256_andnot_si256(gamma, v_avx); + v_avx = _mm256_max_epi32(v_avx, avx_1); - // if context is zero, use one MEL event - if (c_q == 0) //zero context - { - run -= 2; //subtract 2, since events number is multiplied by 2 + inf_u_q = _mm256_srli_epi32(inf_u_q, 16); + v_avx = _mm256_add_epi32(inf_u_q, v_avx); - // Is the run terminated in 1? if so, use decoded VLC code, - // otherwise, discard decoded data, since we will decoded again - // using a different context - t0 = (run == -1) ? t0 : 0; + w0 = _mm256_cmpgt_epi32(v_avx, avx_mmsbp2); + if (!_mm256_testz_si256(w0, w0)) { + return false; + } - // is run -1 or -2? this means a run has been consumed - if (run < 0) - run = mel_get_run(&mel); // get another run + _mm256_storeu_si256((__m256i*)vp_32, v_avx); } - //run -= (c_q == 0) ? 2 : 0; - //t0 = (c_q != 0 || run == -1) ? t0 : 0; - //if (run < 0) - // run = mel_get_run(&mel); // get another run - sp[0] = t0; - x += 2; - - // prepare context for the next quad; eqn. 2 in ITU T.814 - // sigma_q (w, sw) - c_q = ((t0 & 0x40U) << 2) | ((t0 & 0x80U) << 1); - // sigma_q (nw) - c_q |= sp[0 - (si32)sstr] & 0x80; - // sigma_q (n, ne, nf) - c_q |= ((sp[2 - (si32)sstr] & 0xA0U) << 2); - c_q |= ((sp[4 - (si32)sstr] & 0x20U) << 4); - - //remove data from vlc stream (0 bits are removed if vlc is unused) - vlc_val = rev_advance(&vlc, t0 & 0x7); - - //second quad - ui16 t1 = 0; - - //decode VLC using the context c_q and the head of VLC bitstream - t1 = vlc_tbl1[ c_q + (vlc_val & 0x7F)]; + } - // if context is zero, use one MEL event - if (c_q == 0 && x < width) //zero context - { - run -= 2; //subtract 2, since events number if multiplied by 2 + ui16 *vp = v_n_scratch; + ui32* vp_32 = v_n_scratch_32; + ui16 *sp = scratch + (y >> 1) * sstr; + ui32 *dp = decoded_data + y * stride; + vp[0] = 2; // for easy calculation of emax - // if event is 0, discard decoded t1 - t1 = (run == -1) ? t1 : 0; + for (ui32 x = 0; x < width; x += 8, sp += 8, vp += 4, dp += 8, vp_32 += 4) { + ////process four quads + __m128i inf_u_q = _mm_loadu_si128((__m128i*)sp); + __m128i U_q = _mm_loadu_si128((__m128i*)vp_32); - if (run < 0) // have we consumed all events in a run - run = mel_get_run(&mel); // if yes, then get another run - } - t1 = x < width ? t1 : 0; - //run -= (c_q == 0 && x < width) ? 2 : 0; - //t1 = (c_q != 0 || run == -1) ? t1 : 0; - //if (run < 0) - // run = mel_get_run(&mel); // get another run - sp[2] = t1; - x += 2; + __m128i vn = _mm_set1_epi16(2); + __m256i row = decode_four_quad16(inf_u_q, U_q, dbuf, limit, pos, p, vn); - // partial c_q, will be completed when we process the next quad - // sigma_q (w, sw) - c_q = ((t1 & 0x40U) << 2) | ((t1 & 0x80U) << 1); - // sigma_q (nw) - c_q |= sp[2 - (si32)sstr] & 0x80; + __m128i w = _mm_cvtsi32_si128(*(unsigned short const*)(vp)); + _mm_storeu_si128((__m128i*)vp, _mm_or_si128(w, vn)); - //remove data from vlc stream, if qinf is not used, cwdlen is 0 - vlc_val = rev_advance(&vlc, t1 & 0x7); + __m256i w0 = _mm256_shuffle_epi8(row, _mm256_set_epi16(0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1, 0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1)); + __m256i w1 = _mm256_shuffle_epi8(row, _mm256_set_epi16(0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1, 0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1)); - // decode u - ///////////// - // uvlc_mode is made up of u_offset bits from the quad pair - ui32 uvlc_mode = ((t0 & 0x8U) << 3) | ((t1 & 0x8U) << 4); - ui32 uvlc_entry = uvlc_tbl1[uvlc_mode + (vlc_val & 0x3F)]; - //remove total prefix length - vlc_val = rev_advance(&vlc, uvlc_entry & 0x7); - uvlc_entry >>= 3; - //extract suffixes for quad 0 and 1 - ui32 len = uvlc_entry & 0xF; //suffix length for 2 quads - ui32 tmp = vlc_val & ((1 << len) - 1); //suffix value for 2 quads - vlc_val = rev_advance(&vlc, len); - ojph_unused(vlc_val); //static code analysis: unused value - uvlc_entry >>= 4; - // quad 0 length - len = uvlc_entry & 0x7; // quad 0 suffix length - uvlc_entry >>= 3; - ui16 u_q = (ui16)((uvlc_entry & 7) + (tmp & ~(0xFFU << len))); - sp[1] = u_q; - u_q = (ui16)((uvlc_entry >> 3) + (tmp >> len)); // u_q - sp[3] = u_q; + _mm256_storeu_si256((__m256i*)dp, w0); + _mm256_storeu_si256((__m256i*)(dp + stride), w1); } - sp[0] = sp[1] = 0; } - } + return true; + } - // step2 we decode magsgn - // mmsbp2 equals K_max + 1 (we decode up to K_max bits + 1 sign bit) - // The 32 bit path decode 16 bits data, for which one would think - // 16 bits are enough, because we want to put in the center of the - // bin. - // If you have mmsbp2 equals 16 bit, and reversible coding, and - // no bitplanes are missing, then we can decoding using the 16 bit - // path, but we are not doing this here. - if (mmsbp2 >= 16) - { - // We allocate a scratch row for storing v_n values. - // We have 512 quads horizontally. - // We may go beyond the last entry by up to 4 entries. - // Here we allocate additional 8 entries. - // There are two rows in this structure, the bottom - // row is used to store processed entries. + //************************************************************************/ + /** @brief Step-2 MagSgn decode for the 32-bit (2-quad) path. + * + * Outlined for the same reason as decode_cb_step2_16bit: it keeps the + * always-inlined decode_two_quad32_avx2 kernel in its own register + * allocation scope, isolated from step-1 and from the 16-bit path. + * Returns false on a precision-overflow error, true otherwise. + */ + OJPH_NO_INLINE + bool decode_cb_step2_32bit(ui16* scratch, ui32* decoded_data, + ui8* coded_data, ui32 width, ui32 height, + ui32 stride, ui32 sstr, ui32 p, ui32 mmsbp2, + int lcup, int scup) + { const int v_n_size = 512 + 16; - ui32 v_n_scratch[2 * v_n_size] = {0}; // 4+ kB +#ifdef __MINGW64__ + ui32 v_n_scratch[2 * v_n_size] = {0}; +#else + ui32 v_n_scratch[2 * v_n_size]; + memset(v_n_scratch + (width >> 1) + 2, 0, 14 * sizeof(ui32)); +#endif - frwd_struct_avx2 magsgn; - frwd_init<0xFF>(&magsgn, coded_data, lcup - scup); + // maximum consumable MagSgn bits: 4096 samples x (mmsbp2 <= 32) bits + const ui32 dbuf_cap = 4096 * 32 / 8; + ui8 dbuf[dbuf_cap + 72]; + ui32 limit = destuff_frwd<0xFF>(coded_data, lcup - scup, dbuf, dbuf_cap); + ui32 pos = 0; const __m256i avx_mmsbp2 = _mm256_set1_epi32((int)mmsbp2); @@ -1461,7 +1078,7 @@ namespace ojph { return false; } - __m256i row = decode_two_quad32_avx2(inf_u_q, U_q, &magsgn, p, vn); + __m256i row = decode_two_quad32_avx2(inf_u_q, U_q, dbuf, limit, pos, p, vn); row = _mm256_permutevar8x32_epi32(row, _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); _mm_store_si128((__m128i*)dp, _mm256_castsi256_si128(row)); _mm_store_si128((__m128i*)(dp + stride), _mm256_extracti128_si256(row, 0x1)); @@ -1527,7 +1144,7 @@ namespace ojph { __m256i U_q = _mm256_castsi128_si256(_mm_loadl_epi64((__m128i*)(vp + v_n_size))); U_q = _mm256_permutevar8x32_epi32(U_q, _mm256_setr_epi32(0, 0, 0, 0, 1, 1, 1, 1)); - __m256i row = decode_two_quad32_avx2(inf_u_q, U_q, &magsgn, p, vn); + __m256i row = decode_two_quad32_avx2(inf_u_q, U_q, dbuf, limit, pos, p, vn); row = _mm256_permutevar8x32_epi32(row, _mm256_setr_epi32(0, 2, 4, 6, 1, 3, 5, 7)); _mm_store_si128((__m128i*)dp, _mm256_castsi256_si128(row)); _mm_store_si128((__m128i*)(dp + stride), _mm256_extracti128_si256(row, 0x1)); @@ -1537,133 +1154,27 @@ namespace ojph { _mm_storeu_si128((__m128i*)vp, w0); } } - } - else { - - // reduce bitplane by 16 because we now have 16 bits instead of 32 - p -= 16; - - // We allocate a scratch row for storing v_n values. - // We have 512 quads horizontally. - // We may go beyond the last entry by up to 8 entries. - // Therefore we allocate additional 8 entries. - // There are two rows in this structure, the bottom - // row is used to store processed entries. - const int v_n_size = 512 + 16; - ui16 v_n_scratch[v_n_size] = {0}; // 1+ kB - ui32 v_n_scratch_32[v_n_size] = {0}; // 2+ kB + return true; + } - frwd_struct_avx2 magsgn; - frwd_init<0xFF>(&magsgn, coded_data, lcup - scup); - - { - ui16 *sp = scratch; - ui16 *vp = v_n_scratch; - ui32 *dp = decoded_data; - vp[0] = 2; // for easy calculation of emax - - for (ui32 x = 0; x < width; x += 8, sp += 8, vp += 4, dp += 8) { - ////process four quads - __m128i inf_u_q = _mm_loadu_si128((__m128i*)sp); - __m128i U_q = _mm_srli_epi32(inf_u_q, 16); - __m128i w = _mm_cmpgt_epi32(U_q, _mm_set1_epi32((int)mmsbp2)); - if (!_mm_testz_si128(w, w)) { - return false; - } - - __m128i vn = _mm_set1_epi16(2); - __m256i row = decode_four_quad16(inf_u_q, U_q, &magsgn, p, vn); - - w = _mm_cvtsi32_si128(*(unsigned short const*)(vp)); - _mm_storeu_si128((__m128i*)vp, _mm_or_si128(w, vn)); - - __m256i w0 = _mm256_shuffle_epi8(row, _mm256_set_epi16(0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1, 0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1)); - __m256i w1 = _mm256_shuffle_epi8(row, _mm256_set_epi16(0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1, 0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1)); - - _mm256_storeu_si256((__m256i*)dp, w0); - _mm256_storeu_si256((__m256i*)(dp + stride), w1); - } - } - - for (ui32 y = 2; y < height; y += 2) { - { - // perform 15 - count_leading_zeros(*vp) here - ui16 *vp = v_n_scratch; - ui32 *vp_32 = v_n_scratch_32; - - ui16* sp = scratch + (y >> 1) * sstr; - const __m256i avx_mmsbp2 = _mm256_set1_epi32((int)mmsbp2); - const __m256i avx_31 = _mm256_set1_epi32(31); - const __m256i avx_f0 = _mm256_set1_epi32(0xF0); - const __m256i avx_1 = _mm256_set1_epi32(1); - const __m256i avx_0 = _mm256_setzero_si256(); - - for (ui32 x = 0; x <= width; x += 16, vp += 8, sp += 16, vp_32 += 8) { - __m128i v = _mm_loadu_si128((__m128i*)vp); - __m128i v_p1 = _mm_loadu_si128((__m128i*)(vp + 1)); - v = _mm_or_si128(v, v_p1); - - __m256i v_avx = _mm256_cvtepu16_epi32(v); - v_avx = avx2_lzcnt_epi32(v_avx); - v_avx = _mm256_sub_epi32(avx_31, v_avx); - - __m256i inf_u_q = _mm256_loadu_si256((__m256i*)sp); - __m256i gamma = _mm256_and_si256(inf_u_q, avx_f0); - __m256i w0 = _mm256_sub_epi32(gamma, avx_1); - gamma = _mm256_and_si256(gamma, w0); - gamma = _mm256_cmpeq_epi32(gamma, avx_0); - - v_avx = _mm256_andnot_si256(gamma, v_avx); - v_avx = _mm256_max_epi32(v_avx, avx_1); - - inf_u_q = _mm256_srli_epi32(inf_u_q, 16); - v_avx = _mm256_add_epi32(inf_u_q, v_avx); - - w0 = _mm256_cmpgt_epi32(v_avx, avx_mmsbp2); - if (!_mm256_testz_si256(w0, w0)) { - return false; - } - - _mm256_storeu_si256((__m256i*)vp_32, v_avx); - } - } - - ui16 *vp = v_n_scratch; - ui32* vp_32 = v_n_scratch_32; - ui16 *sp = scratch + (y >> 1) * sstr; - ui32 *dp = decoded_data + y * stride; - vp[0] = 2; // for easy calculation of emax - - for (ui32 x = 0; x < width; x += 8, sp += 8, vp += 4, dp += 8, vp_32 += 4) { - ////process four quads - __m128i inf_u_q = _mm_loadu_si128((__m128i*)sp); - __m128i U_q = _mm_loadu_si128((__m128i*)vp_32); - - __m128i vn = _mm_set1_epi16(2); - __m256i row = decode_four_quad16(inf_u_q, U_q, &magsgn, p, vn); - - __m128i w = _mm_cvtsi32_si128(*(unsigned short const*)(vp)); - _mm_storeu_si128((__m128i*)vp, _mm_or_si128(w, vn)); - - __m256i w0 = _mm256_shuffle_epi8(row, _mm256_set_epi16(0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1, 0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1)); - __m256i w1 = _mm256_shuffle_epi8(row, _mm256_set_epi16(0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1, 0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1)); - - _mm256_storeu_si256((__m256i*)dp, w0); - _mm256_storeu_si256((__m256i*)(dp + stride), w1); - } - } - - // increase bitplane back by 16 because we need to process 32 bits - p += 16; - } - - if (num_passes > 1) - { - // We use scratch again, we can divide it into multiple regions - // sigma holds all the significant samples, and it cannot - // be modified after it is set. it will be used during the - // Magnitude Refinement Pass - ui16* const sigma = scratch; + //************************************************************************/ + /** @brief Significance-Propagation and Magnitude-Refinement passes. + * + * Outlined from ojph_decode_codeblock_avx2 so the (lossless cleanup-only) + * common path does not pay the register-allocation cost of this ~375-line + * block. Only runs when num_passes > 1. + */ + OJPH_NO_INLINE + void decode_cb_spp_mrp(ui16* scratch, ui32* decoded_data, ui8* coded_data, + ui32 width, ui32 height, ui32 stride, ui32 sstr, + ui32 p, ui32 num_passes, ui32 lengths1, + ui32 lengths2, bool stripe_causal) + { + // We use scratch again, we can divide it into multiple regions + // sigma holds all the significant samples, and it cannot + // be modified after it is set. it will be used during the + // Magnitude Refinement Pass + ui16* const sigma = scratch; ui32 mstr = (width + 3u) >> 2; // divide by 4, since each // ui16 contains 4 columns @@ -1702,7 +1213,8 @@ namespace ojph { __m128i r = _mm_or_si128(t0, t1); r = _mm_shuffle_epi8(r, shuffle_mask); - *(ui32*)dp = (ui32)_mm_extract_epi32(r, 0); + ui32 t = (ui32)_mm_extract_epi32(r, 0); + memcpy(dp, &t, 4); } dp[0] = 0; // set an extra entry on the right with 0 } @@ -1729,8 +1241,13 @@ namespace ojph { // We add an extra 8 entries, just in case we need more ui16 prev_row_sig[256 + 8] = {0}; // 528 Bytes - frwd_struct_avx2 sigprop; - frwd_init<0>(&sigprop, coded_data + lengths1, (int)lengths2); + // maximum consumable SPP bits: 4096 samples x 2 bits (one + // significance bit and one sign bit per sample) + const ui32 spp_cap = 4096 * 2 / 8; + ui8 spp_buf[spp_cap + 72]; + ui32 spp_limit = destuff_frwd<0>(coded_data + lengths1, + (int)lengths2, spp_buf, spp_cap); + ui32 spp_pos = 0; for (ui32 y = 0; y < height; y += 4) { @@ -1769,13 +1286,14 @@ namespace ojph { // We need data for at least 5 columns out of 8. // Therefore loading 32 bits is easier than loading 16 bits // twice. - ui32 ps = *(ui32*)prev_sig; - ui32 ns = *(ui32*)(cur_sig + mstr); + ui32 ps, ns, cs; + memcpy(&ps, prev_sig, 4); + memcpy(&ns, cur_sig + mstr, 4); ui32 u = (ps & 0x88888888) >> 3; // the row on top if (!stripe_causal) u |= (ns & 0x11111111) << 3; // the row below - ui32 cs = *(ui32*)cur_sig; + memcpy(&cs, cur_sig, 4); // vertical integration ui32 mbr = cs; // this sig. info. mbr |= (cs & 0x77777777) << 1; //above neighbors @@ -1795,8 +1313,7 @@ namespace ojph { ui32 new_sig = mbr; if (new_sig) { - __m128i cwd_vec = frwd_fetch<0>(&sigprop); - ui32 cwd = (ui32)_mm_extract_epi16(cwd_vec, 0); + ui64 cwd = dfetch64(spp_buf, spp_limit, spp_pos); ui32 cnt = 0; ui32 col_mask = 0xFu; @@ -1858,7 +1375,8 @@ namespace ojph { if (new_sig) { - cwd |= (ui32)_mm_extract_epi16(cwd_vec, 1) << (16 - cnt); + // the sign bits sit right after the cnt consumed bits + // of cwd; no reassembly is needed // Spread new_sig, such that each bit is in one byte with a // value of 0 if new_sig bit is 0, and 0xFF if new_sig is 1 @@ -1884,7 +1402,7 @@ namespace ojph { // Spread cwd, such that each bit is in one byte // with a value of 0 or 1. - cwd_vec = _mm_set1_epi16((si16)cwd); + __m128i cwd_vec = _mm_set1_epi16((si16)cwd); cwd_vec = _mm_shuffle_epi8(cwd_vec, _mm_set_epi8(1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0)); cwd_vec = _mm_and_si128(cwd_vec, @@ -1927,7 +1445,7 @@ namespace ojph { m = _mm_add_epi32(m, _mm_set1_epi32(1)); } } - frwd_advance(&sigprop, cnt); + spp_pos += cnt; } new_sig |= cs; @@ -1948,8 +1466,13 @@ namespace ojph { // We perform Magnitude Refinement Pass here if (num_passes > 2) { - rev_struct magref; - rev_init_mrp(&magref, coded_data, (int)lengths1, (int)lengths2); + // de-stuff the MRP segment; consumption is at most 1 bit per + // sample = 512 bytes, so 1024 bytes always suffice + const ui32 mrp_cap = 1024; + ui8 mrp_buf[mrp_cap + 72]; + ui32 mrp_limit = destuff_mrp(coded_data, (int)lengths1, + (int)lengths2, mrp_buf, mrp_cap); + ui32 mrp_pos = 0; for (ui32 y = 0; y < height; y += 4) { @@ -1959,7 +1482,7 @@ namespace ojph { { //Process one entry from sigma array at a time // Each nibble (4 bits) in the sigma array represents 4 rows, - ui32 cwd = rev_fetch_mrp(&magref); // get 32 bit data + ui64 cwd = dfetch64(mrp_buf, mrp_limit, mrp_pos); ui16 sig = *cur_sig++; // 16 bit that will be processed now int total_bits = 0; if (sig) // if any of the 32 bits are set @@ -2030,12 +1553,421 @@ namespace ojph { } } // consume data according to the number of bits set - rev_advance_mrp(&magref, (ui32)total_bits); + mrp_pos += (ui32)total_bits; + } + } + } + } + + //************************************************************************/ + /** @brief Step-1: decode the VLC and MEL segments into the scratch buffer. + * + * Outlined so the serial scalar VLC/MEL chain gets its own register + * allocation, isolated from the step-2 / SPP-MRP code paths. + */ + OJPH_NO_INLINE + void decode_cb_step1_vlc(ui16* scratch, ui8* coded_data, int lcup, + int scup, ui32 width, ui32 height, ui32 sstr) + { + // init structures + dec_mel_st mel; + mel_init(&mel, coded_data, lcup, scup); + + // de-stuff the VLC segment; its size is at most scup - 1 < 4095 + // bytes (scup is a 12-bit value), and consumption per quad pair is + // at most 7 + 7 + 30 bits, so 4096 bytes always suffice + const ui32 vlc_cap = 4096; + ui8 vlc_buf[vlc_cap + 72]; + ui32 vlc_limit = destuff_vlc(coded_data, lcup, scup, + vlc_buf, vlc_cap); + ui32 vlc_off = 0; + ui32 vlc_bits = 0; + + int run = mel_get_run(&mel); // decode runs of events from MEL bitstrm + // data represented as runs of 0 events + // See mel_decode description + + ui64 vlc_val = 0; + ui32 c_q = 0; + ui16 *sp = scratch; + //initial quad row + for (ui32 x = 0; x < width; sp += 4) + { + // decode VLC + ///////////// + + // first quad + drefill(vlc_val, vlc_bits, vlc_off, vlc_buf, vlc_limit); + + //decode VLC using the context c_q and the head of VLC bitstream + ui16 t0 = vlc_tbl0[ c_q + (vlc_val & 0x7F) ]; + + // if context is zero, use one MEL event + if (c_q == 0) //zero context + { + run -= 2; //subtract 2, since events number if multiplied by 2 + + // Is the run terminated in 1? if so, use decoded VLC code, + // otherwise, discard decoded data, since we will decoded again + // using a different context + t0 = (run == -1) ? t0 : 0; + + // is run -1 or -2? this means a run has been consumed + if (run < 0) + run = mel_get_run(&mel); // get another run + } + //run -= (c_q == 0) ? 2 : 0; + //t0 = (c_q != 0 || run == -1) ? t0 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + sp[0] = t0; + x += 2; + + // prepare context for the next quad; eqn. 1 in ITU T.814 + c_q = ((t0 & 0x10U) << 3) | ((t0 & 0xE0U) << 2); + + //remove data from vlc stream (0 bits are removed if vlc is not used) + dconsume(vlc_val, vlc_bits, t0 & 0x7u); + + //second quad + ui16 t1 = 0; + + //decode VLC using the context c_q and the head of VLC bitstream + t1 = vlc_tbl0[c_q + (vlc_val & 0x7F)]; + + // if context is zero, use one MEL event + if (c_q == 0 && x < width) //zero context + { + run -= 2; //subtract 2, since events number if multiplied by 2 + + // if event is 0, discard decoded t1 + t1 = (run == -1) ? t1 : 0; + + if (run < 0) // have we consumed all events in a run + run = mel_get_run(&mel); // if yes, then get another run + } + t1 = x < width ? t1 : 0; + //run -= (c_q == 0 && x < width) ? 2 : 0; + //t1 = (c_q != 0 || run == -1) ? t1 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + sp[2] = t1; + x += 2; + + //prepare context for the next quad, eqn. 1 in ITU T.814 + c_q = ((t1 & 0x10U) << 3) | ((t1 & 0xE0U) << 2); + + //remove data from vlc stream, if qinf is not used, cwdlen is 0 + dconsume(vlc_val, vlc_bits, t1 & 0x7u); + + // decode u + ///////////// + // uvlc_mode is made up of u_offset bits from the quad pair + ui32 uvlc_mode = ((t0 & 0x8U) << 3) | ((t1 & 0x8U) << 4); + if (uvlc_mode == 0xc0)// if both u_offset are set, get an event from + { // the MEL run of events + run -= 2; //subtract 2, since events number if multiplied by 2 + + uvlc_mode += (run == -1) ? 0x40 : 0; // increment uvlc_mode by + // is 0x40 + + if (run < 0)//if run is consumed (run is -1 or -2), get another run + run = mel_get_run(&mel); + } + //run -= (uvlc_mode == 0xc0) ? 2 : 0; + //uvlc_mode += (uvlc_mode == 0xc0 && run == -1) ? 0x40 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + + //decode uvlc_mode to get u for both quads + ui32 uvlc_entry = uvlc_tbl0[uvlc_mode + (vlc_val & 0x3F)]; + //remove total prefix length + dconsume(vlc_val, vlc_bits, uvlc_entry & 0x7u); + uvlc_entry >>= 3; + //extract suffixes for quad 0 and 1 + ui32 len = uvlc_entry & 0xF; //suffix length for 2 quads + ui32 tmp = (ui32)vlc_val & ((1u << len) - 1); //suffix value, 2 quads + dconsume(vlc_val, vlc_bits, len); + uvlc_entry >>= 4; + // quad 0 length + len = uvlc_entry & 0x7; // quad 0 suffix length + uvlc_entry >>= 3; + ui16 u_q = (ui16)(1 + (uvlc_entry&7) + (tmp&~(0xFFU<> 3) + (tmp >> len)); //kappa == 1 + sp[3] = u_q; + } + sp[0] = sp[1] = 0; + + //non initial quad rows + for (ui32 y = 2; y < height; y += 2) + { + c_q = 0; // context + ui16 *sp = scratch + (y >> 1) * sstr; // this row of quads + + for (ui32 x = 0; x < width; sp += 4) + { + // decode VLC + ///////////// + + // sigma_q (n, ne, nf) + c_q |= ((sp[0 - (si32)sstr] & 0xA0U) << 2); + c_q |= ((sp[2 - (si32)sstr] & 0x20U) << 4); + + // first quad + drefill(vlc_val, vlc_bits, vlc_off, vlc_buf, vlc_limit); + + //decode VLC using the context c_q and the head of VLC bitstream + ui16 t0 = vlc_tbl1[ c_q + (vlc_val & 0x7F) ]; + + // if context is zero, use one MEL event + if (c_q == 0) //zero context + { + run -= 2; //subtract 2, since events number is multiplied by 2 + + // Is the run terminated in 1? if so, use decoded VLC code, + // otherwise, discard decoded data, since we will decoded again + // using a different context + t0 = (run == -1) ? t0 : 0; + + // is run -1 or -2? this means a run has been consumed + if (run < 0) + run = mel_get_run(&mel); // get another run + } + //run -= (c_q == 0) ? 2 : 0; + //t0 = (c_q != 0 || run == -1) ? t0 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + sp[0] = t0; + x += 2; + + // prepare context for the next quad; eqn. 2 in ITU T.814 + // sigma_q (w, sw) + c_q = ((t0 & 0x40U) << 2) | ((t0 & 0x80U) << 1); + // sigma_q (nw) + c_q |= sp[0 - (si32)sstr] & 0x80; + // sigma_q (n, ne, nf) + c_q |= ((sp[2 - (si32)sstr] & 0xA0U) << 2); + c_q |= ((sp[4 - (si32)sstr] & 0x20U) << 4); + + //remove data from vlc stream (0 bits are removed if vlc is unused) + dconsume(vlc_val, vlc_bits, t0 & 0x7u); + + //second quad + ui16 t1 = 0; + + //decode VLC using the context c_q and the head of VLC bitstream + t1 = vlc_tbl1[ c_q + (vlc_val & 0x7F)]; + + // if context is zero, use one MEL event + if (c_q == 0 && x < width) //zero context + { + run -= 2; //subtract 2, since events number if multiplied by 2 + + // if event is 0, discard decoded t1 + t1 = (run == -1) ? t1 : 0; + + if (run < 0) // have we consumed all events in a run + run = mel_get_run(&mel); // if yes, then get another run + } + t1 = x < width ? t1 : 0; + //run -= (c_q == 0 && x < width) ? 2 : 0; + //t1 = (c_q != 0 || run == -1) ? t1 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + sp[2] = t1; + x += 2; + + // partial c_q, will be completed when we process the next quad + // sigma_q (w, sw) + c_q = ((t1 & 0x40U) << 2) | ((t1 & 0x80U) << 1); + // sigma_q (nw) + c_q |= sp[2 - (si32)sstr] & 0x80; + + //remove data from vlc stream, if qinf is not used, cwdlen is 0 + dconsume(vlc_val, vlc_bits, t1 & 0x7u); + + // decode u using wide UVLC table + ///////////// + ui32 uvlc_mode = (((t0 >> 3) & 1u) | (((t1 >> 3) & 1u) << 1)); + ui32 uvlc_entry = + uvlc_tbl1_wide[(uvlc_mode << 10) | (vlc_val & 0x3FF)]; + ui32 total_bits = uvlc_entry & 0x1F; + if (total_bits < 0x1F) { + sp[1] = (ui16)((uvlc_entry >> 5) & 0xFF); + sp[3] = (ui16)((uvlc_entry >> 13) & 0xFF); + dconsume(vlc_val, vlc_bits, total_bits); + } else { + uvlc_mode = ((t0 & 0x8U) << 3) | ((t1 & 0x8U) << 4); + uvlc_entry = uvlc_tbl1[uvlc_mode + (vlc_val & 0x3F)]; + dconsume(vlc_val, vlc_bits, uvlc_entry & 0x7u); + uvlc_entry >>= 3; + ui32 len = uvlc_entry & 0xF; + ui32 tmp = (ui32)vlc_val & ((1u << len) - 1); + dconsume(vlc_val, vlc_bits, len); + uvlc_entry >>= 4; + len = uvlc_entry & 0x7; + uvlc_entry >>= 3; + sp[1] = (ui16)((uvlc_entry & 7) + (tmp & ~(0xFFU << len))); + sp[3] = (ui16)((uvlc_entry >> 3) + (tmp >> len)); } } + sp[0] = sp[1] = 0; } + } + + bool ojph_decode_codeblock_avx2(ui8* coded_data, ui32* decoded_data, + ui32 missing_msbs, ui32 num_passes, + ui32 lengths1, ui32 lengths2, + ui32 width, ui32 height, ui32 stride, + bool stripe_causal) + { + static bool insufficient_precision = false; + static bool modify_code = false; + static bool truncate_spp_mrp = false; + + if (num_passes > 1 && lengths2 == 0) + { + OJPH_WARN(0x00010001, "A malformed codeblock that has more than " + "one coding pass, but zero length for " + "2nd and potential 3rd pass."); + num_passes = 1; } + if (num_passes > 3) + { + OJPH_WARN(0x00010002, "We do not support more than 3 coding passes; " + "This codeblocks has %d passes.", + num_passes); + return false; + } + + if (missing_msbs > 30) // p < 0 + { + if (insufficient_precision == false) + { + insufficient_precision = true; + OJPH_WARN(0x00010003, "32 bits are not enough to decode this " + "codeblock. This message will not be " + "displayed again."); + } + return false; + } + else if (missing_msbs == 30) // p == 0 + { // not enough precision to decode and set the bin center to 1 + if (modify_code == false) { + modify_code = true; + OJPH_WARN(0x00010004, "Not enough precision to decode the cleanup " + "pass. The code can be modified to support " + "this case. This message will not be " + "displayed again."); + } + return false; // 32 bits are not enough to decode this + } + else if (missing_msbs == 29) // if p is 1, then num_passes must be 1 + { + if (num_passes > 1) { + num_passes = 1; + if (truncate_spp_mrp == false) { + truncate_spp_mrp = true; + OJPH_WARN(0x00010005, "Not enough precision to decode the SgnProp " + "nor MagRef passes; both will be skipped. " + "This message will not be displayed " + "again."); + } + } + } + ui32 p = 30 - missing_msbs; // The least significant bitplane for CUP + // There is a way to handle the case of p == 0, but a different path + // is required + + if (lengths1 < 2) + { + OJPH_WARN(0x00010006, "Wrong codeblock length."); + return false; + } + + // read scup and fix the bytes there + int lcup, scup; + lcup = (int)lengths1; // length of CUP + //scup is the length of MEL + VLC + scup = (((int)coded_data[lcup-1]) << 4) + (coded_data[lcup-2] & 0xF); + if (scup < 2 || scup > lcup || scup > 4079) //something is wrong + return false; + + // The temporary storage scratch holds two types of data in an + // interleaved fashion. The interleaving allows us to use one + // memory pointer. + // We have one entry for a decoded VLC code, and one entry for UVLC. + // Entries are 16 bits each, corresponding to one quad, + // but since we want to use XMM registers of the SSE family + // of SIMD; we allocated 16 bytes or more per quad row; that is, + // the width is no smaller than 16 bytes (or 8 entries), and the + // height is 512 quads + // Each VLC entry contains, in the following order, starting + // from MSB + // e_k (4bits), e_1 (4bits), rho (4bits), useless for step 2 (4bits) + // Each entry in UVLC contains u_q + // One extra row to handle the case of SPP propagating downwards + // when codeblock width is 4 + // We need an extra two entries (one inf and one u_q) beyond + // the last column. + // If the block width is 4 (2 quads), then we use sstr of 8 + // (enough for 4 quads). If width is 8 (4 quads) we use + // sstr is 16 (enough for 8 quads). For a width of 16 (8 + // quads), we use 24 (enough for 12 quads). + ui32 sstr = ((width + 2u) + 7u) & ~7u; // multiples of 8 + +#ifdef __MINGW64__ + ui16 scratch[8 * 513] = {0}; +#else + ui16 scratch[8 * 513]; + ui32 quad_rows = (height + 1u) >> 1; + size_t scratch_zero = (size_t)(quad_rows + 1) * sstr; + if (scratch_zero > 8 * 513) scratch_zero = 8 * 513; + memset(scratch, 0, scratch_zero * sizeof(ui16)); +#endif + + assert((stride & 0x3) == 0); + + ui32 mmsbp2 = missing_msbs + 2; + + // The cleanup pass is decoded in two steps; in step one, + // the VLC and MEL segments are decoded, generating a record that + // has 2 bytes per quad. The 2 bytes contain, u, rho, e^1 & e^k. + // This information should be sufficient for the next step. + // In step 2, we decode the MagSgn segment. + + // step 1: decode VLC and MEL segments into scratch + decode_cb_step1_vlc(scratch, coded_data, lcup, scup, width, height, sstr); + + // step2 we decode magsgn + // mmsbp2 equals K_max + 1 (we decode up to K_max bits + 1 sign bit) + // The 32 bit path decode 16 bits data, for which one would think + // 16 bits are enough, because we want to put in the center of the + // bin. + // If you have mmsbp2 equals 16 bit, and reversible coding, and + // no bitplanes are missing, then we can decoding using the 16 bit + // path, but we are not doing this here. + if (mmsbp2 >= 16) + { + if (!decode_cb_step2_32bit(scratch, decoded_data, coded_data, + width, height, stride, sstr, p, mmsbp2, + lcup, scup)) + return false; + } + else { + if (!decode_cb_step2_16bit(scratch, decoded_data, coded_data, + width, height, stride, sstr, p, mmsbp2, + lcup, scup)) + return false; + } + + if (num_passes > 1) + decode_cb_spp_mrp(scratch, decoded_data, coded_data, width, height, + stride, sstr, p, num_passes, lengths1, lengths2, + stripe_causal); + return true; } } diff --git a/external/OpenJPH/src/core/coding/ojph_block_decoder_ssse3.cpp b/external/OpenJPH/src/core/coding/ojph_block_decoder_ssse3.cpp index 3e3e00e45..2a98bd70f 100644 --- a/external/OpenJPH/src/core/coding/ojph_block_decoder_ssse3.cpp +++ b/external/OpenJPH/src/core/coding/ojph_block_decoder_ssse3.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2022, Aous Naman +// Copyright (c) 2022, Aous Naman // Copyright (c) 2022, Kakadu Software Pty Ltd, Australia // Copyright (c) 2022, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -63,7 +63,7 @@ namespace ojph { * A number of events is decoded from the MEL bitstream ahead of time * and stored in run/num_runs. * Each run represents the number of zero events before a one event. - */ + */ struct dec_mel_st { dec_mel_st() : data(NULL), tmp(0), bits(0), size(0), unstuff(false), k(0), num_runs(0), runs(0) @@ -83,7 +83,7 @@ namespace ojph { //************************************************************************/ /** @brief Reads and unstuffs the MEL bitstream - * + * * This design needs more bytes in the codeblock buffer than the length * of the cleanup pass by up to 2 bytes. * @@ -101,14 +101,14 @@ namespace ojph { ui32 val = 0xFFFFFFFF; // feed in 0xFF if buffer is exhausted if (melp->size > 4) { // if there is data in the MEL segment - val = *(ui32*)melp->data; // read 32 bits from MEL data + memcpy(&val, melp->data, sizeof(val)); // read 32 bits from MEL data melp->data += 4; // advance pointer melp->size -= 4; // reduce counter } else if (melp->size > 0) { // 4 or less int i = 0; - while (melp->size > 1) { + while (melp->size > 1) { ui32 v = *melp->data++; // read one byte at a time ui32 m = ~(0xFFu << i); // mask of location val = (val & m) | (v << i);// put one byte in its correct location @@ -116,21 +116,21 @@ namespace ojph { i += 8; } // size equal to 1 - ui32 v = *melp->data++; // the one before the last is different + ui32 v = *melp->data++; // the one before the last is different v |= 0xF; // MEL and VLC segments can overlap ui32 m = ~(0xFFu << i); val = (val & m) | (v << i); --melp->size; } - + // next we unstuff them before adding them to the buffer int bits = 32 - melp->unstuff; // number of bits in val, subtract 1 if - // the previously read byte requires + // the previously read byte requires // unstuffing // data is unstuffed and accumulated in t // bits has the number of bits in t - ui32 t = val & 0xFF; + ui32 t = val & 0xFF; bool unstuff = ((val & 0xFF) == 0xFF); // true if we need unstuffing bits -= unstuff; // there is one less bit in t if unstuffing is needed t = t << (8 - unstuff); // move up to make room for the next byte @@ -157,14 +157,14 @@ namespace ojph { //************************************************************************/ /** @brief Decodes unstuffed MEL segment bits stored in tmp to runs - * + * * Runs are stored in "runs" and the number of runs in "num_runs". - * Each run represents a number of zero events that may or may not + * Each run represents a number of zero events that may or may not * terminate in a 1 event. * Each run is stored in 7 bits. The LSB is 1 if the run terminates in - * a 1 event, 0 otherwise. The next 6 bits, for the case terminating - * with 1, contain the number of consecutive 0 zero events * 2; for the - * case terminating with 0, they store (number of consecutive 0 zero + * a 1 event, 0 otherwise. The next 6 bits, for the case terminating + * with 1, contain the number of consecutive 0 zero events * 2; for the + * case terminating with 0, they store (number of consecutive 0 zero * events - 1) * 2. * A total of 6 bits (made up of 1 + 5) should have been enough. * @@ -189,7 +189,7 @@ namespace ojph { int run = 0; if (melp->tmp & (1ull<<63)) //The next bit to decode (stored in MSB) { //one is found - run = 1 << eval; + run = 1 << eval; run--; // consecutive runs of 0 events - 1 melp->k = melp->k + 1 < 12 ? melp->k + 1 : 12;//increment, max is 12 melp->tmp <<= 1; // consume one bit from tmp @@ -207,14 +207,14 @@ namespace ojph { eval = melp->num_runs * 7; // 7 bits per run melp->runs &= ~((ui64)0x3F << eval); // 6 bits are sufficient melp->runs |= ((ui64)run) << eval; // store the value in runs - melp->num_runs++; // increment count + melp->num_runs++; // increment count } } //************************************************************************/ /** @brief Initiates a dec_mel_st structure for MEL decoding and reads * some bytes in order to get the read address to a multiple - * of 4 + * of 4 * * @param [in] melp is a pointer to dec_mel_st structure * @param [in] bbuf is a pointer to byte buffer @@ -229,7 +229,7 @@ namespace ojph { melp->tmp = 0; // melp->unstuff = false; // no unstuffing melp->size = scup - 1; // size is the length of MEL+VLC-1 - melp->k = 0; // 0 for state + melp->k = 0; // 0 for state melp->num_runs = 0; // num_runs is 0 melp->runs = 0; // @@ -247,7 +247,7 @@ namespace ojph { int d_bits = 8 - melp->unstuff; //if unstuffing is needed, reduce by 1 melp->tmp = (melp->tmp << d_bits) | d; //store bits in tmp melp->bits += d_bits; //increment tmp by number of bits - melp->unstuff = ((d & 0xFF) == 0xFF); //true of next byte needs + melp->unstuff = ((d & 0xFF) == 0xFF); //true of next byte needs //unstuffing } melp->tmp <<= (64 - melp->bits); //push all the way up so the first bit @@ -259,7 +259,7 @@ namespace ojph { * MEL segment is decoded * * @param [in] melp is a pointer to dec_mel_st structure - */ + */ static inline int mel_get_run(dec_mel_st *melp) { @@ -275,7 +275,7 @@ namespace ojph { //************************************************************************/ /** @brief A structure for reading and unstuffing a segment that grows * backward, such as VLC and MRP - */ + */ struct rev_struct { rev_struct() : data(NULL), tmp(0), bits(0), size(0), unstuff(false) {} @@ -292,41 +292,41 @@ namespace ojph { /** @brief Read and unstuff data from a backwardly-growing segment * * This reader can read up to 8 bytes from before the VLC segment. - * Care must be taken not read from unreadable memory, causing a + * Care must be taken not read from unreadable memory, causing a * segmentation fault. - * + * * Note that there is another subroutine rev_read_mrp that is slightly * different. The other one fills zeros when the buffer is exhausted. * This one basically does not care if the bytes are consumed, because * any extra data should not be used in the actual decoding. * - * Unstuffing is needed to prevent sequences more than 0xFF8F from + * Unstuffing is needed to prevent sequences more than 0xFF8F from * appearing in the bits stream; since we are reading backward, we keep - * watch when a value larger than 0x8F appears in the bitstream. - * If the byte following this is 0x7F, we unstuff this byte (ignore the + * watch when a value larger than 0x8F appears in the bitstream. + * If the byte following this is 0x7F, we unstuff this byte (ignore the * MSB of that byte, which should be 0). * * @param [in] vlcp is a pointer to rev_struct structure */ - static inline + static inline void rev_read(rev_struct *vlcp) { //process 4 bytes at a time - if (vlcp->bits > 32) // if there are more than 32 bits in tmp, then + if (vlcp->bits > 32) // if there are more than 32 bits in tmp, then return; // reading 32 bits can overflow vlcp->tmp ui32 val = 0; //the next line (the if statement) needs to be tested first if (vlcp->size > 3) // if there are more than 3 bytes left in VLC { // (vlcp->data - 3) move pointer back to read 32 bits at once - val = *(ui32*)(vlcp->data - 3); // then read 32 bits + memcpy(&val, vlcp->data - 3, sizeof(val)); // then read 32 bits vlcp->data -= 4; // move data pointer back by 4 vlcp->size -= 4; // reduce available byte by 4 } else if (vlcp->size > 0) { // 4 or less int i = 24; - while (vlcp->size > 0) { + while (vlcp->size > 0) { ui32 v = *vlcp->data--; // read one byte at a time val |= (v << i); // put byte in its correct location --vlcp->size; @@ -361,7 +361,7 @@ namespace ojph { } //************************************************************************/ - /** @brief Initiates the rev_struct structure and reads a few bytes to + /** @brief Initiates the rev_struct structure and reads a few bytes to * move the read address to multiple of 4 * * There is another similar rev_init_mrp subroutine. The difference is @@ -374,7 +374,7 @@ namespace ojph { * @param [in] lcup is the length of MagSgn+MEL+VLC segments * @param [in] scup is the length of MEL+VLC segments */ - static inline + static inline void rev_init(rev_struct *vlcp, ui8* data, int lcup, int scup) { //first byte has only the upper 4 bits @@ -409,13 +409,13 @@ namespace ojph { } //************************************************************************/ - /** @brief Retrieves 32 bits from the head of a rev_struct structure + /** @brief Retrieves 32 bits from the head of a rev_struct structure * * By the end of this call, vlcp->tmp must have no less than 33 bits * * @param [in] vlcp is a pointer to rev_struct structure */ - static inline + static inline ui32 rev_fetch(rev_struct *vlcp) { if (vlcp->bits < 32) // if there are less then 32 bits, read more @@ -433,7 +433,7 @@ namespace ojph { * @param [in] vlcp is a pointer to rev_struct structure * @param [in] num_bits is the number of bits to be removed */ - static inline + static inline ui32 rev_advance(rev_struct *vlcp, ui32 num_bits) { assert(num_bits <= vlcp->bits); // vlcp->tmp must have more than num_bits @@ -453,7 +453,7 @@ namespace ojph { * * @param [in] mrp is a pointer to rev_struct structure */ - static inline + static inline void rev_read_mrp(rev_struct *mrp) { //process 4 bytes at a time @@ -462,14 +462,14 @@ namespace ojph { ui32 val = 0; if (mrp->size > 3) // If there are 3 byte or more { // (mrp->data - 3) move pointer back to read 32 bits at once - val = *(ui32*)(mrp->data - 3); // read 32 bits + memcpy(&val, mrp->data - 3, sizeof(val)); // read 32 bits mrp->data -= 4; // move back pointer mrp->size -= 4; // reduce count } else if (mrp->size > 0) { int i = 24; - while (mrp->size > 0) { + while (mrp->size > 0) { ui32 v = *mrp->data--; // read one byte at a time val |= (v << i); // put byte in its correct location --mrp->size; @@ -509,7 +509,7 @@ namespace ojph { * an architecture that read size must be compatible with the * alignment of the read address * - * There is another similar subroutine rev_init. This subroutine does + * There is another similar subroutine rev_init. This subroutine does * NOT skip the first 12 bits, and starts with unstuff set to true. * * @param [in] mrp is a pointer to rev_struct structure @@ -517,7 +517,7 @@ namespace ojph { * @param [in] lcup is the length of MagSgn+MEL+VLC segments * @param [in] len2 is the length of SPP+MRP segments */ - static inline + static inline void rev_init_mrp(rev_struct *mrp, ui8* data, int lcup, int len2) { mrp->data = data + lcup + len2 - 1; @@ -534,7 +534,7 @@ namespace ojph { for (int i = 0; i < num; ++i) { ui64 d; //read a byte, 0 if no more data - d = (mrp->size-- > 0) ? *mrp->data-- : 0; + d = (mrp->size-- > 0) ? *mrp->data-- : 0; //check if unstuffing is needed ui32 d_bits = 8 - ((mrp->unstuff && ((d & 0x7F) == 0x7F)) ? 1 : 0); mrp->tmp |= d << mrp->bits; // move data to vlcp->tmp @@ -545,13 +545,13 @@ namespace ojph { } //************************************************************************/ - /** @brief Retrieves 32 bits from the head of a rev_struct structure + /** @brief Retrieves 32 bits from the head of a rev_struct structure * * By the end of this call, mrp->tmp must have no less than 33 bits * * @param [in] mrp is a pointer to rev_struct structure */ - static inline + static inline ui32 rev_fetch_mrp(rev_struct *mrp) { if (mrp->bits < 32) // if there are less than 32 bits in mrp->tmp @@ -578,7 +578,7 @@ namespace ojph { } //************************************************************************/ - /** @brief State structure for reading and unstuffing of forward-growing + /** @brief State structure for reading and unstuffing of forward-growing * bitstreams; these are: MagSgn and SPP bitstreams */ struct frwd_struct_ssse3 { @@ -591,7 +591,7 @@ namespace ojph { //************************************************************************/ /** @brief Read and unstuffs 16 bytes from forward-growing bitstream - * + * * A template is used to accommodate a different requirement for * MagSgn and SPP bitstreams; in particular, when MagSgn bitstream is * consumed, 0xFF's are fed, while when SPP is exhausted 0's are fed in. @@ -608,7 +608,7 @@ namespace ojph { * */ template - static inline + static inline void frwd_read(frwd_struct_ssse3 *msp) { assert(msp->bits <= 128); @@ -629,19 +629,19 @@ namespace ojph { val = _mm_or_si128(t, val); // fill with 0xFF } else if (X == 0) - val = _mm_and_si128(validity, val); // fill with zeros + val = _mm_and_si128(validity, val); // fill with zeros else assert(0); __m128i ff_bytes; ff_bytes = _mm_cmpeq_epi8(val, all_xff); ff_bytes = _mm_and_si128(ff_bytes, validity); - ui32 flags = (ui32)_mm_movemask_epi8(ff_bytes); + ui32 flags = (ui32)_mm_movemask_epi8(ff_bytes); flags <<= 1; // unstuff following byte ui32 next_unstuff = flags >> 16; flags |= msp->unstuff; flags &= 0xFFFF; - while (flags) + while (flags) { // bit unstuffing occurs on average once every 256 bytes // therefore it is not an issue if it is a bit slow // here we process 16 bytes @@ -659,7 +659,7 @@ namespace ojph { t = _mm_srli_si128(t, 8); // 8 bytes left t = _mm_slli_epi64(t, 63); // keep the MSB only t = _mm_or_si128(t, c); // combine the above 3 steps - + val = _mm_or_si128(t, _mm_andnot_si128(m, val)); } @@ -689,7 +689,7 @@ namespace ojph { //************************************************************************/ /** @brief Initialize frwd_struct_ssse3 struct and reads some bytes - * + * * @tparam X is the value fed in when the bitstream is exhausted. * See frwd_read regarding the template * @param [in] msp is a pointer to frwd_struct_ssse3 @@ -697,7 +697,7 @@ namespace ojph { * @param [in] size is the number of byte in the bitstream */ template - static inline + static inline void frwd_init(frwd_struct_ssse3 *msp, const ui8* data, int size) { msp->data = data; @@ -718,7 +718,7 @@ namespace ojph { * @param [in] msp is a pointer to frwd_struct_ssse3 * @param [in] num_bits is the number of bit to consume */ - static inline + static inline void frwd_advance(frwd_struct_ssse3 *msp, ui32 num_bits) { assert(num_bits > 0 && num_bits <= msp->bits && num_bits < 128); @@ -784,7 +784,7 @@ namespace ojph { * @return __m128i decoded quad */ template - static inline + static inline __m128i decode_one_quad32(const __m128i inf_u_q, __m128i U_q, frwd_struct_ssse3* magsgn, ui32 p, __m128i& vn) { @@ -802,7 +802,7 @@ namespace ojph { { U_q = _mm_shuffle_epi32(U_q, _MM_SHUFFLE(N, N, N, N)); flags = _mm_mullo_epi16(flags, _mm_set_epi16(1,1,2,2,4,4,8,8)); - __m128i ms_vec = frwd_fetch<0xFF>(magsgn); + __m128i ms_vec = frwd_fetch<0xFF>(magsgn); // U_q holds U_q for this quad // flags has e_k, e_1, and rho such that e_k is sitting in the @@ -825,7 +825,7 @@ namespace ojph { // find the starting byte and starting bit __m128i byte_idx = _mm_srli_epi32(ex_sum, 3); __m128i bit_idx = _mm_and_si128(ex_sum, _mm_set1_epi32(7)); - byte_idx = _mm_shuffle_epi8(byte_idx, + byte_idx = _mm_shuffle_epi8(byte_idx, _mm_set_epi32(0x0C0C0C0C, 0x08080808, 0x04040404, 0x00000000)); byte_idx = _mm_add_epi32(byte_idx, _mm_set1_epi32(0x03020100)); __m128i d0 = _mm_shuffle_epi8(ms_vec, byte_idx); @@ -850,7 +850,7 @@ namespace ojph { __m128i twos = _mm_set1_epi32(2); __m128i U_q_m1 = _mm_sub_epi32(U_q, ones); U_q_m1 = _mm_and_si128(U_q_m1, _mm_set_epi32(0,0,0,0x1F)); - w0 = _mm_sub_epi32(twos, w0); + w0 = _mm_sub_epi32(twos, w0); shift = _mm_sll_epi32(w0, U_q_m1); // U_q_m1 must be no more than 31 ms_vec = _mm_and_si128(d0, _mm_sub_epi32(shift, ones)); @@ -869,10 +869,10 @@ namespace ojph { ms_vec = _mm_andnot_si128(insig, tvn); // significant only if (N == 0) // the compiler should remove one - tvn = _mm_shuffle_epi8(ms_vec, + tvn = _mm_shuffle_epi8(ms_vec, _mm_set_epi32(-1, -1, 0x0F0E0D0C, 0x07060504)); else if (N == 1) - tvn = _mm_shuffle_epi8(ms_vec, + tvn = _mm_shuffle_epi8(ms_vec, _mm_set_epi32(-1, 0x0F0E0D0C, 0x07060504, -1)); else assert(0); @@ -894,8 +894,8 @@ namespace ojph { * @param vn used for handling E values (stores v_n values) * @return __m128i decoded quad */ - static inline - __m128i decode_two_quad16(const __m128i inf_u_q, __m128i U_q, + static inline + __m128i decode_two_quad16(const __m128i inf_u_q, __m128i U_q, frwd_struct_ssse3* magsgn, ui32 p, __m128i& vn) { __m128i w0; // workers @@ -904,21 +904,21 @@ namespace ojph { __m128i row; // decoded row row = _mm_setzero_si128(); - w0 = _mm_shuffle_epi8(inf_u_q, + w0 = _mm_shuffle_epi8(inf_u_q, _mm_set_epi16(0x0504, 0x0504, 0x0504, 0x0504, 0x0100, 0x0100, 0x0100, 0x0100)); // we keeps e_k, e_1, and rho in w2 - flags = _mm_and_si128(w0, + flags = _mm_and_si128(w0, _mm_set_epi16((si16)0x8880, 0x4440, 0x2220, 0x1110, (si16)0x8880, 0x4440, 0x2220, 0x1110)); insig = _mm_cmpeq_epi16(flags, _mm_setzero_si128()); if (_mm_movemask_epi8(insig) != 0xFFFF) //are all insignificant? { - U_q = _mm_shuffle_epi8(U_q, + U_q = _mm_shuffle_epi8(U_q, _mm_set_epi16(0x0504, 0x0504, 0x0504, 0x0504, 0x0100, 0x0100, 0x0100, 0x0100)); flags = _mm_mullo_epi16(flags, _mm_set_epi16(1,2,4,8,1,2,4,8)); - __m128i ms_vec = frwd_fetch<0xFF>(magsgn); + __m128i ms_vec = frwd_fetch<0xFF>(magsgn); // U_q holds U_q for this quad // flags has e_k, e_1, and rho such that e_k is sitting in the @@ -942,8 +942,8 @@ namespace ojph { // find the starting byte and starting bit __m128i byte_idx = _mm_srli_epi16(ex_sum, 3); __m128i bit_idx = _mm_and_si128(ex_sum, _mm_set1_epi16(7)); - byte_idx = _mm_shuffle_epi8(byte_idx, - _mm_set_epi16(0x0E0E, 0x0C0C, 0x0A0A, 0x0808, + byte_idx = _mm_shuffle_epi8(byte_idx, + _mm_set_epi16(0x0E0E, 0x0C0C, 0x0A0A, 0x0808, 0x0606, 0x0404, 0x0202, 0x0000)); byte_idx = _mm_add_epi16(byte_idx, _mm_set1_epi16(0x0100)); __m128i d0 = _mm_shuffle_epi8(ms_vec, byte_idx); @@ -990,10 +990,10 @@ namespace ojph { row = _mm_andnot_si128(insig, ms_vec); // significant only ms_vec = _mm_andnot_si128(insig, tvn); // significant only - w0 = _mm_shuffle_epi8(ms_vec, + w0 = _mm_shuffle_epi8(ms_vec, _mm_set_epi16(-1, -1, -1, -1, -1, -1, 0x0706, 0x0302)); vn = _mm_or_si128(vn, w0); - w0 = _mm_shuffle_epi8(ms_vec, + w0 = _mm_shuffle_epi8(ms_vec, _mm_set_epi16(-1, -1, -1, -1, -1, 0x0F0E, 0x0B0A, -1)); vn = _mm_or_si128(vn, w0); @@ -1016,9 +1016,9 @@ namespace ojph { * @param [in] lengths1 is the length of cleanup pass * @param [in] lengths2 is the length of refinement passes (either SPP * only or SPP+MRP) - * @param [in] width is the decoded codeblock width + * @param [in] width is the decoded codeblock width * @param [in] height is the decoded codeblock height - * @param [in] stride is the decoded codeblock buffer stride + * @param [in] stride is the decoded codeblock buffer stride * @param [in] stripe_causal is true for stripe causal mode */ bool ojph_decode_codeblock_ssse3(ui8* coded_data, ui32* decoded_data, @@ -1049,7 +1049,7 @@ namespace ojph { if (missing_msbs > 30) // p < 0 { - if (insufficient_precision == false) + if (insufficient_precision == false) { insufficient_precision = true; OJPH_WARN(0x00010003, "32 bits are not enough to decode this " @@ -1057,7 +1057,7 @@ namespace ojph { "displayed again."); } return false; - } + } else if (missing_msbs == 30) // p == 0 { // not enough precision to decode and set the bin center to 1 if (modify_code == false) { @@ -1100,16 +1100,16 @@ namespace ojph { if (scup < 2 || scup > lcup || scup > 4079) //something is wrong return false; - // The temporary storage scratch holds two types of data in an + // The temporary storage scratch holds two types of data in an // interleaved fashion. The interleaving allows us to use one // memory pointer. // We have one entry for a decoded VLC code, and one entry for UVLC. - // Entries are 16 bits each, corresponding to one quad, - // but since we want to use XMM registers of the SSE family + // Entries are 16 bits each, corresponding to one quad, + // but since we want to use XMM registers of the SSE family // of SIMD; we allocated 16 bytes or more per quad row; that is, // the width is no smaller than 16 bytes (or 8 entries), and the // height is 512 quads - // Each VLC entry contains, in the following order, starting + // Each VLC entry contains, in the following order, starting // from MSB // e_k (4bits), e_1 (4bits), rho (4bits), useless for step 2 (4bits) // Each entry in UVLC contains u_q @@ -1118,10 +1118,10 @@ namespace ojph { ui16 scratch[8 * 513] = {0}; // 8+ kB // We need an extra two entries (one inf and one u_q) beyond - // the last column. - // If the block width is 4 (2 quads), then we use sstr of 8 - // (enough for 4 quads). If width is 8 (4 quads) we use - // sstr is 16 (enough for 8 quads). For a width of 16 (8 + // the last column. + // If the block width is 4 (2 quads), then we use sstr of 8 + // (enough for 4 quads). If width is 8 (4 quads) we use + // sstr is 16 (enough for 8 quads). For a width of 16 (8 // quads), we use 24 (enough for 12 quads). ui32 sstr = ((width + 2u) + 7u) & ~7u; // multiples of 8 @@ -1130,11 +1130,11 @@ namespace ojph { ui32 mmsbp2 = missing_msbs + 2; // The cleanup pass is decoded in two steps; in step one, - // the VLC and MEL segments are decoded, generating a record that + // the VLC and MEL segments are decoded, generating a record that // has 2 bytes per quad. The 2 bytes contain, u, rho, e^1 & e^k. // This information should be sufficient for the next step. // In step 2, we decode the MagSgn segment. - + // step 1 decoding VLC and MEL segments { // init structures @@ -1167,20 +1167,20 @@ namespace ojph { { run -= 2; //subtract 2, since events number if multiplied by 2 - // Is the run terminated in 1? if so, use decoded VLC code, - // otherwise, discard decoded data, since we will decoded again + // Is the run terminated in 1? if so, use decoded VLC code, + // otherwise, discard decoded data, since we will decoded again // using a different context t0 = (run == -1) ? t0 : 0; // is run -1 or -2? this means a run has been consumed - if (run < 0) + if (run < 0) run = mel_get_run(&mel); // get another run } //run -= (c_q == 0) ? 2 : 0; //t0 = (c_q != 0 || run == -1) ? t0 : 0; //if (run < 0) // run = mel_get_run(&mel); // get another run - sp[0] = t0; + sp[0] = t0; x += 2; // prepare context for the next quad; eqn. 1 in ITU T.814 @@ -1193,7 +1193,7 @@ namespace ojph { ui16 t1 = 0; //decode VLC using the context c_q and the head of VLC bitstream - t1 = vlc_tbl0[c_q + (vlc_val & 0x7F)]; + t1 = vlc_tbl0[c_q + (vlc_val & 0x7F)]; // if context is zero, use one MEL event if (c_q == 0 && x < width) //zero context @@ -1219,7 +1219,7 @@ namespace ojph { //remove data from vlc stream, if qinf is not used, cwdlen is 0 vlc_val = rev_advance(&vlc, t1 & 0x7); - + // decode u ///////////// // uvlc_mode is made up of u_offset bits from the quad pair @@ -1242,8 +1242,8 @@ namespace ojph { //decode uvlc_mode to get u for both quads ui32 uvlc_entry = uvlc_tbl0[uvlc_mode + (vlc_val & 0x3F)]; //remove total prefix length - vlc_val = rev_advance(&vlc, uvlc_entry & 0x7); - uvlc_entry >>= 3; + vlc_val = rev_advance(&vlc, uvlc_entry & 0x7); + uvlc_entry >>= 3; //extract suffixes for quad 0 and 1 ui32 len = uvlc_entry & 0xF; //suffix length for 2 quads ui32 tmp = vlc_val & ((1 << len) - 1); //suffix value for 2 quads @@ -1253,9 +1253,9 @@ namespace ojph { len = uvlc_entry & 0x7; // quad 0 suffix length uvlc_entry >>= 3; ui16 u_q = (ui16)(1 + (uvlc_entry&7) + (tmp&~(0xFFU<> 3) + (tmp >> len)); //kappa == 1 - sp[3] = u_q; + sp[3] = u_q; } sp[0] = sp[1] = 0; @@ -1285,13 +1285,13 @@ namespace ojph { { run -= 2; //subtract 2, since events number is multiplied by 2 - // Is the run terminated in 1? if so, use decoded VLC code, - // otherwise, discard decoded data, since we will decoded again + // Is the run terminated in 1? if so, use decoded VLC code, + // otherwise, discard decoded data, since we will decoded again // using a different context t0 = (run == -1) ? t0 : 0; // is run -1 or -2? this means a run has been consumed - if (run < 0) + if (run < 0) run = mel_get_run(&mel); // get another run } //run -= (c_q == 0) ? 2 : 0; @@ -1317,7 +1317,7 @@ namespace ojph { ui16 t1 = 0; //decode VLC using the context c_q and the head of VLC bitstream - t1 = vlc_tbl1[ c_q + (vlc_val & 0x7F)]; + t1 = vlc_tbl1[ c_q + (vlc_val & 0x7F)]; // if context is zero, use one MEL event if (c_q == 0 && x < width) //zero context @@ -1335,7 +1335,7 @@ namespace ojph { //t1 = (c_q != 0 || run == -1) ? t1 : 0; //if (run < 0) // run = mel_get_run(&mel); // get another run - sp[2] = t1; + sp[2] = t1; x += 2; // partial c_q, will be completed when we process the next quad @@ -1346,7 +1346,7 @@ namespace ojph { //remove data from vlc stream, if qinf is not used, cwdlen is 0 vlc_val = rev_advance(&vlc, t1 & 0x7); - + // decode u ///////////// // uvlc_mode is made up of u_offset bits from the quad pair @@ -1422,9 +1422,9 @@ namespace ojph { w0 = _mm_loadu_si128((__m128i*)vp); w0 = _mm_and_si128(w0, _mm_set_epi32(0,0,0,-1)); w0 = _mm_or_si128(w0, vn); - _mm_storeu_si128((__m128i*)vp, w0); + _mm_storeu_si128((__m128i*)vp, w0); - //interleave in ssse3 style + //interleave in ssse3 style w0 = _mm_unpacklo_epi32(row0, row1); w1 = _mm_unpackhi_epi32(row0, row1); row0 = _mm_unpacklo_epi32(w0, w1); @@ -1493,9 +1493,9 @@ namespace ojph { gamma = _mm_and_si128(gamma, w0); gamma = _mm_cmpeq_epi32(gamma, _mm_setzero_si128()); - emax = _mm_loadu_si128((__m128i*)(vp + v_n_size)); + emax = _mm_loadu_si128((__m128i*)(vp + v_n_size)); w0 = _mm_bsrli_si128(emax, 4); - emax = _mm_max_epi16(w0, emax); // no max_epi32 in ssse3 + emax = _mm_max_epi16(w0, emax); // no max_epi32 in ssse3 emax = _mm_andnot_si128(gamma, emax); kappa = _mm_set1_epi32(1); @@ -1516,7 +1516,7 @@ namespace ojph { w0 = _mm_loadu_si128((__m128i*)vp); w0 = _mm_and_si128(w0, _mm_set_epi32(0,0,0,-1)); w0 = _mm_or_si128(w0, vn); - _mm_storeu_si128((__m128i*)vp, w0); + _mm_storeu_si128((__m128i*)vp, w0); //interleave in ssse3 style w0 = _mm_unpacklo_epi32(row0, row1); @@ -1528,7 +1528,7 @@ namespace ojph { } } } - else + else { // reduce bitplane by 16 because we now have 16 bits instead of 32 p -= 16; @@ -1572,14 +1572,14 @@ namespace ojph { w0 = _mm_loadu_si128((__m128i*)vp); w0 = _mm_and_si128(w0, _mm_set_epi16(0,0,0,0,0,0,0,-1)); w0 = _mm_or_si128(w0, vn); - _mm_storeu_si128((__m128i*)vp, w0); + _mm_storeu_si128((__m128i*)vp, w0); - //interleave in ssse3 style - w0 = _mm_shuffle_epi8(row, + //interleave in ssse3 style + w0 = _mm_shuffle_epi8(row, _mm_set_epi16(0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1)); _mm_store_si128((__m128i*)dp, w0); - w1 = _mm_shuffle_epi8(row, + w1 = _mm_shuffle_epi8(row, _mm_set_epi16(0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1)); _mm_store_si128((__m128i*)(dp + stride), w1); @@ -1640,11 +1640,11 @@ namespace ojph { gamma = _mm_and_si128(gamma, w0); gamma = _mm_cmpeq_epi32(gamma, _mm_setzero_si128()); - emax = _mm_loadu_si128((__m128i*)(vp + v_n_size)); + emax = _mm_loadu_si128((__m128i*)(vp + v_n_size)); w0 = _mm_bsrli_si128(emax, 2); emax = _mm_max_epi16(w0, emax); // no max_epi32 in ssse3 - emax = _mm_shuffle_epi8(emax, - _mm_set_epi16(-1, 0x0706, -1, 0x0504, + emax = _mm_shuffle_epi8(emax, + _mm_set_epi16(-1, 0x0706, -1, 0x0504, -1, 0x0302, -1, 0x0100)); emax = _mm_andnot_si128(gamma, emax); @@ -1665,13 +1665,13 @@ namespace ojph { w0 = _mm_loadu_si128((__m128i*)vp); w0 = _mm_and_si128(w0, _mm_set_epi16(0,0,0,0,0,0,0,-1)); w0 = _mm_or_si128(w0, vn); - _mm_storeu_si128((__m128i*)vp, w0); + _mm_storeu_si128((__m128i*)vp, w0); - w0 = _mm_shuffle_epi8(row, + w0 = _mm_shuffle_epi8(row, _mm_set_epi16(0x0D0C, -1, 0x0908, -1, 0x0504, -1, 0x0100, -1)); _mm_store_si128((__m128i*)dp, w0); - w1 = _mm_shuffle_epi8(row, + w1 = _mm_shuffle_epi8(row, _mm_set_epi16(0x0F0E, -1, 0x0B0A, -1, 0x0706, -1, 0x0302, -1)); _mm_store_si128((__m128i*)(dp + stride), w1); @@ -1686,7 +1686,7 @@ namespace ojph { { // We use scratch again, we can divide it into multiple regions // sigma holds all the significant samples, and it cannot - // be modified after it is set. it will be used during the + // be modified after it is set. it will be used during the // Magnitude Refinement Pass ui16* const sigma = scratch; @@ -1703,11 +1703,11 @@ namespace ojph { const __m128i mask_3 = _mm_set1_epi32(0x30); const __m128i mask_C = _mm_set1_epi32(0xC0); const __m128i shuffle_mask = _mm_set_epi32(-1, -1, -1, 0x0C080400); - for (y = 0; y < height; y += 4) + for (y = 0; y < height; y += 4) { ui16* sp = scratch + (y >> 1) * sstr; ui16* dp = sigma + (y >> 2) * mstr; - for (ui32 x = 0; x < width; x += 8, sp += 8, dp += 2) + for (ui32 x = 0; x < width; x += 8, sp += 8, dp += 2) { __m128i s0, s1, u3, uC, t0, t1; @@ -1795,13 +1795,13 @@ namespace ojph { // We need data for at least 5 columns out of 8. // Therefore loading 32 bits is easier than loading 16 bits // twice. - ui32 ps = *(ui32*)prev_sig; - ui32 ns = *(ui32*)(cur_sig + mstr); + ui32 ps; memcpy(&ps, prev_sig, sizeof(ps)); + ui32 ns; memcpy(&ns, cur_sig + mstr, sizeof(ns)); ui32 u = (ps & 0x88888888) >> 3; // the row on top if (!stripe_causal) u |= (ns & 0x11111111) << 3; // the row below - ui32 cs = *(ui32*)cur_sig; + ui32 cs; memcpy(&cs, cur_sig, sizeof(cs)); // vertical integration ui32 mbr = cs; // this sig. info. mbr |= (cs & 0x77777777) << 1; //above neighbors @@ -2021,9 +2021,9 @@ namespace ojph { __m128i cwd_vec = _mm_set1_epi16((si16)cwd); cwd_vec = _mm_shuffle_epi8(cwd_vec, _mm_set_epi8(1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0)); - cwd_vec = _mm_and_si128(cwd_vec, + cwd_vec = _mm_and_si128(cwd_vec, _mm_set1_epi64x((si64)0x8040201008040201)); - cwd_vec = _mm_cmpeq_epi8(cwd_vec, + cwd_vec = _mm_cmpeq_epi8(cwd_vec, _mm_set1_epi64x((si64)0x8040201008040201)); cwd_vec = _mm_add_epi8(cwd_vec, _mm_set1_epi8(1)); cwd_vec = _mm_add_epi8(cwd_vec, cwd_vec); @@ -2035,7 +2035,7 @@ namespace ojph { ui32 *dp = dpp; for (int c = 0; c < 4; ++c) { __m128i s0, s0_sig, s0_idx, s0_val; - // load coefficients + // load coefficients s0 = _mm_load_si128((__m128i*)dp); // find significant samples in this row s0_sig = _mm_shuffle_epi8(sig_vec, m); diff --git a/external/OpenJPH/src/core/coding/ojph_block_decoder_vsx.cpp b/external/OpenJPH/src/core/coding/ojph_block_decoder_vsx.cpp new file mode 100644 index 000000000..4ddcde085 --- /dev/null +++ b/external/OpenJPH/src/core/coding/ojph_block_decoder_vsx.cpp @@ -0,0 +1,2226 @@ +//***************************************************************************/ +// This software is released under the 2-Clause BSD license, included +// below. +// +// Copyright (c) 2022, Aous Naman +// Copyright (c) 2022, Kakadu Software Pty Ltd, Australia +// Copyright (c) 2022, The University of New South Wales, Australia +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************/ +// This file is part of the OpenJPH software implementation. +// File: ojph_block_decoder_vsx.cpp +// Author: Aous Naman +// Date: 13 May 2022 +//***************************************************************************/ + +//***************************************************************************/ +/** @file ojph_block_decoder_vsx.cpp + * @brief implements a faster HTJ2K block decoder using POWER VSX simd + */ + +#include +#include + +#include +#include +#include "ojph_block_common.h" +#include "ojph_block_decoder.h" +#include "ojph_arch.h" +#include "ojph_message.h" + +#include "ojph_simd_vsx.h" + +namespace ojph { + namespace local { + + //************************************************************************/ + /** @brief Macros that help with typing and space + */ + #define OJPH_REPEAT2(a) a,a + #define OJPH_REPEAT4(a) a,a,a,a + #define OJPH_REPEAT8(a) a,a,a,a,a,a,a,a + #define OJPH_REPEAT16(a) a,a,a,a,a,a,a,a,a,a,a,a,a,a,a,a + + //************************************************************************/ + /** @brief MEL state structure for reading and decoding the MEL bitstream + * + * A number of events is decoded from the MEL bitstream ahead of time + * and stored in run/num_runs. + * Each run represents the number of zero events before a one event. + */ + struct dec_mel_st { + dec_mel_st() : data(NULL), tmp(0), bits(0), size(0), unstuff(false), + k(0), num_runs(0), runs(0) + {} + // data decoding machinery + ui8* data; //!bits > 32) //there are enough bits in the tmp variable + return; // return without reading new data + + ui32 val = 0xFFFFFFFF; // feed in 0xFF if buffer is exhausted + if (melp->size > 4) { // if there is data in the MEL segment + memcpy(&val, melp->data, sizeof(val)); // read 32 bits from MEL data + melp->data += 4; // advance pointer + melp->size -= 4; // reduce counter + } + else if (melp->size > 0) + { // 4 or less + int i = 0; + while (melp->size > 1) { + ui32 v = *melp->data++; // read one byte at a time + ui32 m = ~(0xFFu << i); // mask of location + val = (val & m) | (v << i);// put one byte in its correct location + --melp->size; + i += 8; + } + // size equal to 1 + ui32 v = *melp->data++; // the one before the last is different + v |= 0xF; // MEL and VLC segments can overlap + ui32 m = ~(0xFFu << i); + val = (val & m) | (v << i); + --melp->size; + } + + // next we unstuff them before adding them to the buffer + int bits = 32 - melp->unstuff; // number of bits in val, subtract 1 if + // the previously read byte requires + // unstuffing + + // data is unstuffed and accumulated in t + // bits has the number of bits in t + ui32 t = val & 0xFF; + bool unstuff = ((val & 0xFF) == 0xFF); // true if we need unstuffing + bits -= unstuff; // there is one less bit in t if unstuffing is needed + t = t << (8 - unstuff); // move up to make room for the next byte + + //this is a repeat of the above + t |= (val>>8) & 0xFF; + unstuff = (((val >> 8) & 0xFF) == 0xFF); + bits -= unstuff; + t = t << (8 - unstuff); + + t |= (val>>16) & 0xFF; + unstuff = (((val >> 16) & 0xFF) == 0xFF); + bits -= unstuff; + t = t << (8 - unstuff); + + t |= (val>>24) & 0xFF; + melp->unstuff = (((val >> 24) & 0xFF) == 0xFF); + + // move t to tmp, and push the result all the way up, so we read from + // the MSB + melp->tmp |= ((ui64)t) << (64 - bits - melp->bits); + melp->bits += bits; //increment the number of bits in tmp + } + + //************************************************************************/ + /** @brief Decodes unstuffed MEL segment bits stored in tmp to runs + * + * Runs are stored in "runs" and the number of runs in "num_runs". + * Each run represents a number of zero events that may or may not + * terminate in a 1 event. + * Each run is stored in 7 bits. The LSB is 1 if the run terminates in + * a 1 event, 0 otherwise. The next 6 bits, for the case terminating + * with 1, contain the number of consecutive 0 zero events * 2; for the + * case terminating with 0, they store (number of consecutive 0 zero + * events - 1) * 2. + * A total of 6 bits (made up of 1 + 5) should have been enough. + * + * @param [in] melp is a pointer to dec_mel_st structure + */ + static inline + void mel_decode(dec_mel_st *melp) + { + static const int mel_exp[13] = { //MEL exponents + 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5 + }; + + if (melp->bits < 6) // if there are less than 6 bits in tmp + mel_read(melp); // then read from the MEL bitstream + // 6 bits is the largest decodable MEL cwd + + //repeat so long that there is enough decodable bits in tmp, + // and the runs store is not full (num_runs < 8) + while (melp->bits >= 6 && melp->num_runs < 8) + { + int eval = mel_exp[melp->k]; // number of bits associated with state + int run = 0; + if (melp->tmp & (1ull<<63)) //The next bit to decode (stored in MSB) + { //one is found + run = 1 << eval; + run--; // consecutive runs of 0 events - 1 + melp->k = melp->k + 1 < 12 ? melp->k + 1 : 12;//increment, max is 12 + melp->tmp <<= 1; // consume one bit from tmp + melp->bits -= 1; + run = run << 1; // a stretch of zeros not terminating in one + } + else + { //0 is found + run = (int)(melp->tmp >> (63 - eval)) & ((1 << eval) - 1); + melp->k = melp->k - 1 > 0 ? melp->k - 1 : 0; //decrement, min is 0 + melp->tmp <<= eval + 1; //consume eval + 1 bits (max is 6) + melp->bits -= eval + 1; + run = (run << 1) + 1; // a stretch of zeros terminating with one + } + eval = melp->num_runs * 7; // 7 bits per run + melp->runs &= ~((ui64)0x3F << eval); // 6 bits are sufficient + melp->runs |= ((ui64)run) << eval; // store the value in runs + melp->num_runs++; // increment count + } + } + + //************************************************************************/ + /** @brief Initiates a dec_mel_st structure for MEL decoding and reads + * some bytes in order to get the read address to a multiple + * of 4 + * + * @param [in] melp is a pointer to dec_mel_st structure + * @param [in] bbuf is a pointer to byte buffer + * @param [in] lcup is the length of MagSgn+MEL+VLC segments + * @param [in] scup is the length of MEL+VLC segments + */ + static inline + void mel_init(dec_mel_st *melp, ui8* bbuf, int lcup, int scup) + { + melp->data = bbuf + lcup - scup; // move the pointer to the start of MEL + melp->bits = 0; // 0 bits in tmp + melp->tmp = 0; // + melp->unstuff = false; // no unstuffing + melp->size = scup - 1; // size is the length of MEL+VLC-1 + melp->k = 0; // 0 for state + melp->num_runs = 0; // num_runs is 0 + melp->runs = 0; // + + //This code is borrowed; original is for a different architecture + //These few lines take care of the case where data is not at a multiple + // of 4 boundary. It reads 1,2,3 up to 4 bytes from the MEL segment + int num = 4 - (int)(intptr_t(melp->data) & 0x3); + for (int i = 0; i < num; ++i) { // this code is similar to mel_read + assert(melp->unstuff == false || melp->data[0] <= 0x8F); + ui64 d = (melp->size > 0) ? *melp->data : 0xFF;//if buffer is consumed + //set data to 0xFF + if (melp->size == 1) d |= 0xF; //if this is MEL+VLC-1, set LSBs to 0xF + // see the standard + melp->data += melp->size-- > 0; //increment if the end is not reached + int d_bits = 8 - melp->unstuff; //if unstuffing is needed, reduce by 1 + melp->tmp = (melp->tmp << d_bits) | d; //store bits in tmp + melp->bits += d_bits; //increment tmp by number of bits + melp->unstuff = ((d & 0xFF) == 0xFF); //true of next byte needs + //unstuffing + } + melp->tmp <<= (64 - melp->bits); //push all the way up so the first bit + // is the MSB + } + + //************************************************************************/ + /** @brief Retrieves one run from dec_mel_st; if there are no runs stored + * MEL segment is decoded + * + * @param [in] melp is a pointer to dec_mel_st structure + */ + static inline + int mel_get_run(dec_mel_st *melp) + { + if (melp->num_runs == 0) //if no runs, decode more bit from MEL segment + mel_decode(melp); + + int t = melp->runs & 0x7F; //retrieve one run + melp->runs >>= 7; // remove the retrieved run + melp->num_runs--; + return t; // return run + } + + //************************************************************************/ + /** @brief A structure for reading and unstuffing a segment that grows + * backward, such as VLC and MRP + */ + struct rev_struct { + rev_struct() : data(NULL), tmp(0), bits(0), size(0), unstuff(false) + {} + //storage + ui8* data; //!bits > 32) // if there are more than 32 bits in tmp, then + return; // reading 32 bits can overflow vlcp->tmp + ui32 val = 0; + //the next line (the if statement) needs to be tested first + if (vlcp->size > 3) // if there are more than 3 bytes left in VLC + { + // (vlcp->data - 3) move pointer back to read 32 bits at once + memcpy(&val, vlcp->data - 3, sizeof(val)); // then read 32 bits + vlcp->data -= 4; // move data pointer back by 4 + vlcp->size -= 4; // reduce available byte by 4 + } + else if (vlcp->size > 0) + { // 4 or less + int i = 24; + while (vlcp->size > 0) { + ui32 v = *vlcp->data--; // read one byte at a time + val |= (v << i); // put byte in its correct location + --vlcp->size; + i -= 8; + } + } + + //accumulate in tmp, number of bits in tmp are stored in bits + ui32 tmp = val >> 24; //start with the MSB byte + ui32 bits; + + // test unstuff (previous byte is >0x8F), and this byte is 0x7F + bits = 8 - ((vlcp->unstuff && (((val >> 24) & 0x7F) == 0x7F)) ? 1 : 0); + bool unstuff = (val >> 24) > 0x8F; //this is for the next byte + + tmp |= ((val >> 16) & 0xFF) << bits; //process the next byte + bits += 8 - ((unstuff && (((val >> 16) & 0x7F) == 0x7F)) ? 1 : 0); + unstuff = ((val >> 16) & 0xFF) > 0x8F; + + tmp |= ((val >> 8) & 0xFF) << bits; + bits += 8 - ((unstuff && (((val >> 8) & 0x7F) == 0x7F)) ? 1 : 0); + unstuff = ((val >> 8) & 0xFF) > 0x8F; + + tmp |= (val & 0xFF) << bits; + bits += 8 - ((unstuff && ((val & 0x7F) == 0x7F)) ? 1 : 0); + unstuff = (val & 0xFF) > 0x8F; + + // now move the read and unstuffed bits into vlcp->tmp + vlcp->tmp |= (ui64)tmp << vlcp->bits; + vlcp->bits += bits; + vlcp->unstuff = unstuff; // this for the next read + } + + //************************************************************************/ + /** @brief Initiates the rev_struct structure and reads a few bytes to + * move the read address to multiple of 4 + * + * There is another similar rev_init_mrp subroutine. The difference is + * that this one, rev_init, discards the first 12 bits (they have the + * sum of the lengths of VLC and MEL segments), and first unstuff depends + * on first 4 bits. + * + * @param [in] vlcp is a pointer to rev_struct structure + * @param [in] data is a pointer to byte at the start of the cleanup pass + * @param [in] lcup is the length of MagSgn+MEL+VLC segments + * @param [in] scup is the length of MEL+VLC segments + */ + static inline + void rev_init(rev_struct *vlcp, ui8* data, int lcup, int scup) + { + //first byte has only the upper 4 bits + vlcp->data = data + lcup - 2; + + //size can not be larger than this, in fact it should be smaller + vlcp->size = scup - 2; + + ui32 d = *vlcp->data--; // read one byte (this is a half byte) + vlcp->tmp = d >> 4; // both initialize and set + vlcp->bits = 4 - ((vlcp->tmp & 7) == 7); //check standard + vlcp->unstuff = (d | 0xF) > 0x8F; //this is useful for the next byte + + //This code is designed for an architecture that read address should + // align to the read size (address multiple of 4 if read size is 4) + //These few lines take care of the case where data is not at a multiple + // of 4 boundary. It reads 1,2,3 up to 4 bytes from the VLC bitstream. + // To read 32 bits, read from (vlcp->data - 3) + int num = 1 + (int)(intptr_t(vlcp->data) & 0x3); + int tnum = num < vlcp->size ? num : vlcp->size; + for (int i = 0; i < tnum; ++i) { + ui64 d; + d = *vlcp->data--; // read one byte and move read pointer + //check if the last byte was >0x8F (unstuff == true) and this is 0x7F + ui32 d_bits = 8 - ((vlcp->unstuff && ((d & 0x7F) == 0x7F)) ? 1 : 0); + vlcp->tmp |= d << vlcp->bits; // move data to vlcp->tmp + vlcp->bits += d_bits; + vlcp->unstuff = d > 0x8F; // for next byte + } + vlcp->size -= tnum; + rev_read(vlcp); // read another 32 buts + } + + //************************************************************************/ + /** @brief Retrieves 32 bits from the head of a rev_struct structure + * + * By the end of this call, vlcp->tmp must have no less than 33 bits + * + * @param [in] vlcp is a pointer to rev_struct structure + */ + static inline + ui32 rev_fetch(rev_struct *vlcp) + { + if (vlcp->bits < 32) // if there are less then 32 bits, read more + { + rev_read(vlcp); // read 32 bits, but unstuffing might reduce this + if (vlcp->bits < 32)// if there is still space in vlcp->tmp for 32 bits + rev_read(vlcp); // read another 32 + } + return (ui32)vlcp->tmp; // return the head (bottom-most) of vlcp->tmp + } + + //************************************************************************/ + /** @brief Consumes num_bits from a rev_struct structure + * + * @param [in] vlcp is a pointer to rev_struct structure + * @param [in] num_bits is the number of bits to be removed + */ + static inline + ui32 rev_advance(rev_struct *vlcp, ui32 num_bits) + { + assert(num_bits <= vlcp->bits); // vlcp->tmp must have more than num_bits + vlcp->tmp >>= num_bits; // remove bits + vlcp->bits -= num_bits; // decrement the number of bits + return (ui32)vlcp->tmp; + } + + //************************************************************************/ + /** @brief Reads and unstuffs from rev_struct + * + * This is different than rev_read in that this fills in zeros when the + * the available data is consumed. The other does not care about the + * values when all data is consumed. + * + * See rev_read for more information about unstuffing + * + * @param [in] mrp is a pointer to rev_struct structure + */ + static inline + void rev_read_mrp(rev_struct *mrp) + { + //process 4 bytes at a time + if (mrp->bits > 32) + return; + ui32 val = 0; + if (mrp->size > 3) // If there are 3 byte or more + { // (mrp->data - 3) move pointer back to read 32 bits at once + memcpy(&val, mrp->data - 3, sizeof(val)); // read 32 bits + mrp->data -= 4; // move back pointer + mrp->size -= 4; // reduce count + } + else if (mrp->size > 0) + { + int i = 24; + while (mrp->size > 0) { + ui32 v = *mrp->data--; // read one byte at a time + val |= (v << i); // put byte in its correct location + --mrp->size; + i -= 8; + } + } + + //accumulate in tmp, and keep count in bits + ui32 bits, tmp = val >> 24; + + //test if the last byte > 0x8F (unstuff must be true) and this is 0x7F + bits = 8 - ((mrp->unstuff && (((val >> 24) & 0x7F) == 0x7F)) ? 1 : 0); + bool unstuff = (val >> 24) > 0x8F; + + //process the next byte + tmp |= ((val >> 16) & 0xFF) << bits; + bits += 8 - ((unstuff && (((val >> 16) & 0x7F) == 0x7F)) ? 1 : 0); + unstuff = ((val >> 16) & 0xFF) > 0x8F; + + tmp |= ((val >> 8) & 0xFF) << bits; + bits += 8 - ((unstuff && (((val >> 8) & 0x7F) == 0x7F)) ? 1 : 0); + unstuff = ((val >> 8) & 0xFF) > 0x8F; + + tmp |= (val & 0xFF) << bits; + bits += 8 - ((unstuff && ((val & 0x7F) == 0x7F)) ? 1 : 0); + unstuff = (val & 0xFF) > 0x8F; + + mrp->tmp |= (ui64)tmp << mrp->bits; // move data to mrp pointer + mrp->bits += bits; + mrp->unstuff = unstuff; // next byte + } + + //************************************************************************/ + /** @brief Initialized rev_struct structure for MRP segment, and reads + * a number of bytes such that the next 32 bits read are from + * an address that is a multiple of 4. Note this is designed for + * an architecture that read size must be compatible with the + * alignment of the read address + * + * There is another similar subroutine rev_init. This subroutine does + * NOT skip the first 12 bits, and starts with unstuff set to true. + * + * @param [in] mrp is a pointer to rev_struct structure + * @param [in] data is a pointer to byte at the start of the cleanup pass + * @param [in] lcup is the length of MagSgn+MEL+VLC segments + * @param [in] len2 is the length of SPP+MRP segments + */ + static inline + void rev_init_mrp(rev_struct *mrp, ui8* data, int lcup, int len2) + { + mrp->data = data + lcup + len2 - 1; + mrp->size = len2; + mrp->unstuff = true; + mrp->bits = 0; + mrp->tmp = 0; + + //This code is designed for an architecture that read address should + // align to the read size (address multiple of 4 if read size is 4) + //These few lines take care of the case where data is not at a multiple + // of 4 boundary. It reads 1,2,3 up to 4 bytes from the MRP stream + int num = 1 + (int)(intptr_t(mrp->data) & 0x3); + for (int i = 0; i < num; ++i) { + ui64 d; + //read a byte, 0 if no more data + d = (mrp->size-- > 0) ? *mrp->data-- : 0; + //check if unstuffing is needed + ui32 d_bits = 8 - ((mrp->unstuff && ((d & 0x7F) == 0x7F)) ? 1 : 0); + mrp->tmp |= d << mrp->bits; // move data to vlcp->tmp + mrp->bits += d_bits; + mrp->unstuff = d > 0x8F; // for next byte + } + rev_read_mrp(mrp); + } + + //************************************************************************/ + /** @brief Retrieves 32 bits from the head of a rev_struct structure + * + * By the end of this call, mrp->tmp must have no less than 33 bits + * + * @param [in] mrp is a pointer to rev_struct structure + */ + static inline + ui32 rev_fetch_mrp(rev_struct *mrp) + { + if (mrp->bits < 32) // if there are less than 32 bits in mrp->tmp + { + rev_read_mrp(mrp); // read 30-32 bits from mrp + if (mrp->bits < 32) // if there is a space of 32 bits + rev_read_mrp(mrp); // read more + } + return (ui32)mrp->tmp; // return the head of mrp->tmp + } + + //************************************************************************/ + /** @brief Consumes num_bits from a rev_struct structure + * + * @param [in] mrp is a pointer to rev_struct structure + * @param [in] num_bits is the number of bits to be removed + */ + inline ui32 rev_advance_mrp(rev_struct *mrp, ui32 num_bits) + { + assert(num_bits <= mrp->bits); // we must not consume more than mrp->bits + mrp->tmp >>= num_bits; // discard the lowest num_bits bits + mrp->bits -= num_bits; + return (ui32)mrp->tmp; // return data after consumption + } + + //************************************************************************/ + /** @brief State structure for reading and unstuffing of forward-growing + * bitstreams; these are: MagSgn and SPP bitstreams + */ + struct frwd_struct { + const ui8* data; //! + static inline + void frwd_read(frwd_struct *msp) + { + assert(msp->bits <= 128); + + v128_t offset, val, validity, all_xff; + val = vsx_v128_load(msp->data); + int bytes = msp->size >= 16 ? 16 : msp->size; + validity = vsx_i8x16_splat((char)bytes); + msp->data += bytes; + msp->size -= bytes; + ui32 bits = 128; + offset = vsx_i64x2_const(0x0706050403020100,0x0F0E0D0C0B0A0908); + validity = vsx_i8x16_gt(validity, offset); + all_xff = vsx_i8x16_const(OJPH_REPEAT16(-1)); + if (X == 0xFF) // the compiler should remove this if statement + { + v128_t t = vsx_v128_xor(validity, all_xff); // complement + val = vsx_v128_or(t, val); // fill with 0xFF + } + else if (X == 0) + val = vsx_v128_and(validity, val); // fill with zeros + else + assert(0); + + v128_t ff_bytes; + ff_bytes = vsx_i8x16_eq(val, all_xff); + ff_bytes = vsx_v128_and(ff_bytes, validity); + ui32 flags = vsx_i8x16_bitmask(ff_bytes); + flags <<= 1; // unstuff following byte + ui32 next_unstuff = flags >> 16; + flags |= msp->unstuff; + flags &= 0xFFFF; + while (flags) + { // bit unstuffing occurs on average once every 256 bytes + // therefore it is not an issue if it is a bit slow + // here we process 16 bytes + --bits; // consuming one stuffing bit + + ui32 loc = 31 - count_leading_zeros(flags); + flags ^= 1 << loc; + + v128_t m, t, c; + t = vsx_i8x16_splat((char)loc); + m = vsx_i8x16_gt(offset, t); + + t = vsx_v128_and(m, val); // keep bits at locations larger than loc + c = vsx_u64x2_shr(t, 1); // 1 bits left + t = vsx_i64x2_shuffle(t, vsx_i64x2_const(0, 0), 1, 2); + t = vsx_i64x2_shl(t, 63); // keep the MSB only + t = vsx_v128_or(t, c); // combine the above 3 steps + + val = vsx_v128_or(t, vsx_v128_andnot(val, m)); + } + + // combine with earlier data + assert(msp->bits >= 0 && msp->bits <= 128); + int cur_bytes = msp->bits >> 3; + ui32 cur_bits = msp->bits & 7; + v128_t b1, b2; + b1 = vsx_i64x2_shl(val, cur_bits); + //next shift 8 bytes right + b2 = vsx_i64x2_shuffle(vsx_i64x2_const(0, 0), val, 1, 2); + b2 = vsx_u64x2_shr(b2, 64u - cur_bits); + b2 = (cur_bits > 0) ? b2 : vsx_i64x2_const(0, 0); + b1 = vsx_v128_or(b1, b2); + b2 = vsx_v128_load(msp->tmp + cur_bytes); + b2 = vsx_v128_or(b1, b2); + vsx_v128_store(msp->tmp + cur_bytes, b2); + + ui32 consumed_bits = bits < 128u - cur_bits ? bits : 128u - cur_bits; + cur_bytes = (msp->bits + consumed_bits + 7) >> 3; // round up + int upper = vsx_u16x8_extract_lane(val, 7); + upper >>= consumed_bits + 16 - 128; + msp->tmp[cur_bytes] = (ui8)upper; // copy byte + + msp->bits += bits; + msp->unstuff = next_unstuff; // next unstuff + assert(msp->unstuff == 0 || msp->unstuff == 1); + } + + //************************************************************************/ + /** @brief Initialize frwd_struct struct and reads some bytes + * + * @tparam X is the value fed in when the bitstream is exhausted. + * See frwd_read regarding the template + * @param [in] msp is a pointer to frwd_struct + * @param [in] data is a pointer to the start of data + * @param [in] size is the number of byte in the bitstream + */ + template + static inline + void frwd_init(frwd_struct *msp, const ui8* data, int size) + { + msp->data = data; + vsx_v128_store(msp->tmp, vsx_i64x2_const(0, 0)); + vsx_v128_store(msp->tmp + 16, vsx_i64x2_const(0, 0)); + vsx_v128_store(msp->tmp + 32, vsx_i64x2_const(0, 0)); + + msp->bits = 0; + msp->unstuff = 0; + msp->size = size; + + frwd_read(msp); // read 128 bits more + } + + //************************************************************************/ + /** @brief Consume num_bits bits from the bitstream of frwd_struct + * + * @param [in] msp is a pointer to frwd_struct + * @param [in] num_bits is the number of bit to consume + */ + static inline + void frwd_advance(frwd_struct *msp, ui32 num_bits) + { + assert(num_bits > 0 && num_bits <= msp->bits && num_bits < 128); + msp->bits -= num_bits; + + v128_t *p = (v128_t*)(msp->tmp + ((num_bits >> 3) & 0x18)); + num_bits &= 63; + + v128_t v0, v1, c0, c1, t; + v0 = vsx_v128_load(p); + v1 = vsx_v128_load(p + 1); + + // shift right by num_bits + c0 = vsx_u64x2_shr(v0, num_bits); + t = vsx_i64x2_shuffle(v0, vsx_i64x2_const(0, 0), 1, 2); + t = vsx_i64x2_shl(t, 64 - num_bits); + t = (num_bits > 0) ? t : vsx_i64x2_const(0, 0); + c0 = vsx_v128_or(c0, t); + t = vsx_i64x2_shuffle(vsx_i64x2_const(0, 0), v1, 1, 2); + t = vsx_i64x2_shl(t, 64 - num_bits); + t = (num_bits > 0) ? t : vsx_i64x2_const(0, 0); + c0 = vsx_v128_or(c0, t); + + vsx_v128_store(msp->tmp, c0); + + c1 = vsx_u64x2_shr(v1, num_bits); + t = vsx_i64x2_shuffle(v1, vsx_i64x2_const(0, 0), 1, 2); + t = vsx_i64x2_shl(t, 64 - num_bits); + t = (num_bits > 0) ? t : vsx_i64x2_const(0, 0); + c1 = vsx_v128_or(c1, t); + + vsx_v128_store(msp->tmp + 16, c1); + } + + //************************************************************************/ + /** @brief Fetches 32 bits from the frwd_struct bitstream + * + * @tparam X is the value fed in when the bitstream is exhausted. + * See frwd_read regarding the template + * @param [in] msp is a pointer to frwd_struct + */ + template + static inline + v128_t frwd_fetch(frwd_struct *msp) + { + if (msp->bits <= 128) + { + frwd_read(msp); + if (msp->bits <= 128) //need to test + frwd_read(msp); + } + v128_t t = vsx_v128_load(msp->tmp); + return t; + } + + //************************************************************************/ + /** @brief Destuffs a bitstream into a contiguous buffer, upfront + * + * Removes bit stuffing once for the whole stream, so that fetching + * bits at a given position later needs no serial state; this keeps + * the per-quad bit-consumption chain in general-purpose registers + * (pos += bits) instead of shifting a vector-resident window, which + * on POWER costs vector stores/reloads through the stack. + * + * @tparam X is the value fed in when the bitstream is exhausted + * @param [in] src is a pointer to the start of the bitstream + * @param [in] size is the number of bytes in the bitstream + * @param [in] dst is the destination buffer, of capacity cap + 72 + * @param [in] cap is the destination capacity, excluding padding + * @return the clamp offset; bytes at or beyond this offset hold + * no stream bits (they read as X) + */ + template + static inline + ui32 destuff_frwd(const ui8* src, int size, ui8* dst, ui32 cap) + { + if (size < 0) + size = 0; + ui8* o = dst; + ui8* o_end = dst + cap; + const ui8* s = src; + const ui8* s_end = src + size; + ui64 acc = 0; // partial output byte, low nb bits are valid + ui32 nb = 0; // number of valid bits in acc; always < 8 + bool prev_ff = false; + + // fast path; 16 source bytes at a time when they contain no 0xFF + while (s + 16 <= s_end && o + 24 <= o_end) + { + v128_t v = vsx_v128_load(s); + if (vec_any_eq((vsx_v_u8)v, vec_splats((unsigned char)0xFF)) + || prev_ff) + { // process these 16 bytes one at a time + for (int i = 0; i < 16; ++i) { + ui8 b = *s++; + acc |= (ui64)b << nb; + nb += prev_ff ? 7u : 8u; + prev_ff = (b == 0xFFu); + if (nb >= 8) { *o++ = (ui8)acc; acc >>= 8; nb -= 8; } + } + continue; + } + ui64 v0, v1; + memcpy(&v0, s, 8); + memcpy(&v1, s + 8, 8); + ui64 w0 = acc | (v0 << nb); + ui64 w1 = (v1 << nb) | (nb ? (v0 >> (64 - nb)) : 0); + memcpy(o, &w0, 8); + memcpy(o + 8, &w1, 8); + acc = nb ? (v1 >> (64 - nb)) : 0; + o += 16; + s += 16; + } + // tail; one byte at a time + while (s < s_end && o < o_end) + { + ui8 b = *s++; + acc |= (ui64)b << nb; + nb += prev_ff ? 7u : 8u; + prev_ff = (b == 0xFFu); + if (nb >= 8) { *o++ = (ui8)acc; acc >>= 8; nb -= 8; } + } + // fill the bits above nb with X, and pad with X bytes + ui32 fill = (X == 0xFF) ? (0xFFu << nb) : 0; + *o = (ui8)((ui32)acc | fill); + memset(o + 1, X, 64); + return (ui32)(o - dst) + 1; + } + + //************************************************************************/ + /** @brief Fetches 128 bits from a destuffed bitstream buffer + * + * Returns the 128 bits starting at bit position pos of the buffer + * produced by destuff_frwd. Carries no serial state; fetches at + * independent positions can execute out of order. The byte offset + * is clamped to limit so that positions past the end of the stream + * read from the padding without leaving the buffer. + * + * @param [in] dbuf is the destuffed bitstream buffer + * @param [in] limit is the clamp offset returned by destuff_frwd + * @param [in] pos is the absolute bit position to fetch from + */ + static inline + v128_t vsx_dfetch(const ui8* dbuf, ui32 limit, ui32 pos) + { + ui32 off = pos >> 3; + off = off < limit ? off : limit; + const ui8* p = dbuf + off; + v128_t v = vsx_v128_load(p); + v128_t w = vsx_v128_load(p + 8); + int k = (int)(pos & 7); + v128_t r = vsx_u64x2_shr(v, k); + // shift left by 64 - k without branching on k == 0; vector shifts + // are modulo 64, so the shift is split into 1 and 63 - k + v128_t c = vsx_i64x2_shl(vsx_i64x2_shl(w, 1), 63 - k); + return vsx_v128_or(r, c); + } + + //************************************************************************/ + /** @brief decodes one quad, using 32 bit data + * + * @tparam N 0 for the first quad and 1 for the second quad in an + * octet + * @param inf_u_q decoded VLC code, with interleaved u values + * @param U_q U values + * @param magsgn structure for forward data buffer + * @param p bitplane at which we are decoding + * @param vn used for handling E values (stores v_n values) + * @return v128_t decoded quad + */ + template + static inline + v128_t decode_one_quad32(const v128_t inf_u_q, v128_t U_q, + frwd_struct* magsgn, ui32 p, v128_t& vn) + { + v128_t w0; // workers + v128_t insig; // lanes hold FF's if samples are insignificant + v128_t flags; // lanes hold e_k, e_1, and rho + v128_t row; // decoded row + + row = vsx_i64x2_const(0, 0); + w0 = vsx_i32x4_shuffle(inf_u_q, inf_u_q, N, N, N, N); + // we keeps e_k, e_1, and rho in w2 + flags = vsx_v128_and(w0, vsx_i32x4_const(0x1110,0x2220,0x4440,0x8880)); + insig = vsx_i32x4_eq(flags, vsx_i64x2_const(0, 0)); + if (vsx_i8x16_bitmask(insig) != 0xFFFF) //are all insignificant? + { + U_q = vsx_i32x4_shuffle(U_q, U_q, N, N, N, N); + flags = vsx_i16x8_mul(flags, vsx_i16x8_const(8,8,4,4,2,2,1,1)); + v128_t ms_vec = frwd_fetch<0xFF>(magsgn); + + // U_q holds U_q for this quad + // flags has e_k, e_1, and rho such that e_k is sitting in the + // 0x8000, e_1 in 0x800, and rho in 0x80 + + // next e_k and m_n + v128_t m_n; + w0 = vsx_u32x4_shr(flags, 15); // e_k + m_n = vsx_i32x4_sub(U_q, w0); + m_n = vsx_v128_andnot(m_n, insig); + + // find cumulative sums + // to find at which bit in ms_vec the sample starts + v128_t ex_sum, shfl, inc_sum = m_n; // inclusive scan + shfl = vsx_i32x4_shuffle(vsx_i64x2_const(0,0), inc_sum, 3, 4, 5, 6); + inc_sum = vsx_i32x4_add(inc_sum, shfl); + shfl = vsx_i64x2_shuffle(vsx_i64x2_const(0,0), inc_sum, 1, 2); + inc_sum = vsx_i32x4_add(inc_sum, shfl); + int total_mn = vsx_u16x8_extract_lane(inc_sum, 6); + ex_sum = vsx_i32x4_shuffle(vsx_i64x2_const(0,0), inc_sum, 3, 4, 5, 6); + + // find the starting byte and starting bit + v128_t byte_idx = vsx_u32x4_shr(ex_sum, 3); + v128_t bit_idx = + vsx_v128_and(ex_sum, vsx_i32x4_const(OJPH_REPEAT4(7))); + byte_idx = vsx_i8x16_swizzle(byte_idx, + vsx_i32x4_const(0x00000000, 0x04040404, 0x08080808, 0x0C0C0C0C)); + byte_idx = + vsx_i32x4_add(byte_idx, vsx_i32x4_const(OJPH_REPEAT4(0x03020100))); + v128_t d0 = vsx_i8x16_swizzle(ms_vec, byte_idx); + byte_idx = + vsx_i32x4_add(byte_idx, vsx_i32x4_const(OJPH_REPEAT4(0x01010101))); + v128_t d1 = vsx_i8x16_swizzle(ms_vec, byte_idx); + + // shift samples values to correct location + bit_idx = vsx_v128_or(bit_idx, vsx_i32x4_shl(bit_idx, 16)); + v128_t bit_shift = vsx_i8x16_swizzle( + vsx_i8x16_const(-1, 127, 63, 31, 15, 7, 3, 1, + -1, 127, 63, 31, 15, 7, 3, 1), bit_idx); + bit_shift = + vsx_i16x8_add(bit_shift, vsx_i16x8_const(OJPH_REPEAT8(0x0101))); + d0 = vsx_i16x8_mul(d0, bit_shift); + d0 = vsx_u16x8_shr(d0, 8); // we should have 8 bits in the LSB + d1 = vsx_i16x8_mul(d1, bit_shift); + d1 = // 8 in MSB + vsx_v128_and(d1, vsx_u32x4_const(OJPH_REPEAT4(0xFF00FF00))); + d0 = vsx_v128_or(d0, d1); + + // find location of e_k and mask + v128_t shift; + v128_t ones = vsx_i32x4_const(OJPH_REPEAT4(1)); + v128_t twos = vsx_i32x4_const(OJPH_REPEAT4(2)); + ui32 U_q_m1 = vsx_u32x4_extract_lane(U_q, 0) - 1u; + w0 = vsx_i32x4_sub(twos, w0); + shift = vsx_i32x4_shl(w0, U_q_m1); + ms_vec = vsx_v128_and(d0, vsx_i32x4_sub(shift, ones)); + + // next e_1 + w0 = vsx_v128_and(flags, vsx_i32x4_const(OJPH_REPEAT4(0x800))); + w0 = vsx_i32x4_eq(w0, vsx_i64x2_const(0, 0)); + w0 = vsx_v128_andnot(shift, w0); // e_1 in correct position + ms_vec = vsx_v128_or(ms_vec, w0); // e_1 + w0 = vsx_i32x4_shl(ms_vec, 31); // sign + ms_vec = vsx_v128_or(ms_vec, ones); // bin center + v128_t tvn = ms_vec; + ms_vec = vsx_i32x4_add(ms_vec, twos);// + 2 + ms_vec = vsx_i32x4_shl(ms_vec, p - 1); + ms_vec = vsx_v128_or(ms_vec, w0); // sign + row = vsx_v128_andnot(ms_vec, insig); // significant only + + ms_vec = vsx_v128_andnot(tvn, insig); // significant only + if (N == 0) // the compiler should remove one + tvn = vsx_i8x16_swizzle(ms_vec, + vsx_i32x4_const(0x07060504, 0x0F0E0D0C, -1, -1)); + else if (N == 1) + tvn = vsx_i8x16_swizzle(ms_vec, + vsx_i32x4_const(-1, 0x07060504, 0x0F0E0D0C, -1)); + else + assert(0); + vn = vsx_v128_or(vn, tvn); + + if (total_mn) + frwd_advance(magsgn, (ui32)total_mn); + } + return row; + } + + //************************************************************************/ + /** @brief decodes twos consecutive quads (one octet), using 16 bit data + * + * @param inf_u_q decoded VLC code, with interleaved u values + * @param U_q U values + * @param magsgn structure for forward data buffer + * @param p bitplane at which we are decoding + * @param vn used for handling E values (stores v_n values) + * @return v128_t decoded quad + */ + static inline + v128_t decode_two_quad16(const v128_t inf_u_q, v128_t U_q, + const ui8* dbuf, ui32 limit, ui32& pos, + ui32 p, v128_t& vn) + { + v128_t w0; // workers + v128_t insig; // lanes hold FF's if samples are insignificant + v128_t flags; // lanes hold e_k, e_1, and rho + v128_t row; // decoded row + + row = vsx_i64x2_const(0, 0); + w0 = vsx_i8x16_swizzle(inf_u_q, + vsx_i16x8_const(0x0100, 0x0100, 0x0100, 0x0100, + 0x0504, 0x0504, 0x0504, 0x0504)); + // we keeps e_k, e_1, and rho in w2 + flags = vsx_v128_and(w0, + vsx_u16x8_const(0x1110, 0x2220, 0x4440, 0x8880, + 0x1110, 0x2220, 0x4440, 0x8880)); + insig = vsx_i16x8_eq(flags, vsx_i64x2_const(0, 0)); + if (vsx_i8x16_bitmask(insig) != 0xFFFF) //are all insignificant? + { + U_q = vsx_i8x16_swizzle(U_q, + vsx_i16x8_const(0x0100, 0x0100, 0x0100, 0x0100, + 0x0504, 0x0504, 0x0504, 0x0504)); + flags = vsx_i16x8_mul(flags, vsx_i16x8_const(8,4,2,1,8,4,2,1)); + v128_t ms_vec = vsx_dfetch(dbuf, limit, pos); + + // U_q holds U_q for this quad + // flags has e_k, e_1, and rho such that e_k is sitting in the + // 0x8000, e_1 in 0x800, and rho in 0x80 + + // next e_k and m_n + v128_t m_n; + w0 = vsx_u16x8_shr(flags, 15); // e_k + m_n = vsx_i16x8_sub(U_q, w0); + m_n = vsx_v128_andnot(m_n, insig); + + // find cumulative sums + // to find at which bit in ms_vec the sample starts + v128_t ex_sum, shfl, inc_sum = m_n; // inclusive scan + shfl = vsx_i16x8_shuffle(vsx_i64x2_const(0,0), + inc_sum, 7, 8, 9, 10, 11, 12, 13, 14); + inc_sum = vsx_i16x8_add(inc_sum, shfl); + shfl = vsx_i32x4_shuffle(vsx_i64x2_const(0,0), inc_sum, 3, 4, 5, 6); + inc_sum = vsx_i16x8_add(inc_sum, shfl); + shfl = vsx_i64x2_shuffle(vsx_i64x2_const(0,0), inc_sum, 1, 2); + inc_sum = vsx_i16x8_add(inc_sum, shfl); + int total_mn = vsx_u16x8_extract_lane(inc_sum, 7); + ex_sum = vsx_i16x8_shuffle(vsx_i64x2_const(0,0), + inc_sum, 7, 8, 9, 10, 11, 12, 13, 14); + + // find the starting byte and starting bit + v128_t byte_idx = vsx_u16x8_shr(ex_sum, 3); + v128_t bit_idx = + vsx_v128_and(ex_sum, vsx_i16x8_const(OJPH_REPEAT8(7))); + byte_idx = vsx_i8x16_swizzle(byte_idx, + vsx_i16x8_const(0x0000, 0x0202, 0x0404, 0x0606, + 0x0808, 0x0A0A, 0x0C0C, 0x0E0E)); + byte_idx = + vsx_i16x8_add(byte_idx, vsx_i16x8_const(OJPH_REPEAT8(0x0100))); + v128_t d0 = vsx_i8x16_swizzle(ms_vec, byte_idx); + byte_idx = + vsx_i16x8_add(byte_idx, vsx_i16x8_const(OJPH_REPEAT8(0x0101))); + v128_t d1 = vsx_i8x16_swizzle(ms_vec, byte_idx); + + // shift samples values to correct location + v128_t bit_shift = vsx_i8x16_swizzle( + vsx_i8x16_const(-1, 127, 63, 31, 15, 7, 3, 1, + -1, 127, 63, 31, 15, 7, 3, 1), bit_idx); + bit_shift = + vsx_i16x8_add(bit_shift, vsx_i16x8_const(OJPH_REPEAT8(0x0101))); + d0 = vsx_i16x8_mul(d0, bit_shift); + d0 = vsx_u16x8_shr(d0, 8); // we should have 8 bits in the LSB + d1 = vsx_i16x8_mul(d1, bit_shift); + d1 = // 8 in MSB + vsx_v128_and(d1, vsx_i16x8_const(OJPH_REPEAT8((si16)0xFF00))); + d0 = vsx_v128_or(d0, d1); + + // find location of e_k and mask + v128_t shift, t0, t1; + v128_t ones = vsx_i16x8_const(OJPH_REPEAT8(1)); + v128_t twos = vsx_i16x8_const(OJPH_REPEAT8(2)); + v128_t U_q_m1 = vsx_i32x4_sub(U_q, ones); + ui32 Uq0 = vsx_u16x8_extract_lane(U_q_m1, 0); + ui32 Uq1 = vsx_u16x8_extract_lane(U_q_m1, 4); + w0 = vsx_i16x8_sub(twos, w0); + t0 = vsx_v128_and(w0, vsx_i64x2_const(-1, 0)); + t1 = vsx_v128_and(w0, vsx_i64x2_const(0, -1)); + t0 = vsx_i32x4_shl(t0, Uq0); + t1 = vsx_i32x4_shl(t1, Uq1); + shift = vsx_v128_or(t0, t1); + ms_vec = vsx_v128_and(d0, vsx_i16x8_sub(shift, ones)); + + // next e_1 + w0 = vsx_v128_and(flags, vsx_i16x8_const(OJPH_REPEAT8(0x800))); + w0 = vsx_i16x8_eq(w0, vsx_i64x2_const(0, 0)); + w0 = vsx_v128_andnot(shift, w0); // e_1 in correct position + ms_vec = vsx_v128_or(ms_vec, w0); // e_1 + w0 = vsx_i16x8_shl(ms_vec, 15); // sign + ms_vec = vsx_v128_or(ms_vec, ones); // bin center + v128_t tvn = ms_vec; + ms_vec = vsx_i16x8_add(ms_vec, twos);// + 2 + ms_vec = vsx_i16x8_shl(ms_vec, p - 1); + ms_vec = vsx_v128_or(ms_vec, w0); // sign + row = vsx_v128_andnot(ms_vec, insig); // significant only + + ms_vec = vsx_v128_andnot(tvn, insig); // significant only + w0 = vsx_i8x16_swizzle(ms_vec, + vsx_i16x8_const(0x0302, 0x0706, -1, -1, -1, -1, -1, -1)); + vn = vsx_v128_or(vn, w0); + w0 = vsx_i8x16_swizzle(ms_vec, + vsx_i16x8_const(-1, 0x0B0A, 0x0F0E, -1, -1, -1, -1, -1)); + vn = vsx_v128_or(vn, w0); + + pos += (ui32)total_mn; + } + return row; + } + + + //************************************************************************/ + /** @brief Decodes one codeblock, processing the cleanup, siginificance + * propagation, and magnitude refinement pass + * + * @param [in] coded_data is a pointer to bitstream + * @param [in] decoded_data is a pointer to decoded codeblock data buf. + * @param [in] missing_msbs is the number of missing MSBs + * @param [in] num_passes is the number of passes: 1 if CUP only, + * 2 for CUP+SPP, and 3 for CUP+SPP+MRP + * @param [in] lengths1 is the length of cleanup pass + * @param [in] lengths2 is the length of refinement passes (either SPP + * only or SPP+MRP) + * @param [in] width is the decoded codeblock width + * @param [in] height is the decoded codeblock height + * @param [in] stride is the decoded codeblock buffer stride + * @param [in] stripe_causal is true for stripe causal mode + */ + bool ojph_decode_codeblock_vsx(ui8* coded_data, ui32* decoded_data, + ui32 missing_msbs, ui32 num_passes, + ui32 lengths1, ui32 lengths2, + ui32 width, ui32 height, ui32 stride, + bool stripe_causal) + { + static bool insufficient_precision = false; + static bool modify_code = false; + static bool truncate_spp_mrp = false; + + if (num_passes > 1 && lengths2 == 0) + { + OJPH_WARN(0x00010001, "A malformed codeblock that has more than " + "one coding pass, but zero length for " + "2nd and potential 3rd pass.\n"); + num_passes = 1; + } + + if (num_passes > 3) + { + OJPH_WARN(0x00010002, "We do not support more than 3 coding passes; " + "This codeblocks has %d passes.\n", + num_passes); + return false; + } + + if (missing_msbs > 30) // p < 0 + { + if (insufficient_precision == false) + { + insufficient_precision = true; + OJPH_WARN(0x00010003, "32 bits are not enough to decode this " + "codeblock. This message will not be " + "displayed again.\n"); + } + return false; + } + else if (missing_msbs == 30) // p == 0 + { // not enough precision to decode and set the bin center to 1 + if (modify_code == false) { + modify_code = true; + OJPH_WARN(0x00010004, "Not enough precision to decode the cleanup " + "pass. The code can be modified to support " + "this case. This message will not be " + "displayed again.\n"); + } + return false; // 32 bits are not enough to decode this + } + else if (missing_msbs == 29) // if p is 1, then num_passes must be 1 + { + if (num_passes > 1) { + num_passes = 1; + if (truncate_spp_mrp == false) { + truncate_spp_mrp = true; + OJPH_WARN(0x00010005, "Not enough precision to decode the SgnProp " + "nor MagRef passes; both will be skipped. " + "This message will not be displayed " + "again.\n"); + } + } + } + ui32 p = 30 - missing_msbs; // The least significant bitplane for CUP + // There is a way to handle the case of p == 0, but a different path + // is required + + if (lengths1 < 2) + { + OJPH_WARN(0x00010006, "Wrong codeblock length.\n"); + return false; + } + + // read scup and fix the bytes there + int lcup, scup; + lcup = (int)lengths1; // length of CUP + //scup is the length of MEL + VLC + scup = (((int)coded_data[lcup-1]) << 4) + (coded_data[lcup-2] & 0xF); + if (scup < 2 || scup > lcup || scup > 4079) //something is wrong + return false; + + // The temporary storage scratch holds two types of data in an + // interleaved fashion. The interleaving allows us to use one + // memory pointer. + // We have one entry for a decoded VLC code, and one entry for UVLC. + // Entries are 16 bits each, corresponding to one quad, + // but since we want to use XMM registers of the SSE family + // of SIMD; we allocated 16 bytes or more per quad row; that is, + // the width is no smaller than 16 bytes (or 8 entries), and the + // height is 512 quads + // Each VLC entry contains, in the following order, starting + // from MSB + // e_k (4bits), e_1 (4bits), rho (4bits), useless for step 2 (4bits) + // Each entry in UVLC contains u_q + // One extra row to handle the case of SPP propagating downwards + // when codeblock width is 4 + ui16 scratch[8 * 513] = {0}; // 8+ kB + + // We need an extra two entries (one inf and one u_q) beyond + // the last column. + // If the block width is 4 (2 quads), then we use sstr of 8 + // (enough for 4 quads). If width is 8 (4 quads) we use + // sstr is 16 (enough for 8 quads). For a width of 16 (8 + // quads), we use 24 (enough for 12 quads). + ui32 sstr = ((width + 2u) + 7u) & ~7u; // multiples of 8 + + assert((stride & 0x3) == 0); + + ui32 mmsbp2 = missing_msbs + 2; + + // The cleanup pass is decoded in two steps; in step one, + // the VLC and MEL segments are decoded, generating a record that + // has 2 bytes per quad. The 2 bytes contain, u, rho, e^1 & e^k. + // This information should be sufficient for the next step. + // In step 2, we decode the MagSgn segment. + + // step 1 decoding VLC and MEL segments + { + // init structures + dec_mel_st mel; + mel_init(&mel, coded_data, lcup, scup); + rev_struct vlc; + rev_init(&vlc, coded_data, lcup, scup); + + int run = mel_get_run(&mel); // decode runs of events from MEL bitstrm + // data represented as runs of 0 events + // See mel_decode description + + ui32 vlc_val; + ui32 c_q = 0; + ui16 *sp = scratch; + //initial quad row + for (ui32 x = 0; x < width; sp += 4) + { + // decode VLC + ///////////// + + // first quad + vlc_val = rev_fetch(&vlc); + + //decode VLC using the context c_q and the head of VLC bitstream + ui16 t0 = vlc_tbl0[ c_q + (vlc_val & 0x7F) ]; + + // if context is zero, use one MEL event + if (c_q == 0) //zero context + { + run -= 2; //subtract 2, since events number if multiplied by 2 + + // Is the run terminated in 1? if so, use decoded VLC code, + // otherwise, discard decoded data, since we will decoded again + // using a different context + t0 = (run == -1) ? t0 : 0; + + // is run -1 or -2? this means a run has been consumed + if (run < 0) + run = mel_get_run(&mel); // get another run + } + //run -= (c_q == 0) ? 2 : 0; + //t0 = (c_q != 0 || run == -1) ? t0 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + sp[0] = t0; + x += 2; + + // prepare context for the next quad; eqn. 1 in ITU T.814 + c_q = ((t0 & 0x10U) << 3) | ((t0 & 0xE0U) << 2); + + //remove data from vlc stream (0 bits are removed if vlc is not used) + vlc_val = rev_advance(&vlc, t0 & 0x7); + + //second quad + ui16 t1 = 0; + + //decode VLC using the context c_q and the head of VLC bitstream + t1 = vlc_tbl0[c_q + (vlc_val & 0x7F)]; + + // if context is zero, use one MEL event + if (c_q == 0 && x < width) //zero context + { + run -= 2; //subtract 2, since events number if multiplied by 2 + + // if event is 0, discard decoded t1 + t1 = (run == -1) ? t1 : 0; + + if (run < 0) // have we consumed all events in a run + run = mel_get_run(&mel); // if yes, then get another run + } + t1 = x < width ? t1 : 0; + //run -= (c_q == 0 && x < width) ? 2 : 0; + //t1 = (c_q != 0 || run == -1) ? t1 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + sp[2] = t1; + x += 2; + + //prepare context for the next quad, eqn. 1 in ITU T.814 + c_q = ((t1 & 0x10U) << 3) | ((t1 & 0xE0U) << 2); + + //remove data from vlc stream, if qinf is not used, cwdlen is 0 + vlc_val = rev_advance(&vlc, t1 & 0x7); + + // decode u + ///////////// + // uvlc_mode is made up of u_offset bits from the quad pair + ui32 uvlc_mode = ((t0 & 0x8U) << 3) | ((t1 & 0x8U) << 4); + if (uvlc_mode == 0xc0)// if both u_offset are set, get an event from + { // the MEL run of events + run -= 2; //subtract 2, since events number if multiplied by 2 + + uvlc_mode += (run == -1) ? 0x40 : 0; // increment uvlc_mode by + // is 0x40 + + if (run < 0)//if run is consumed (run is -1 or -2), get another run + run = mel_get_run(&mel); + } + //run -= (uvlc_mode == 0xc0) ? 2 : 0; + //uvlc_mode += (uvlc_mode == 0xc0 && run == -1) ? 0x40 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + + //decode uvlc_mode to get u for both quads + ui32 uvlc_entry = uvlc_tbl0[uvlc_mode + (vlc_val & 0x3F)]; + //remove total prefix length + vlc_val = rev_advance(&vlc, uvlc_entry & 0x7); + uvlc_entry >>= 3; + //extract suffixes for quad 0 and 1 + ui32 len = uvlc_entry & 0xF; //suffix length for 2 quads + ui32 tmp = vlc_val & ((1 << len) - 1); //suffix value for 2 quads + vlc_val = rev_advance(&vlc, len); + uvlc_entry >>= 4; + // quad 0 length + len = uvlc_entry & 0x7; // quad 0 suffix length + uvlc_entry >>= 3; + ui16 u_q = (ui16)(1 + (uvlc_entry&7) + (tmp&~(0xFFU<> 3) + (tmp >> len)); //kappa == 1 + sp[3] = u_q; + } + sp[0] = sp[1] = 0; + + //non initial quad rows + for (ui32 y = 2; y < height; y += 2) + { + c_q = 0; // context + ui16 *sp = scratch + (y >> 1) * sstr; // this row of quads + + for (ui32 x = 0; x < width; sp += 4) + { + // decode VLC + ///////////// + + // sigma_q (n, ne, nf) + c_q |= ((sp[0 - (si32)sstr] & 0xA0U) << 2); + c_q |= ((sp[2 - (si32)sstr] & 0x20U) << 4); + + // first quad + vlc_val = rev_fetch(&vlc); + + //decode VLC using the context c_q and the head of VLC bitstream + ui16 t0 = vlc_tbl1[ c_q + (vlc_val & 0x7F) ]; + + // if context is zero, use one MEL event + if (c_q == 0) //zero context + { + run -= 2; //subtract 2, since events number is multiplied by 2 + + // Is the run terminated in 1? if so, use decoded VLC code, + // otherwise, discard decoded data, since we will decoded again + // using a different context + t0 = (run == -1) ? t0 : 0; + + // is run -1 or -2? this means a run has been consumed + if (run < 0) + run = mel_get_run(&mel); // get another run + } + //run -= (c_q == 0) ? 2 : 0; + //t0 = (c_q != 0 || run == -1) ? t0 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + sp[0] = t0; + x += 2; + + // prepare context for the next quad; eqn. 2 in ITU T.814 + // sigma_q (w, sw) + c_q = ((t0 & 0x40U) << 2) | ((t0 & 0x80U) << 1); + // sigma_q (nw) + c_q |= sp[0 - (si32)sstr] & 0x80; + // sigma_q (n, ne, nf) + c_q |= ((sp[2 - (si32)sstr] & 0xA0U) << 2); + c_q |= ((sp[4 - (si32)sstr] & 0x20U) << 4); + + //remove data from vlc stream (0 bits are removed if vlc is unused) + vlc_val = rev_advance(&vlc, t0 & 0x7); + + //second quad + ui16 t1 = 0; + + //decode VLC using the context c_q and the head of VLC bitstream + t1 = vlc_tbl1[ c_q + (vlc_val & 0x7F)]; + + // if context is zero, use one MEL event + if (c_q == 0 && x < width) //zero context + { + run -= 2; //subtract 2, since events number if multiplied by 2 + + // if event is 0, discard decoded t1 + t1 = (run == -1) ? t1 : 0; + + if (run < 0) // have we consumed all events in a run + run = mel_get_run(&mel); // if yes, then get another run + } + t1 = x < width ? t1 : 0; + //run -= (c_q == 0 && x < width) ? 2 : 0; + //t1 = (c_q != 0 || run == -1) ? t1 : 0; + //if (run < 0) + // run = mel_get_run(&mel); // get another run + sp[2] = t1; + x += 2; + + // partial c_q, will be completed when we process the next quad + // sigma_q (w, sw) + c_q = ((t1 & 0x40U) << 2) | ((t1 & 0x80U) << 1); + // sigma_q (nw) + c_q |= sp[2 - (si32)sstr] & 0x80; + + //remove data from vlc stream, if qinf is not used, cwdlen is 0 + vlc_val = rev_advance(&vlc, t1 & 0x7); + + // decode u + ///////////// + // uvlc_mode is made up of u_offset bits from the quad pair + ui32 uvlc_mode = ((t0 & 0x8U) << 3) | ((t1 & 0x8U) << 4); + ui32 uvlc_entry = uvlc_tbl1[uvlc_mode + (vlc_val & 0x3F)]; + //remove total prefix length + vlc_val = rev_advance(&vlc, uvlc_entry & 0x7); + uvlc_entry >>= 3; + //extract suffixes for quad 0 and 1 + ui32 len = uvlc_entry & 0xF; //suffix length for 2 quads + ui32 tmp = vlc_val & ((1 << len) - 1); //suffix value for 2 quads + vlc_val = rev_advance(&vlc, len); + uvlc_entry >>= 4; + // quad 0 length + len = uvlc_entry & 0x7; // quad 0 suffix length + uvlc_entry >>= 3; + ui16 u_q = (ui16)((uvlc_entry & 7) + (tmp & ~(0xFU << len))); //u_q + sp[1] = u_q; + u_q = (ui16)((uvlc_entry >> 3) + (tmp >> len)); // u_q + sp[3] = u_q; + } + sp[0] = sp[1] = 0; + } + } + + // step2 we decode magsgn + // mmsbp2 equals K_max + 1 (we decode up to K_max bits + 1 sign bit) + // The 32 bit path decode 16 bits data, for which one would think + // 16 bits are enough, because we want to put in the center of the + // bin. + // If you have mmsbp2 equals 16 bit, and reversible coding, and + // no bitplanes are missing, then we can decoding using the 16 bit + // path, but we are not doing this here. + if (mmsbp2 >= 16) + { + // We allocate a scratch row for storing v_n values. + // We have 512 quads horizontally. + // We may go beyond the last entry by up to 4 entries. + // Here we allocate additional 8 entries. + // There are two rows in this structure, the bottom + // row is used to store processed entries. + const int v_n_size = 512 + 8; + ui32 v_n_scratch[2 * v_n_size] = {0}; // 4+ kB + + frwd_struct magsgn; + frwd_init<0xFF>(&magsgn, coded_data, lcup - scup); + + { + ui16 *sp = scratch; + ui32 *vp = v_n_scratch; + ui32 *dp = decoded_data; + vp[0] = 2; // for easy calculation of emax + + for (ui32 x = 0; x < width; x += 4, sp += 4, vp += 2, dp += 4) + { + //here we process two quads + v128_t w0, w1; // workers + v128_t inf_u_q, U_q; + // determine U_q + { + inf_u_q = vsx_v128_load(sp); + U_q = vsx_u32x4_shr(inf_u_q, 16); + + w0 = vsx_i32x4_gt(U_q, vsx_u32x4_splat(mmsbp2)); + ui32 i = vsx_i8x16_bitmask(w0); + if (i & 0xFF) // only the lower two U_q + return false; + } + + v128_t vn = vsx_i32x4_const(OJPH_REPEAT4(2)); + v128_t row0 = decode_one_quad32<0>(inf_u_q, U_q, &magsgn, p, vn); + v128_t row1 = decode_one_quad32<1>(inf_u_q, U_q, &magsgn, p, vn); + w0 = vsx_v128_load(vp); + w0 = vsx_v128_and(w0, vsx_i32x4_const(-1,0,0,0)); + w0 = vsx_v128_or(w0, vn); + vsx_v128_store(vp, w0); + + //interleave in ssse3 style + + w0 = vsx_i32x4_shuffle(row0, row1, 0, 4, 1, 5); + w1 = vsx_i32x4_shuffle(row0, row1, 2, 6, 3, 7); + row0 = vsx_i32x4_shuffle(w0, w1, 0, 4, 1, 5); + row1 = vsx_i32x4_shuffle(w0, w1, 2, 6, 3, 7); + vsx_v128_store(dp, row0); + vsx_v128_store(dp + stride, row1); + } + } + + for (ui32 y = 2; y < height; y += 2) + { + { + // perform 31 - count_leading_zeros(*vp) here + ui32 *vp = v_n_scratch; + const v128_t lut_lo = vsx_i8x16_const( + 31, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4 + ); + const v128_t lut_hi = vsx_i8x16_const( + 31, 3, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0 + ); + const v128_t nibble_mask = vsx_i8x16_const(OJPH_REPEAT16(0x0F)); + const v128_t byte_offset8 = vsx_i16x8_const(OJPH_REPEAT8(8)); + const v128_t byte_offset16 = vsx_i16x8_const(OJPH_REPEAT8(16)); + const v128_t cc = vsx_i32x4_const(OJPH_REPEAT4(31)); + for (ui32 x = 0; x <= width; x += 8, vp += 4) + { + v128_t v, t; // workers + v = vsx_v128_load(vp); + + t = vsx_v128_and(nibble_mask, v); + v = vsx_v128_and(vsx_u16x8_shr(v, 4), nibble_mask); + t = vsx_i8x16_swizzle(lut_lo, t); + v = vsx_i8x16_swizzle(lut_hi, v); + v = vsx_u8x16_min(v, t); + + t = vsx_u16x8_shr(v, 8); + v = vsx_v128_or(v, byte_offset8); + v = vsx_u8x16_min(v, t); + + t = vsx_u32x4_shr(v, 16); + v = vsx_v128_or(v, byte_offset16); + v = vsx_u8x16_min(v, t); + + v = vsx_i16x8_sub(cc, v); + vsx_v128_store(vp + v_n_size, v); + } + } + + ui32 *vp = v_n_scratch; + ui16 *sp = scratch + (y >> 1) * sstr; + ui32 *dp = decoded_data + y * stride; + vp[0] = 2; // for easy calculation of emax + + for (ui32 x = 0; x < width; x += 4, sp += 4, vp += 2, dp += 4) + { + //process two quads + v128_t w0, w1; // workers + v128_t inf_u_q, U_q; + // determine U_q + { + v128_t gamma, emax, kappa, u_q; // needed locally + + inf_u_q = vsx_v128_load(sp); + gamma = + vsx_v128_and(inf_u_q, vsx_i32x4_const(OJPH_REPEAT4(0xF0))); + w0 = vsx_i32x4_sub(gamma, vsx_i32x4_const(OJPH_REPEAT4(1))); + gamma = vsx_v128_and(gamma, w0); + gamma = vsx_i32x4_eq(gamma, vsx_i64x2_const(0, 0)); + + emax = vsx_v128_load(vp + v_n_size); + w0 = vsx_i32x4_shuffle(emax, vsx_i64x2_const(0,0), 1, 2, 3, 4); + emax = vsx_i16x8_max(w0, emax); // no max_epi32 in ssse3 + emax = vsx_v128_andnot(emax, gamma); + + kappa = vsx_i32x4_const(OJPH_REPEAT4(1)); + kappa = vsx_i16x8_max(emax, kappa); // no max_epi32 in ssse3 + + u_q = vsx_u32x4_shr(inf_u_q, 16); + U_q = vsx_i32x4_add(u_q, kappa); + + w0 = vsx_i32x4_gt(U_q, vsx_u32x4_splat(mmsbp2)); + ui32 i = vsx_i8x16_bitmask(w0); + if (i & 0xFF) // only the lower two U_q + return false; + } + + v128_t vn = vsx_i32x4_const(OJPH_REPEAT4(2)); + v128_t row0 = decode_one_quad32<0>(inf_u_q, U_q, &magsgn, p, vn); + v128_t row1 = decode_one_quad32<1>(inf_u_q, U_q, &magsgn, p, vn); + w0 = vsx_v128_load(vp); + w0 = vsx_v128_and(w0, vsx_i32x4_const(-1,0,0,0)); + w0 = vsx_v128_or(w0, vn); + vsx_v128_store(vp, w0); + + //interleave in ssse3 style + w0 = vsx_i32x4_shuffle(row0, row1, 0, 4, 1, 5); + w1 = vsx_i32x4_shuffle(row0, row1, 2, 6, 3, 7); + row0 = vsx_i32x4_shuffle(w0, w1, 0, 4, 1, 5); + row1 = vsx_i32x4_shuffle(w0, w1, 2, 6, 3, 7); + vsx_v128_store(dp, row0); + vsx_v128_store(dp + stride, row1); + } + } + } + else + { + // reduce bitplane by 16 because we now have 16 bits instead of 32 + p -= 16; + + // We allocate a scratch row for storing v_n values. + // We have 512 quads horizontally. + // We may go beyond the last entry by up to 8 entries. + // Therefore we allocate additional 8 entries. + // There are two rows in this structure, the bottom + // row is used to store processed entries. + const int v_n_size = 512 + 8; + ui16 v_n_scratch[2 * v_n_size] = {0}; // 2+ kB + + // destuff the MagSgn bitstream upfront; per-quad consumption then + // advances a bit position in a GPR (see destuff_frwd) + const ui32 dbuf_cap = 4096 * 15 / 8; + ui8 dbuf[dbuf_cap + 72]; + ui32 limit = destuff_frwd<0xFF>(coded_data, lcup - scup, + dbuf, dbuf_cap); + ui32 pos = 0; + + { + ui16 *sp = scratch; + ui16 *vp = v_n_scratch; + ui32 *dp = decoded_data; + vp[0] = 2; // for easy calculation of emax + + for (ui32 x = 0; x < width; x += 4, sp += 4, vp += 2, dp += 4) + { + //here we process two quads + v128_t w0, w1; // workers + v128_t inf_u_q, U_q; + // determine U_q + { + inf_u_q = vsx_v128_load(sp); + U_q = vsx_u32x4_shr(inf_u_q, 16); + + w0 = vsx_i32x4_gt(U_q, vsx_u32x4_splat(mmsbp2)); + ui32 i = vsx_i8x16_bitmask(w0); + if (i & 0xFF) // only the lower two U_q + return false; + } + + v128_t vn = vsx_i16x8_const(OJPH_REPEAT8(2)); + v128_t row = decode_two_quad16(inf_u_q, U_q, dbuf, limit, pos, p, vn); + w0 = vsx_v128_load(vp); + w0 = vsx_v128_and(w0, vsx_i16x8_const(-1,0,0,0,0,0,0,0)); + w0 = vsx_v128_or(w0, vn); + vsx_v128_store(vp, w0); + + //interleave in ssse3 style + w0 = vsx_i8x16_swizzle(row, + vsx_i16x8_const(-1, 0x0100, -1, 0x0504, + -1, 0x0908, -1, 0x0D0C)); + vsx_v128_store(dp, w0); + w1 = vsx_i8x16_swizzle(row, + vsx_i16x8_const(-1, 0x0302, -1, 0x0706, + -1, 0x0B0A, -1, 0x0F0E)); + vsx_v128_store(dp + stride, w1); + } + } + + for (ui32 y = 2; y < height; y += 2) + { + { + // perform 15 - count_leading_zeros(*vp) here + ui16 *vp = v_n_scratch; + const v128_t lut_lo = vsx_i8x16_const( + 15, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4 + ); + const v128_t lut_hi = vsx_i8x16_const( + 15, 3, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0 + ); + const v128_t nibble_mask = vsx_i8x16_const(OJPH_REPEAT16(0x0F)); + const v128_t byte_offset8 = vsx_i16x8_const(OJPH_REPEAT8(8)); + const v128_t cc = vsx_i16x8_const(OJPH_REPEAT8(15)); + for (ui32 x = 0; x <= width; x += 16, vp += 8) + { + v128_t v, t; // workers + v = vsx_v128_load(vp); + + t = vsx_v128_and(nibble_mask, v); + v = vsx_v128_and(vsx_u16x8_shr(v, 4), nibble_mask); + t = vsx_i8x16_swizzle(lut_lo, t); + v = vsx_i8x16_swizzle(lut_hi, v); + v = vsx_u8x16_min(v, t); + + t = vsx_u16x8_shr(v, 8); + v = vsx_v128_or(v, byte_offset8); + v = vsx_u8x16_min(v, t); + + v = vsx_i16x8_sub(cc, v); + vsx_v128_store(vp + v_n_size, v); + } + } + + ui16 *vp = v_n_scratch; + ui16 *sp = scratch + (y >> 1) * sstr; + ui32 *dp = decoded_data + y * stride; + vp[0] = 2; // for easy calculation of emax + + for (ui32 x = 0; x < width; x += 4, sp += 4, vp += 2, dp += 4) + { + //process two quads + v128_t w0, w1; // workers + v128_t inf_u_q, U_q; + // determine U_q + { + v128_t gamma, emax, kappa, u_q; // needed locally + + inf_u_q = vsx_v128_load(sp); + gamma = + vsx_v128_and(inf_u_q, vsx_i32x4_const(OJPH_REPEAT4(0xF0))); + w0 = vsx_i32x4_sub(gamma, vsx_i32x4_const(OJPH_REPEAT4(1))); + gamma = vsx_v128_and(gamma, w0); + gamma = vsx_i32x4_eq(gamma, vsx_i64x2_const(0, 0)); + + emax = vsx_v128_load(vp + v_n_size); + w0 = vsx_i16x8_shuffle(emax, + vsx_i64x2_const(0, 0), 1, 2, 3, 4, 5, 6, 7, 8); + emax = vsx_i16x8_max(w0, emax); // no max_epi32 in ssse3 + emax = vsx_i8x16_swizzle(emax, + vsx_i16x8_const(0x0100, -1, 0x0302, -1, + 0x0504, -1, 0x0706, -1)); + emax = vsx_v128_andnot(emax, gamma); + + kappa = vsx_i32x4_const(OJPH_REPEAT4(1)); + kappa = vsx_i16x8_max(emax, kappa); // no max_epi32 in ssse3 + + u_q = vsx_u32x4_shr(inf_u_q, 16); + U_q = vsx_i32x4_add(u_q, kappa); + + w0 = vsx_i32x4_gt(U_q, vsx_u32x4_splat(mmsbp2)); + ui32 i = vsx_i8x16_bitmask(w0); + if (i & 0xFF) // only the lower two U_q + return false; + } + + v128_t vn = vsx_i16x8_const(OJPH_REPEAT8(2)); + v128_t row = decode_two_quad16(inf_u_q, U_q, dbuf, limit, pos, p, vn); + w0 = vsx_v128_load(vp); + w0 = vsx_v128_and(w0, vsx_i16x8_const(-1,0,0,0,0,0,0,0)); + w0 = vsx_v128_or(w0, vn); + vsx_v128_store(vp, w0); + + w0 = vsx_i8x16_swizzle(row, + vsx_i16x8_const(-1, 0x0100, -1, 0x0504, + -1, 0x0908, -1, 0x0D0C)); + vsx_v128_store(dp, w0); + w1 = vsx_i8x16_swizzle(row, + vsx_i16x8_const(-1, 0x0302, -1, 0x0706, + -1, 0x0B0A, -1, 0x0F0E)); + vsx_v128_store(dp + stride, w1); + } + } + + // increase bitplane back by 16 because we need to process 32 bits + p += 16; + } + + if (num_passes > 1) + { + // We use scratch again, we can divide it into multiple regions + // sigma holds all the significant samples, and it cannot + // be modified after it is set. it will be used during the + // Magnitude Refinement Pass + ui16* const sigma = scratch; + + ui32 mstr = (width + 3u) >> 2; // divide by 4, since each + // ui16 contains 4 columns + mstr = ((mstr + 2u) + 7u) & ~7u; // multiples of 8 + + // We re-arrange quad significance, where each 4 consecutive + // bits represent one quad, into column significance, where, + // each 4 consequtive bits represent one column of 4 rows + { + ui32 y; + + const v128_t mask_3 = vsx_i32x4_const(OJPH_REPEAT4(0x30)); + const v128_t mask_C = vsx_i32x4_const(OJPH_REPEAT4(0xC0)); + const v128_t shuffle_mask = vsx_i32x4_const(0x0C080400,-1,-1,-1); + for (y = 0; y < height; y += 4) + { + ui16* sp = scratch + (y >> 1) * sstr; + ui16* dp = sigma + (y >> 2) * mstr; + for (ui32 x = 0; x < width; x += 8, sp += 8, dp += 2) + { + v128_t s0, s1, u3, uC, t0, t1; + + s0 = vsx_v128_load(sp); + u3 = vsx_v128_and(s0, mask_3); + u3 = vsx_u32x4_shr(u3, 4); + uC = vsx_v128_and(s0, mask_C); + uC = vsx_u32x4_shr(uC, 2); + t0 = vsx_v128_or(u3, uC); + + s1 = vsx_v128_load(sp + sstr); + u3 = vsx_v128_and(s1, mask_3); + u3 = vsx_u32x4_shr(u3, 2); + uC = vsx_v128_and(s1, mask_C); + t1 = vsx_v128_or(u3, uC); + + v128_t r = vsx_v128_or(t0, t1); + r = vsx_i8x16_swizzle(r, shuffle_mask); + + vsx_v128_store32_lane(dp, r, 0); + } + dp[0] = 0; // set an extra entry on the right with 0 + } + { + // reset one row after the codeblock + ui16* dp = sigma + (y >> 2) * mstr; + v128_t zero = vsx_i64x2_const(0, 0); + for (ui32 x = 0; x < width; x += 32, dp += 8) + vsx_v128_store(dp, zero); + dp[0] = 0; // set an extra entry on the right with 0 + } + } + + // We perform Significance Propagation Pass here + { + // This stores significance information of the previous + // 4 rows. Significance information in this array includes + // all signicant samples in bitplane p - 1; that is, + // significant samples for bitplane p (discovered during the + // cleanup pass and stored in sigma) and samples that have recently + // became significant (during the SPP) in bitplane p-1. + // We store enough for the widest row, containing 1024 columns, + // which is equivalent to 256 of ui16, since each stores 4 columns. + // We add an extra 8 entries, just in case we need more + ui16 prev_row_sig[256 + 8] = {0}; // 528 Bytes + + frwd_struct sigprop; + frwd_init<0>(&sigprop, coded_data + lengths1, (int)lengths2); + + for (ui32 y = 0; y < height; y += 4) + { + ui32 pattern = 0xFFFFu; // a pattern needed samples + if (height - y < 4) { + pattern = 0x7777u; + if (height - y < 3) { + pattern = 0x3333u; + if (height - y < 2) + pattern = 0x1111u; + } + } + + // prev holds sign. info. for the previous quad, together + // with the rows on top of it and below it. + ui32 prev = 0; + ui16 *prev_sig = prev_row_sig; + ui16 *cur_sig = sigma + (y >> 2) * mstr; + ui32 *dpp = decoded_data + y * stride; + for (ui32 x = 0; x < width; x += 4, dpp += 4, ++cur_sig, ++prev_sig) + { + // only rows and columns inside the stripe are included + si32 s = (si32)x + 4 - (si32)width; + s = ojph_max(s, 0); + pattern = pattern >> (s * 4); + + // We first find locations that need to be tested (potential + // SPP members); these location will end up in mbr + // In each iteration, we produce 16 bits because cwd can have + // up to 16 bits of significance information, followed by the + // corresponding 16 bits of sign information; therefore, it is + // sufficient to fetch 32 bit data per loop. + + // Althougth we are interested in 16 bits only, we load 32 bits. + // For the 16 bits we are producing, we need the next 4 bits -- + // We need data for at least 5 columns out of 8. + // Therefore loading 32 bits is easier than loading 16 bits + // twice. + ui32 ps; memcpy(&ps, prev_sig, sizeof(ps)); + ui32 ns; memcpy(&ns, cur_sig + mstr, sizeof(ns)); + ui32 u = (ps & 0x88888888) >> 3; // the row on top + if (!stripe_causal) + u |= (ns & 0x11111111) << 3; // the row below + + ui32 cs; memcpy(&cs, cur_sig, sizeof(cs)); + // vertical integration + ui32 mbr = cs; // this sig. info. + mbr |= (cs & 0x77777777) << 1; //above neighbors + mbr |= (cs & 0xEEEEEEEE) >> 1; //below neighbors + mbr |= u; + // horizontal integration + ui32 t = mbr; + mbr |= t << 4; // neighbors on the left + mbr |= t >> 4; // neighbors on the right + mbr |= prev >> 12; // significance of previous group + + // remove outside samples, and already significant samples + mbr &= pattern; + mbr &= ~cs; + + // find samples that become significant during the SPP + ui32 new_sig = mbr; + if (new_sig) + { + v128_t cwd_vec = frwd_fetch<0>(&sigprop); + ui32 cwd = vsx_u32x4_extract_lane(cwd_vec, 0); + + ui32 cnt = 0; + ui32 col_mask = 0xFu; + ui32 inv_sig = ~cs & pattern; + for (int i = 0; i < 16; i += 4, col_mask <<= 4) + { + if ((col_mask & new_sig) == 0) + continue; + + //scan one column + ui32 sample_mask = 0x1111u & col_mask; + if (new_sig & sample_mask) + { + new_sig &= ~sample_mask; + if (cwd & 1) + { + ui32 t = 0x33u << i; + new_sig |= t & inv_sig; + } + cwd >>= 1; ++cnt; + } + + sample_mask <<= 1; + if (new_sig & sample_mask) + { + new_sig &= ~sample_mask; + if (cwd & 1) + { + ui32 t = 0x76u << i; + new_sig |= t & inv_sig; + } + cwd >>= 1; ++cnt; + } + + sample_mask <<= 1; + if (new_sig & sample_mask) + { + new_sig &= ~sample_mask; + if (cwd & 1) + { + ui32 t = 0xECu << i; + new_sig |= t & inv_sig; + } + cwd >>= 1; ++cnt; + } + + sample_mask <<= 1; + if (new_sig & sample_mask) + { + new_sig &= ~sample_mask; + if (cwd & 1) + { + ui32 t = 0xC8u << i; + new_sig |= t & inv_sig; + } + cwd >>= 1; ++cnt; + } + } + + if (new_sig) + { + // Spread new_sig, such that each bit is in one byte with a + // value of 0 if new_sig bit is 0, and 0xFF if new_sig is 1 + v128_t new_sig_vec = vsx_i16x8_splat((si16)new_sig); + new_sig_vec = vsx_i8x16_swizzle(new_sig_vec, + vsx_i8x16_const(0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1)); + new_sig_vec = vsx_v128_and(new_sig_vec, + vsx_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); + new_sig_vec = vsx_i8x16_eq(new_sig_vec, + vsx_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); + + // find cumulative sums + // to find which bit in cwd we should extract + v128_t ex_sum, shfl, inc_sum = new_sig_vec; // inclusive scan + inc_sum = vsx_i8x16_abs(inc_sum); // cvrt to 0 or 1 + shfl = vsx_i8x16_shuffle(vsx_i64x2_const(0,0), inc_sum, + 15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30); + inc_sum = vsx_i8x16_add(inc_sum, shfl); + shfl = vsx_i16x8_shuffle(vsx_i64x2_const(0,0), inc_sum, + 7, 8, 9, 10, 11, 12, 13, 14); + inc_sum = vsx_i8x16_add(inc_sum, shfl); + shfl = vsx_i32x4_shuffle(vsx_i64x2_const(0,0), inc_sum, + 3, 4, 5, 6); + inc_sum = vsx_i8x16_add(inc_sum, shfl); + shfl = vsx_i64x2_shuffle(vsx_i64x2_const(0,0), inc_sum, + 1, 2); + inc_sum = vsx_i8x16_add(inc_sum, shfl); + cnt += vsx_u8x16_extract_lane(inc_sum, 15); + // exclusive scan + ex_sum = vsx_i8x16_shuffle(vsx_i64x2_const(0,0), inc_sum, + 15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30); + + // Spread cwd, such that each bit is in one byte + // with a value of 0 or 1. + cwd_vec = vsx_i16x8_splat((si16)cwd); + cwd_vec = vsx_i8x16_swizzle(cwd_vec, + vsx_i8x16_const(0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1)); + cwd_vec = vsx_v128_and(cwd_vec, + vsx_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); + cwd_vec = vsx_i8x16_eq(cwd_vec, + vsx_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); + cwd_vec = vsx_i8x16_abs(cwd_vec); + + // Obtain bit from cwd_vec correspondig to ex_sum + // Basically, collect needed bits from cwd_vec + v128_t v = vsx_i8x16_swizzle(cwd_vec, ex_sum); + + // load data and set spp coefficients + v128_t m = vsx_i8x16_const( + 0,-1,-1,-1,4,-1,-1,-1,8,-1,-1,-1,12,-1,-1,-1); + v128_t val = vsx_i32x4_splat(3 << (p - 2)); + ui32 *dp = dpp; + for (int c = 0; c < 4; ++ c) { + v128_t s0, s0_ns, s0_val; + // load coefficients + s0 = vsx_v128_load(dp); + + // epi32 is -1 only for coefficient that + // are changed during the SPP + s0_ns = vsx_i8x16_swizzle(new_sig_vec, m); + s0_ns = vsx_i32x4_eq(s0_ns, + vsx_i32x4_const(OJPH_REPEAT4(0xFF))); + + // obtain sign for coefficients in SPP + s0_val = vsx_i8x16_swizzle(v, m); + s0_val = vsx_i32x4_shl(s0_val, 31); + s0_val = vsx_v128_or(s0_val, val); + s0_val = vsx_v128_and(s0_val, s0_ns); + + // update vector + s0 = vsx_v128_or(s0, s0_val); + // store coefficients + vsx_v128_store(dp, s0); + // prepare for next row + dp += stride; + m = vsx_i32x4_add(m, vsx_i32x4_const(OJPH_REPEAT4(1))); + } + } + frwd_advance(&sigprop, cnt); + } + + new_sig |= cs; + *prev_sig = (ui16)(new_sig); + + // vertical integration for the new sig. info. + t = new_sig; + new_sig |= (t & 0x7777) << 1; //above neighbors + new_sig |= (t & 0xEEEE) >> 1; //below neighbors + // add sig. info. from the row on top and below + prev = new_sig | u; + // we need only the bits in 0xF000 + prev &= 0xF000; + } + } + } + + // We perform Magnitude Refinement Pass here + if (num_passes > 2) + { + rev_struct magref; + rev_init_mrp(&magref, coded_data, (int)lengths1, (int)lengths2); + + for (ui32 y = 0; y < height; y += 4) + { + ui16 *cur_sig = sigma + (y >> 2) * mstr; + ui32 *dpp = decoded_data + y * stride; + for (ui32 i = 0; i < width; i += 4, dpp += 4) + { + //Process one entry from sigma array at a time + // Each nibble (4 bits) in the sigma array represents 4 rows, + ui32 cwd = rev_fetch_mrp(&magref); // get 32 bit data + ui16 sig = *cur_sig++; // 16 bit that will be processed now + int total_bits = 0; + if (sig) // if any of the 32 bits are set + { + // We work on 4 rows, with 4 samples each, since + // data is 32 bit (4 bytes) + + // spread the 16 bits in sig to 0 or 1 bytes in sig_vec + v128_t sig_vec = vsx_i16x8_splat((si16)sig); + sig_vec = vsx_i8x16_swizzle(sig_vec, + vsx_i8x16_const(0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1)); + sig_vec = vsx_v128_and(sig_vec, + vsx_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); + sig_vec = vsx_i8x16_eq(sig_vec, + vsx_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); + sig_vec = vsx_i8x16_abs(sig_vec); + + // find cumulative sums + // to find which bit in cwd we should extract + v128_t ex_sum, shfl, inc_sum = sig_vec; // inclusive scan + shfl = vsx_i8x16_shuffle(vsx_i64x2_const(0,0), inc_sum, + 15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30); + inc_sum = vsx_i8x16_add(inc_sum, shfl); + shfl = vsx_i16x8_shuffle(vsx_i64x2_const(0,0), inc_sum, + 7, 8, 9, 10, 11, 12, 13, 14); + inc_sum = vsx_i8x16_add(inc_sum, shfl); + shfl = vsx_i32x4_shuffle(vsx_i64x2_const(0,0), inc_sum, + 3, 4, 5, 6); + inc_sum = vsx_i8x16_add(inc_sum, shfl); + shfl = vsx_i64x2_shuffle(vsx_i64x2_const(0,0), inc_sum, + 1, 2); + inc_sum = vsx_i8x16_add(inc_sum, shfl); + total_bits = vsx_u8x16_extract_lane(inc_sum, 15); + // exclusive scan + ex_sum = vsx_i8x16_shuffle(vsx_i64x2_const(0,0), inc_sum, + 15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30); + + // Spread the 16 bits in cwd to inverted 0 or 1 bytes in + // cwd_vec. Then, convert these to a form suitable + // for coefficient modifications; in particular, a value + // of 0 is presented as binary 11, and a value of 1 is + // represented as binary 01 + v128_t cwd_vec = vsx_i16x8_splat((si16)cwd); + cwd_vec = vsx_i8x16_swizzle(cwd_vec, + vsx_i8x16_const(0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1)); + cwd_vec = vsx_v128_and(cwd_vec, + vsx_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); + cwd_vec = vsx_i8x16_eq(cwd_vec, + vsx_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); + cwd_vec = + vsx_i8x16_add(cwd_vec, vsx_i8x16_const(OJPH_REPEAT16(1))); + cwd_vec = vsx_i8x16_add(cwd_vec, cwd_vec); + cwd_vec = + vsx_v128_or(cwd_vec, vsx_i8x16_const(OJPH_REPEAT16(1))); + + // load data and insert the mrp bit + v128_t m = vsx_i8x16_const(0,-1,-1,-1,4,-1,-1,-1, + 8,-1,-1,-1,12,-1,-1,-1); + ui32 *dp = dpp; + for (int c = 0; c < 4; ++c) { + v128_t s0, s0_sig, s0_idx, s0_val; + // load coefficients + s0 = vsx_v128_load(dp); + // find significant samples in this row + s0_sig = vsx_i8x16_swizzle(sig_vec, m); + s0_sig = vsx_i8x16_eq(s0_sig, vsx_i64x2_const(0, 0)); + // get MRP bit index, and MRP pattern + s0_idx = vsx_i8x16_swizzle(ex_sum, m); + s0_val = vsx_i8x16_swizzle(cwd_vec, s0_idx); + // keep data from significant samples only + s0_val = vsx_v128_andnot(s0_val, s0_sig); + // move mrp bits to correct position, and employ + s0_val = vsx_i32x4_shl(s0_val, p - 2); + s0 = vsx_v128_xor(s0, s0_val); + // store coefficients + vsx_v128_store(dp, s0); + // prepare for next row + dp += stride; + m = vsx_i32x4_add(m, vsx_i32x4_const(OJPH_REPEAT4(1))); + } + } + // consume data according to the number of bits set + rev_advance_mrp(&magref, (ui32)total_bits); + } + } + } + } + + return true; + } + } +} diff --git a/external/OpenJPH/src/core/coding/ojph_block_decoder_wasm.cpp b/external/OpenJPH/src/core/coding/ojph_block_decoder_wasm.cpp index 42a04b589..f19c65e75 100644 --- a/external/OpenJPH/src/core/coding/ojph_block_decoder_wasm.cpp +++ b/external/OpenJPH/src/core/coding/ojph_block_decoder_wasm.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2022, Aous Naman +// Copyright (c) 2022, Aous Naman // Copyright (c) 2022, Kakadu Software Pty Ltd, Australia // Copyright (c) 2022, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -69,7 +69,7 @@ namespace ojph { * A number of events is decoded from the MEL bitstream ahead of time * and stored in run/num_runs. * Each run represents the number of zero events before a one event. - */ + */ struct dec_mel_st { dec_mel_st() : data(NULL), tmp(0), bits(0), size(0), unstuff(false), k(0), num_runs(0), runs(0) @@ -89,7 +89,7 @@ namespace ojph { //************************************************************************/ /** @brief Reads and unstuffs the MEL bitstream - * + * * This design needs more bytes in the codeblock buffer than the length * of the cleanup pass by up to 2 bytes. * @@ -107,14 +107,14 @@ namespace ojph { ui32 val = 0xFFFFFFFF; // feed in 0xFF if buffer is exhausted if (melp->size > 4) { // if there is data in the MEL segment - val = *(ui32*)melp->data; // read 32 bits from MEL data + memcpy(&val, melp->data, sizeof(val)); // read 32 bits from MEL data melp->data += 4; // advance pointer melp->size -= 4; // reduce counter } else if (melp->size > 0) { // 4 or less int i = 0; - while (melp->size > 1) { + while (melp->size > 1) { ui32 v = *melp->data++; // read one byte at a time ui32 m = ~(0xFFu << i); // mask of location val = (val & m) | (v << i);// put one byte in its correct location @@ -122,21 +122,21 @@ namespace ojph { i += 8; } // size equal to 1 - ui32 v = *melp->data++; // the one before the last is different + ui32 v = *melp->data++; // the one before the last is different v |= 0xF; // MEL and VLC segments can overlap ui32 m = ~(0xFFu << i); val = (val & m) | (v << i); --melp->size; } - + // next we unstuff them before adding them to the buffer int bits = 32 - melp->unstuff; // number of bits in val, subtract 1 if - // the previously read byte requires + // the previously read byte requires // unstuffing // data is unstuffed and accumulated in t // bits has the number of bits in t - ui32 t = val & 0xFF; + ui32 t = val & 0xFF; bool unstuff = ((val & 0xFF) == 0xFF); // true if we need unstuffing bits -= unstuff; // there is one less bit in t if unstuffing is needed t = t << (8 - unstuff); // move up to make room for the next byte @@ -163,14 +163,14 @@ namespace ojph { //************************************************************************/ /** @brief Decodes unstuffed MEL segment bits stored in tmp to runs - * + * * Runs are stored in "runs" and the number of runs in "num_runs". - * Each run represents a number of zero events that may or may not + * Each run represents a number of zero events that may or may not * terminate in a 1 event. * Each run is stored in 7 bits. The LSB is 1 if the run terminates in - * a 1 event, 0 otherwise. The next 6 bits, for the case terminating - * with 1, contain the number of consecutive 0 zero events * 2; for the - * case terminating with 0, they store (number of consecutive 0 zero + * a 1 event, 0 otherwise. The next 6 bits, for the case terminating + * with 1, contain the number of consecutive 0 zero events * 2; for the + * case terminating with 0, they store (number of consecutive 0 zero * events - 1) * 2. * A total of 6 bits (made up of 1 + 5) should have been enough. * @@ -195,7 +195,7 @@ namespace ojph { int run = 0; if (melp->tmp & (1ull<<63)) //The next bit to decode (stored in MSB) { //one is found - run = 1 << eval; + run = 1 << eval; run--; // consecutive runs of 0 events - 1 melp->k = melp->k + 1 < 12 ? melp->k + 1 : 12;//increment, max is 12 melp->tmp <<= 1; // consume one bit from tmp @@ -213,14 +213,14 @@ namespace ojph { eval = melp->num_runs * 7; // 7 bits per run melp->runs &= ~((ui64)0x3F << eval); // 6 bits are sufficient melp->runs |= ((ui64)run) << eval; // store the value in runs - melp->num_runs++; // increment count + melp->num_runs++; // increment count } } //************************************************************************/ /** @brief Initiates a dec_mel_st structure for MEL decoding and reads * some bytes in order to get the read address to a multiple - * of 4 + * of 4 * * @param [in] melp is a pointer to dec_mel_st structure * @param [in] bbuf is a pointer to byte buffer @@ -235,7 +235,7 @@ namespace ojph { melp->tmp = 0; // melp->unstuff = false; // no unstuffing melp->size = scup - 1; // size is the length of MEL+VLC-1 - melp->k = 0; // 0 for state + melp->k = 0; // 0 for state melp->num_runs = 0; // num_runs is 0 melp->runs = 0; // @@ -253,7 +253,7 @@ namespace ojph { int d_bits = 8 - melp->unstuff; //if unstuffing is needed, reduce by 1 melp->tmp = (melp->tmp << d_bits) | d; //store bits in tmp melp->bits += d_bits; //increment tmp by number of bits - melp->unstuff = ((d & 0xFF) == 0xFF); //true of next byte needs + melp->unstuff = ((d & 0xFF) == 0xFF); //true of next byte needs //unstuffing } melp->tmp <<= (64 - melp->bits); //push all the way up so the first bit @@ -265,7 +265,7 @@ namespace ojph { * MEL segment is decoded * * @param [in] melp is a pointer to dec_mel_st structure - */ + */ static inline int mel_get_run(dec_mel_st *melp) { @@ -281,7 +281,7 @@ namespace ojph { //************************************************************************/ /** @brief A structure for reading and unstuffing a segment that grows * backward, such as VLC and MRP - */ + */ struct rev_struct { rev_struct() : data(NULL), tmp(0), bits(0), size(0), unstuff(false) {} @@ -298,41 +298,41 @@ namespace ojph { /** @brief Read and unstuff data from a backwardly-growing segment * * This reader can read up to 8 bytes from before the VLC segment. - * Care must be taken not read from unreadable memory, causing a + * Care must be taken not read from unreadable memory, causing a * segmentation fault. - * + * * Note that there is another subroutine rev_read_mrp that is slightly * different. The other one fills zeros when the buffer is exhausted. * This one basically does not care if the bytes are consumed, because * any extra data should not be used in the actual decoding. * - * Unstuffing is needed to prevent sequences more than 0xFF8F from + * Unstuffing is needed to prevent sequences more than 0xFF8F from * appearing in the bits stream; since we are reading backward, we keep - * watch when a value larger than 0x8F appears in the bitstream. - * If the byte following this is 0x7F, we unstuff this byte (ignore the + * watch when a value larger than 0x8F appears in the bitstream. + * If the byte following this is 0x7F, we unstuff this byte (ignore the * MSB of that byte, which should be 0). * * @param [in] vlcp is a pointer to rev_struct structure */ - static inline + static inline void rev_read(rev_struct *vlcp) { //process 4 bytes at a time - if (vlcp->bits > 32) // if there are more than 32 bits in tmp, then + if (vlcp->bits > 32) // if there are more than 32 bits in tmp, then return; // reading 32 bits can overflow vlcp->tmp ui32 val = 0; //the next line (the if statement) needs to be tested first if (vlcp->size > 3) // if there are more than 3 bytes left in VLC { // (vlcp->data - 3) move pointer back to read 32 bits at once - val = *(ui32*)(vlcp->data - 3); // then read 32 bits + memcpy(&val, vlcp->data - 3, sizeof(val)); // then read 32 bits vlcp->data -= 4; // move data pointer back by 4 vlcp->size -= 4; // reduce available byte by 4 } else if (vlcp->size > 0) { // 4 or less int i = 24; - while (vlcp->size > 0) { + while (vlcp->size > 0) { ui32 v = *vlcp->data--; // read one byte at a time val |= (v << i); // put byte in its correct location --vlcp->size; @@ -367,7 +367,7 @@ namespace ojph { } //************************************************************************/ - /** @brief Initiates the rev_struct structure and reads a few bytes to + /** @brief Initiates the rev_struct structure and reads a few bytes to * move the read address to multiple of 4 * * There is another similar rev_init_mrp subroutine. The difference is @@ -380,7 +380,7 @@ namespace ojph { * @param [in] lcup is the length of MagSgn+MEL+VLC segments * @param [in] scup is the length of MEL+VLC segments */ - static inline + static inline void rev_init(rev_struct *vlcp, ui8* data, int lcup, int scup) { //first byte has only the upper 4 bits @@ -415,13 +415,13 @@ namespace ojph { } //************************************************************************/ - /** @brief Retrieves 32 bits from the head of a rev_struct structure + /** @brief Retrieves 32 bits from the head of a rev_struct structure * * By the end of this call, vlcp->tmp must have no less than 33 bits * * @param [in] vlcp is a pointer to rev_struct structure */ - static inline + static inline ui32 rev_fetch(rev_struct *vlcp) { if (vlcp->bits < 32) // if there are less then 32 bits, read more @@ -439,7 +439,7 @@ namespace ojph { * @param [in] vlcp is a pointer to rev_struct structure * @param [in] num_bits is the number of bits to be removed */ - static inline + static inline ui32 rev_advance(rev_struct *vlcp, ui32 num_bits) { assert(num_bits <= vlcp->bits); // vlcp->tmp must have more than num_bits @@ -459,7 +459,7 @@ namespace ojph { * * @param [in] mrp is a pointer to rev_struct structure */ - static inline + static inline void rev_read_mrp(rev_struct *mrp) { //process 4 bytes at a time @@ -468,14 +468,14 @@ namespace ojph { ui32 val = 0; if (mrp->size > 3) // If there are 3 byte or more { // (mrp->data - 3) move pointer back to read 32 bits at once - val = *(ui32*)(mrp->data - 3); // read 32 bits + memcpy(&val, mrp->data - 3, sizeof(val)); // read 32 bits mrp->data -= 4; // move back pointer mrp->size -= 4; // reduce count } else if (mrp->size > 0) { int i = 24; - while (mrp->size > 0) { + while (mrp->size > 0) { ui32 v = *mrp->data--; // read one byte at a time val |= (v << i); // put byte in its correct location --mrp->size; @@ -515,7 +515,7 @@ namespace ojph { * an architecture that read size must be compatible with the * alignment of the read address * - * There is another similar subroutine rev_init. This subroutine does + * There is another similar subroutine rev_init. This subroutine does * NOT skip the first 12 bits, and starts with unstuff set to true. * * @param [in] mrp is a pointer to rev_struct structure @@ -523,7 +523,7 @@ namespace ojph { * @param [in] lcup is the length of MagSgn+MEL+VLC segments * @param [in] len2 is the length of SPP+MRP segments */ - static inline + static inline void rev_init_mrp(rev_struct *mrp, ui8* data, int lcup, int len2) { mrp->data = data + lcup + len2 - 1; @@ -540,7 +540,7 @@ namespace ojph { for (int i = 0; i < num; ++i) { ui64 d; //read a byte, 0 if no more data - d = (mrp->size-- > 0) ? *mrp->data-- : 0; + d = (mrp->size-- > 0) ? *mrp->data-- : 0; //check if unstuffing is needed ui32 d_bits = 8 - ((mrp->unstuff && ((d & 0x7F) == 0x7F)) ? 1 : 0); mrp->tmp |= d << mrp->bits; // move data to vlcp->tmp @@ -551,13 +551,13 @@ namespace ojph { } //************************************************************************/ - /** @brief Retrieves 32 bits from the head of a rev_struct structure + /** @brief Retrieves 32 bits from the head of a rev_struct structure * * By the end of this call, mrp->tmp must have no less than 33 bits * * @param [in] mrp is a pointer to rev_struct structure */ - static inline + static inline ui32 rev_fetch_mrp(rev_struct *mrp) { if (mrp->bits < 32) // if there are less than 32 bits in mrp->tmp @@ -584,7 +584,7 @@ namespace ojph { } //************************************************************************/ - /** @brief State structure for reading and unstuffing of forward-growing + /** @brief State structure for reading and unstuffing of forward-growing * bitstreams; these are: MagSgn and SPP bitstreams */ struct frwd_struct { @@ -597,7 +597,7 @@ namespace ojph { //************************************************************************/ /** @brief Read and unstuffs 16 bytes from forward-growing bitstream - * + * * A template is used to accommodate a different requirement for * MagSgn and SPP bitstreams; in particular, when MagSgn bitstream is * consumed, 0xFF's are fed, while when SPP is exhausted 0's are fed in. @@ -614,7 +614,7 @@ namespace ojph { * */ template - static inline + static inline void frwd_read(frwd_struct *msp) { assert(msp->bits <= 128); @@ -635,19 +635,19 @@ namespace ojph { val = wasm_v128_or(t, val); // fill with 0xFF } else if (X == 0) - val = wasm_v128_and(validity, val); // fill with zeros + val = wasm_v128_and(validity, val); // fill with zeros else assert(0); v128_t ff_bytes; ff_bytes = wasm_i8x16_eq(val, all_xff); ff_bytes = wasm_v128_and(ff_bytes, validity); - ui32 flags = wasm_i8x16_bitmask(ff_bytes); + ui32 flags = wasm_i8x16_bitmask(ff_bytes); flags <<= 1; // unstuff following byte ui32 next_unstuff = flags >> 16; flags |= msp->unstuff; flags &= 0xFFFF; - while (flags) + while (flags) { // bit unstuffing occurs on average once every 256 bytes // therefore it is not an issue if it is a bit slow // here we process 16 bytes @@ -665,7 +665,7 @@ namespace ojph { t = wasm_i64x2_shuffle(t, wasm_i64x2_const(0, 0), 1, 2); t = wasm_i64x2_shl(t, 63); // keep the MSB only t = wasm_v128_or(t, c); // combine the above 3 steps - + val = wasm_v128_or(t, wasm_v128_andnot(val, m)); } @@ -697,7 +697,7 @@ namespace ojph { //************************************************************************/ /** @brief Initialize frwd_struct struct and reads some bytes - * + * * @tparam X is the value fed in when the bitstream is exhausted. * See frwd_read regarding the template * @param [in] msp is a pointer to frwd_struct @@ -705,7 +705,7 @@ namespace ojph { * @param [in] size is the number of byte in the bitstream */ template - static inline + static inline void frwd_init(frwd_struct *msp, const ui8* data, int size) { msp->data = data; @@ -726,7 +726,7 @@ namespace ojph { * @param [in] msp is a pointer to frwd_struct * @param [in] num_bits is the number of bit to consume */ - static inline + static inline void frwd_advance(frwd_struct *msp, ui32 num_bits) { assert(num_bits > 0 && num_bits <= msp->bits && num_bits < 128); @@ -795,7 +795,7 @@ namespace ojph { * @return v128_t decoded quad */ template - static inline + static inline v128_t decode_one_quad32(const v128_t inf_u_q, v128_t U_q, frwd_struct* magsgn, ui32 p, v128_t& vn) { @@ -813,7 +813,7 @@ namespace ojph { { U_q = wasm_i32x4_shuffle(U_q, U_q, N, N, N, N); flags = wasm_i16x8_mul(flags, wasm_i16x8_const(8,8,4,4,2,2,1,1)); - v128_t ms_vec = frwd_fetch<0xFF>(magsgn); + v128_t ms_vec = frwd_fetch<0xFF>(magsgn); // U_q holds U_q for this quad // flags has e_k, e_1, and rho such that e_k is sitting in the @@ -837,14 +837,14 @@ namespace ojph { // find the starting byte and starting bit v128_t byte_idx = wasm_u32x4_shr(ex_sum, 3); - v128_t bit_idx = + v128_t bit_idx = wasm_v128_and(ex_sum, wasm_i32x4_const(OJPH_REPEAT4(7))); - byte_idx = wasm_i8x16_swizzle(byte_idx, + byte_idx = wasm_i8x16_swizzle(byte_idx, wasm_i32x4_const(0x00000000, 0x04040404, 0x08080808, 0x0C0C0C0C)); - byte_idx = + byte_idx = wasm_i32x4_add(byte_idx, wasm_i32x4_const(OJPH_REPEAT4(0x03020100))); v128_t d0 = wasm_i8x16_swizzle(ms_vec, byte_idx); - byte_idx = + byte_idx = wasm_i32x4_add(byte_idx, wasm_i32x4_const(OJPH_REPEAT4(0x01010101))); v128_t d1 = wasm_i8x16_swizzle(ms_vec, byte_idx); @@ -853,7 +853,7 @@ namespace ojph { v128_t bit_shift = wasm_i8x16_swizzle( wasm_i8x16_const(-1, 127, 63, 31, 15, 7, 3, 1, -1, 127, 63, 31, 15, 7, 3, 1), bit_idx); - bit_shift = + bit_shift = wasm_i16x8_add(bit_shift, wasm_i16x8_const(OJPH_REPEAT8(0x0101))); d0 = wasm_i16x8_mul(d0, bit_shift); d0 = wasm_u16x8_shr(d0, 8); // we should have 8 bits in the LSB @@ -886,10 +886,10 @@ namespace ojph { ms_vec = wasm_v128_andnot(tvn, insig); // significant only if (N == 0) // the compiler should remove one - tvn = wasm_i8x16_swizzle(ms_vec, + tvn = wasm_i8x16_swizzle(ms_vec, wasm_i32x4_const(0x07060504, 0x0F0E0D0C, -1, -1)); else if (N == 1) - tvn = wasm_i8x16_swizzle(ms_vec, + tvn = wasm_i8x16_swizzle(ms_vec, wasm_i32x4_const(-1, 0x07060504, 0x0F0E0D0C, -1)); else assert(0); @@ -911,8 +911,8 @@ namespace ojph { * @param vn used for handling E values (stores v_n values) * @return v128_t decoded quad */ - static inline - v128_t decode_two_quad16(const v128_t inf_u_q, v128_t U_q, + static inline + v128_t decode_two_quad16(const v128_t inf_u_q, v128_t U_q, frwd_struct* magsgn, ui32 p, v128_t& vn) { v128_t w0; // workers @@ -921,21 +921,21 @@ namespace ojph { v128_t row; // decoded row row = wasm_i64x2_const(0, 0); - w0 = wasm_i8x16_swizzle(inf_u_q, + w0 = wasm_i8x16_swizzle(inf_u_q, wasm_i16x8_const(0x0100, 0x0100, 0x0100, 0x0100, 0x0504, 0x0504, 0x0504, 0x0504)); // we keeps e_k, e_1, and rho in w2 - flags = wasm_v128_and(w0, - wasm_u16x8_const(0x1110, 0x2220, 0x4440, 0x8880, + flags = wasm_v128_and(w0, + wasm_u16x8_const(0x1110, 0x2220, 0x4440, 0x8880, 0x1110, 0x2220, 0x4440, 0x8880)); insig = wasm_i16x8_eq(flags, wasm_i64x2_const(0, 0)); if (wasm_i8x16_bitmask(insig) != 0xFFFF) //are all insignificant? { - U_q = wasm_i8x16_swizzle(U_q, - wasm_i16x8_const(0x0100, 0x0100, 0x0100, 0x0100, + U_q = wasm_i8x16_swizzle(U_q, + wasm_i16x8_const(0x0100, 0x0100, 0x0100, 0x0100, 0x0504, 0x0504, 0x0504, 0x0504)); flags = wasm_i16x8_mul(flags, wasm_i16x8_const(8,4,2,1,8,4,2,1)); - v128_t ms_vec = frwd_fetch<0xFF>(magsgn); + v128_t ms_vec = frwd_fetch<0xFF>(magsgn); // U_q holds U_q for this quad // flags has e_k, e_1, and rho such that e_k is sitting in the @@ -950,7 +950,7 @@ namespace ojph { // find cumulative sums // to find at which bit in ms_vec the sample starts v128_t ex_sum, shfl, inc_sum = m_n; // inclusive scan - shfl = wasm_i16x8_shuffle(wasm_i64x2_const(0,0), + shfl = wasm_i16x8_shuffle(wasm_i64x2_const(0,0), inc_sum, 7, 8, 9, 10, 11, 12, 13, 14); inc_sum = wasm_i16x8_add(inc_sum, shfl); shfl = wasm_i32x4_shuffle(wasm_i64x2_const(0,0), inc_sum, 3, 4, 5, 6); @@ -958,20 +958,20 @@ namespace ojph { shfl = wasm_i64x2_shuffle(wasm_i64x2_const(0,0), inc_sum, 1, 2); inc_sum = wasm_i16x8_add(inc_sum, shfl); int total_mn = wasm_u16x8_extract_lane(inc_sum, 7); - ex_sum = wasm_i16x8_shuffle(wasm_i64x2_const(0,0), + ex_sum = wasm_i16x8_shuffle(wasm_i64x2_const(0,0), inc_sum, 7, 8, 9, 10, 11, 12, 13, 14); // find the starting byte and starting bit v128_t byte_idx = wasm_u16x8_shr(ex_sum, 3); - v128_t bit_idx = + v128_t bit_idx = wasm_v128_and(ex_sum, wasm_i16x8_const(OJPH_REPEAT8(7))); - byte_idx = wasm_i8x16_swizzle(byte_idx, - wasm_i16x8_const(0x0000, 0x0202, 0x0404, 0x0606, + byte_idx = wasm_i8x16_swizzle(byte_idx, + wasm_i16x8_const(0x0000, 0x0202, 0x0404, 0x0606, 0x0808, 0x0A0A, 0x0C0C, 0x0E0E)); - byte_idx = + byte_idx = wasm_i16x8_add(byte_idx, wasm_i16x8_const(OJPH_REPEAT8(0x0100))); v128_t d0 = wasm_i8x16_swizzle(ms_vec, byte_idx); - byte_idx = + byte_idx = wasm_i16x8_add(byte_idx, wasm_i16x8_const(OJPH_REPEAT8(0x0101))); v128_t d1 = wasm_i8x16_swizzle(ms_vec, byte_idx); @@ -979,13 +979,13 @@ namespace ojph { v128_t bit_shift = wasm_i8x16_swizzle( wasm_i8x16_const(-1, 127, 63, 31, 15, 7, 3, 1, -1, 127, 63, 31, 15, 7, 3, 1), bit_idx); - bit_shift = + bit_shift = wasm_i16x8_add(bit_shift, wasm_i16x8_const(OJPH_REPEAT8(0x0101))); d0 = wasm_i16x8_mul(d0, bit_shift); d0 = wasm_u16x8_shr(d0, 8); // we should have 8 bits in the LSB d1 = wasm_i16x8_mul(d1, bit_shift); d1 = // 8 in MSB - wasm_v128_and(d1, wasm_i16x8_const(OJPH_REPEAT8((si16)0xFF00))); + wasm_v128_and(d1, wasm_i16x8_const(OJPH_REPEAT8((si16)0xFF00))); d0 = wasm_v128_or(d0, d1); // find location of e_k and mask @@ -1017,10 +1017,10 @@ namespace ojph { row = wasm_v128_andnot(ms_vec, insig); // significant only ms_vec = wasm_v128_andnot(tvn, insig); // significant only - w0 = wasm_i8x16_swizzle(ms_vec, + w0 = wasm_i8x16_swizzle(ms_vec, wasm_i16x8_const(0x0302, 0x0706, -1, -1, -1, -1, -1, -1)); vn = wasm_v128_or(vn, w0); - w0 = wasm_i8x16_swizzle(ms_vec, + w0 = wasm_i8x16_swizzle(ms_vec, wasm_i16x8_const(-1, 0x0B0A, 0x0F0E, -1, -1, -1, -1, -1)); vn = wasm_v128_or(vn, w0); @@ -1043,9 +1043,9 @@ namespace ojph { * @param [in] lengths1 is the length of cleanup pass * @param [in] lengths2 is the length of refinement passes (either SPP * only or SPP+MRP) - * @param [in] width is the decoded codeblock width + * @param [in] width is the decoded codeblock width * @param [in] height is the decoded codeblock height - * @param [in] stride is the decoded codeblock buffer stride + * @param [in] stride is the decoded codeblock buffer stride * @param [in] stripe_causal is true for stripe causal mode */ bool ojph_decode_codeblock_wasm(ui8* coded_data, ui32* decoded_data, @@ -1076,7 +1076,7 @@ namespace ojph { if (missing_msbs > 30) // p < 0 { - if (insufficient_precision == false) + if (insufficient_precision == false) { insufficient_precision = true; OJPH_WARN(0x00010003, "32 bits are not enough to decode this " @@ -1084,7 +1084,7 @@ namespace ojph { "displayed again.\n"); } return false; - } + } else if (missing_msbs == 30) // p == 0 { // not enough precision to decode and set the bin center to 1 if (modify_code == false) { @@ -1127,16 +1127,16 @@ namespace ojph { if (scup < 2 || scup > lcup || scup > 4079) //something is wrong return false; - // The temporary storage scratch holds two types of data in an + // The temporary storage scratch holds two types of data in an // interleaved fashion. The interleaving allows us to use one // memory pointer. // We have one entry for a decoded VLC code, and one entry for UVLC. - // Entries are 16 bits each, corresponding to one quad, - // but since we want to use XMM registers of the SSE family + // Entries are 16 bits each, corresponding to one quad, + // but since we want to use XMM registers of the SSE family // of SIMD; we allocated 16 bytes or more per quad row; that is, // the width is no smaller than 16 bytes (or 8 entries), and the // height is 512 quads - // Each VLC entry contains, in the following order, starting + // Each VLC entry contains, in the following order, starting // from MSB // e_k (4bits), e_1 (4bits), rho (4bits), useless for step 2 (4bits) // Each entry in UVLC contains u_q @@ -1145,10 +1145,10 @@ namespace ojph { ui16 scratch[8 * 513] = {0}; // 8+ kB // We need an extra two entries (one inf and one u_q) beyond - // the last column. - // If the block width is 4 (2 quads), then we use sstr of 8 - // (enough for 4 quads). If width is 8 (4 quads) we use - // sstr is 16 (enough for 8 quads). For a width of 16 (8 + // the last column. + // If the block width is 4 (2 quads), then we use sstr of 8 + // (enough for 4 quads). If width is 8 (4 quads) we use + // sstr is 16 (enough for 8 quads). For a width of 16 (8 // quads), we use 24 (enough for 12 quads). ui32 sstr = ((width + 2u) + 7u) & ~7u; // multiples of 8 @@ -1157,11 +1157,11 @@ namespace ojph { ui32 mmsbp2 = missing_msbs + 2; // The cleanup pass is decoded in two steps; in step one, - // the VLC and MEL segments are decoded, generating a record that + // the VLC and MEL segments are decoded, generating a record that // has 2 bytes per quad. The 2 bytes contain, u, rho, e^1 & e^k. // This information should be sufficient for the next step. // In step 2, we decode the MagSgn segment. - + // step 1 decoding VLC and MEL segments { // init structures @@ -1194,20 +1194,20 @@ namespace ojph { { run -= 2; //subtract 2, since events number if multiplied by 2 - // Is the run terminated in 1? if so, use decoded VLC code, - // otherwise, discard decoded data, since we will decoded again + // Is the run terminated in 1? if so, use decoded VLC code, + // otherwise, discard decoded data, since we will decoded again // using a different context t0 = (run == -1) ? t0 : 0; // is run -1 or -2? this means a run has been consumed - if (run < 0) + if (run < 0) run = mel_get_run(&mel); // get another run } //run -= (c_q == 0) ? 2 : 0; //t0 = (c_q != 0 || run == -1) ? t0 : 0; //if (run < 0) // run = mel_get_run(&mel); // get another run - sp[0] = t0; + sp[0] = t0; x += 2; // prepare context for the next quad; eqn. 1 in ITU T.814 @@ -1220,7 +1220,7 @@ namespace ojph { ui16 t1 = 0; //decode VLC using the context c_q and the head of VLC bitstream - t1 = vlc_tbl0[c_q + (vlc_val & 0x7F)]; + t1 = vlc_tbl0[c_q + (vlc_val & 0x7F)]; // if context is zero, use one MEL event if (c_q == 0 && x < width) //zero context @@ -1246,7 +1246,7 @@ namespace ojph { //remove data from vlc stream, if qinf is not used, cwdlen is 0 vlc_val = rev_advance(&vlc, t1 & 0x7); - + // decode u ///////////// // uvlc_mode is made up of u_offset bits from the quad pair @@ -1269,8 +1269,8 @@ namespace ojph { //decode uvlc_mode to get u for both quads ui32 uvlc_entry = uvlc_tbl0[uvlc_mode + (vlc_val & 0x3F)]; //remove total prefix length - vlc_val = rev_advance(&vlc, uvlc_entry & 0x7); - uvlc_entry >>= 3; + vlc_val = rev_advance(&vlc, uvlc_entry & 0x7); + uvlc_entry >>= 3; //extract suffixes for quad 0 and 1 ui32 len = uvlc_entry & 0xF; //suffix length for 2 quads ui32 tmp = vlc_val & ((1 << len) - 1); //suffix value for 2 quads @@ -1280,9 +1280,9 @@ namespace ojph { len = uvlc_entry & 0x7; // quad 0 suffix length uvlc_entry >>= 3; ui16 u_q = (ui16)(1 + (uvlc_entry&7) + (tmp&~(0xFFU<> 3) + (tmp >> len)); //kappa == 1 - sp[3] = u_q; + sp[3] = u_q; } sp[0] = sp[1] = 0; @@ -1312,13 +1312,13 @@ namespace ojph { { run -= 2; //subtract 2, since events number is multiplied by 2 - // Is the run terminated in 1? if so, use decoded VLC code, - // otherwise, discard decoded data, since we will decoded again + // Is the run terminated in 1? if so, use decoded VLC code, + // otherwise, discard decoded data, since we will decoded again // using a different context t0 = (run == -1) ? t0 : 0; // is run -1 or -2? this means a run has been consumed - if (run < 0) + if (run < 0) run = mel_get_run(&mel); // get another run } //run -= (c_q == 0) ? 2 : 0; @@ -1344,7 +1344,7 @@ namespace ojph { ui16 t1 = 0; //decode VLC using the context c_q and the head of VLC bitstream - t1 = vlc_tbl1[ c_q + (vlc_val & 0x7F)]; + t1 = vlc_tbl1[ c_q + (vlc_val & 0x7F)]; // if context is zero, use one MEL event if (c_q == 0 && x < width) //zero context @@ -1362,7 +1362,7 @@ namespace ojph { //t1 = (c_q != 0 || run == -1) ? t1 : 0; //if (run < 0) // run = mel_get_run(&mel); // get another run - sp[2] = t1; + sp[2] = t1; x += 2; // partial c_q, will be completed when we process the next quad @@ -1373,7 +1373,7 @@ namespace ojph { //remove data from vlc stream, if qinf is not used, cwdlen is 0 vlc_val = rev_advance(&vlc, t1 & 0x7); - + // decode u ///////////// // uvlc_mode is made up of u_offset bits from the quad pair @@ -1449,10 +1449,10 @@ namespace ojph { w0 = wasm_v128_load(vp); w0 = wasm_v128_and(w0, wasm_i32x4_const(-1,0,0,0)); w0 = wasm_v128_or(w0, vn); - wasm_v128_store(vp, w0); + wasm_v128_store(vp, w0); + + //interleave in ssse3 style - //interleave in ssse3 style - w0 = wasm_i32x4_shuffle(row0, row1, 0, 4, 1, 5); w1 = wasm_i32x4_shuffle(row0, row1, 2, 6, 3, 7); row0 = wasm_i32x4_shuffle(w0, w1, 0, 4, 1, 5); @@ -1516,7 +1516,7 @@ namespace ojph { v128_t gamma, emax, kappa, u_q; // needed locally inf_u_q = wasm_v128_load(sp); - gamma = + gamma = wasm_v128_and(inf_u_q, wasm_i32x4_const(OJPH_REPEAT4(0xF0))); w0 = wasm_i32x4_sub(gamma, wasm_i32x4_const(OJPH_REPEAT4(1))); gamma = wasm_v128_and(gamma, w0); @@ -1524,7 +1524,7 @@ namespace ojph { emax = wasm_v128_load(vp + v_n_size); w0 = wasm_i32x4_shuffle(emax, wasm_i64x2_const(0,0), 1, 2, 3, 4); - emax = wasm_i16x8_max(w0, emax); // no max_epi32 in ssse3 + emax = wasm_i16x8_max(w0, emax); // no max_epi32 in ssse3 emax = wasm_v128_andnot(emax, gamma); kappa = wasm_i32x4_const(OJPH_REPEAT4(1)); @@ -1545,19 +1545,19 @@ namespace ojph { w0 = wasm_v128_load(vp); w0 = wasm_v128_and(w0, wasm_i32x4_const(-1,0,0,0)); w0 = wasm_v128_or(w0, vn); - wasm_v128_store(vp, w0); + wasm_v128_store(vp, w0); //interleave in ssse3 style - w0 = wasm_i32x4_shuffle(row0, row1, 0, 4, 1, 5); - w1 = wasm_i32x4_shuffle(row0, row1, 2, 6, 3, 7); - row0 = wasm_i32x4_shuffle(w0, w1, 0, 4, 1, 5); - row1 = wasm_i32x4_shuffle(w0, w1, 2, 6, 3, 7); + w0 = wasm_i32x4_shuffle(row0, row1, 0, 4, 1, 5); + w1 = wasm_i32x4_shuffle(row0, row1, 2, 6, 3, 7); + row0 = wasm_i32x4_shuffle(w0, w1, 0, 4, 1, 5); + row1 = wasm_i32x4_shuffle(w0, w1, 2, 6, 3, 7); wasm_v128_store(dp, row0); wasm_v128_store(dp + stride, row1); } } } - else + else { // reduce bitplane by 16 because we now have 16 bits instead of 32 p -= 16; @@ -1601,15 +1601,15 @@ namespace ojph { w0 = wasm_v128_load(vp); w0 = wasm_v128_and(w0, wasm_i16x8_const(-1,0,0,0,0,0,0,0)); w0 = wasm_v128_or(w0, vn); - wasm_v128_store(vp, w0); + wasm_v128_store(vp, w0); - //interleave in ssse3 style - w0 = wasm_i8x16_swizzle(row, - wasm_i16x8_const(-1, 0x0100, -1, 0x0504, + //interleave in ssse3 style + w0 = wasm_i8x16_swizzle(row, + wasm_i16x8_const(-1, 0x0100, -1, 0x0504, -1, 0x0908, -1, 0x0D0C)); wasm_v128_store(dp, w0); - w1 = wasm_i8x16_swizzle(row, - wasm_i16x8_const(-1, 0x0302, -1, 0x0706, + w1 = wasm_i8x16_swizzle(row, + wasm_i16x8_const(-1, 0x0302, -1, 0x0706, -1, 0x0B0A, -1, 0x0F0E)); wasm_v128_store(dp + stride, w1); } @@ -1664,18 +1664,18 @@ namespace ojph { v128_t gamma, emax, kappa, u_q; // needed locally inf_u_q = wasm_v128_load(sp); - gamma = + gamma = wasm_v128_and(inf_u_q, wasm_i32x4_const(OJPH_REPEAT4(0xF0))); w0 = wasm_i32x4_sub(gamma, wasm_i32x4_const(OJPH_REPEAT4(1))); gamma = wasm_v128_and(gamma, w0); gamma = wasm_i32x4_eq(gamma, wasm_i64x2_const(0, 0)); emax = wasm_v128_load(vp + v_n_size); - w0 = wasm_i16x8_shuffle(emax, + w0 = wasm_i16x8_shuffle(emax, wasm_i64x2_const(0, 0), 1, 2, 3, 4, 5, 6, 7, 8); emax = wasm_i16x8_max(w0, emax); // no max_epi32 in ssse3 - emax = wasm_i8x16_swizzle(emax, - wasm_i16x8_const(0x0100, -1, 0x0302, -1, + emax = wasm_i8x16_swizzle(emax, + wasm_i16x8_const(0x0100, -1, 0x0302, -1, 0x0504, -1, 0x0706, -1)); emax = wasm_v128_andnot(emax, gamma); @@ -1696,14 +1696,14 @@ namespace ojph { w0 = wasm_v128_load(vp); w0 = wasm_v128_and(w0, wasm_i16x8_const(-1,0,0,0,0,0,0,0)); w0 = wasm_v128_or(w0, vn); - wasm_v128_store(vp, w0); + wasm_v128_store(vp, w0); - w0 = wasm_i8x16_swizzle(row, - wasm_i16x8_const(-1, 0x0100, -1, 0x0504, + w0 = wasm_i8x16_swizzle(row, + wasm_i16x8_const(-1, 0x0100, -1, 0x0504, -1, 0x0908, -1, 0x0D0C)); wasm_v128_store(dp, w0); - w1 = wasm_i8x16_swizzle(row, - wasm_i16x8_const(-1, 0x0302, -1, 0x0706, + w1 = wasm_i8x16_swizzle(row, + wasm_i16x8_const(-1, 0x0302, -1, 0x0706, -1, 0x0B0A, -1, 0x0F0E)); wasm_v128_store(dp + stride, w1); } @@ -1717,7 +1717,7 @@ namespace ojph { { // We use scratch again, we can divide it into multiple regions // sigma holds all the significant samples, and it cannot - // be modified after it is set. it will be used during the + // be modified after it is set. it will be used during the // Magnitude Refinement Pass ui16* const sigma = scratch; @@ -1734,11 +1734,11 @@ namespace ojph { const v128_t mask_3 = wasm_i32x4_const(OJPH_REPEAT4(0x30)); const v128_t mask_C = wasm_i32x4_const(OJPH_REPEAT4(0xC0)); const v128_t shuffle_mask = wasm_i32x4_const(0x0C080400,-1,-1,-1); - for (y = 0; y < height; y += 4) + for (y = 0; y < height; y += 4) { ui16* sp = scratch + (y >> 1) * sstr; ui16* dp = sigma + (y >> 2) * mstr; - for (ui32 x = 0; x < width; x += 8, sp += 8, dp += 2) + for (ui32 x = 0; x < width; x += 8, sp += 8, dp += 2) { v128_t s0, s1, u3, uC, t0, t1; @@ -1825,13 +1825,13 @@ namespace ojph { // We need data for at least 5 columns out of 8. // Therefore loading 32 bits is easier than loading 16 bits // twice. - ui32 ps = *(ui32*)prev_sig; - ui32 ns = *(ui32*)(cur_sig + mstr); + ui32 ps; memcpy(&ps, prev_sig, sizeof(ps)); + ui32 ns; memcpy(&ns, cur_sig + mstr, sizeof(ns)); ui32 u = (ps & 0x88888888) >> 3; // the row on top if (!stripe_causal) u |= (ns & 0x11111111) << 3; // the row below - ui32 cs = *(ui32*)cur_sig; + ui32 cs; memcpy(&cs, cur_sig, sizeof(cs)); // vertical integration ui32 mbr = cs; // this sig. info. mbr |= (cs & 0x77777777) << 1; //above neighbors @@ -1927,17 +1927,17 @@ namespace ojph { // find cumulative sums // to find which bit in cwd we should extract v128_t ex_sum, shfl, inc_sum = new_sig_vec; // inclusive scan - inc_sum = wasm_i8x16_abs(inc_sum); // cvrt to 0 or 1 + inc_sum = wasm_i8x16_abs(inc_sum); // cvrt to 0 or 1 shfl = wasm_i8x16_shuffle(wasm_i64x2_const(0,0), inc_sum, 15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30); inc_sum = wasm_i8x16_add(inc_sum, shfl); - shfl = wasm_i16x8_shuffle(wasm_i64x2_const(0,0), inc_sum, + shfl = wasm_i16x8_shuffle(wasm_i64x2_const(0,0), inc_sum, 7, 8, 9, 10, 11, 12, 13, 14); inc_sum = wasm_i8x16_add(inc_sum, shfl); - shfl = wasm_i32x4_shuffle(wasm_i64x2_const(0,0), inc_sum, + shfl = wasm_i32x4_shuffle(wasm_i64x2_const(0,0), inc_sum, 3, 4, 5, 6); inc_sum = wasm_i8x16_add(inc_sum, shfl); - shfl = wasm_i64x2_shuffle(wasm_i64x2_const(0,0), inc_sum, + shfl = wasm_i64x2_shuffle(wasm_i64x2_const(0,0), inc_sum, 1, 2); inc_sum = wasm_i8x16_add(inc_sum, shfl); cnt += wasm_u8x16_extract_lane(inc_sum, 15); @@ -1973,7 +1973,7 @@ namespace ojph { // epi32 is -1 only for coefficient that // are changed during the SPP s0_ns = wasm_i8x16_swizzle(new_sig_vec, m); - s0_ns = wasm_i32x4_eq(s0_ns, + s0_ns = wasm_i32x4_eq(s0_ns, wasm_i32x4_const(OJPH_REPEAT4(0xFF))); // obtain sign for coefficients in SPP @@ -2047,13 +2047,13 @@ namespace ojph { shfl = wasm_i8x16_shuffle(wasm_i64x2_const(0,0), inc_sum, 15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30); inc_sum = wasm_i8x16_add(inc_sum, shfl); - shfl = wasm_i16x8_shuffle(wasm_i64x2_const(0,0), inc_sum, + shfl = wasm_i16x8_shuffle(wasm_i64x2_const(0,0), inc_sum, 7, 8, 9, 10, 11, 12, 13, 14); inc_sum = wasm_i8x16_add(inc_sum, shfl); - shfl = wasm_i32x4_shuffle(wasm_i64x2_const(0,0), inc_sum, + shfl = wasm_i32x4_shuffle(wasm_i64x2_const(0,0), inc_sum, 3, 4, 5, 6); inc_sum = wasm_i8x16_add(inc_sum, shfl); - shfl = wasm_i64x2_shuffle(wasm_i64x2_const(0,0), inc_sum, + shfl = wasm_i64x2_shuffle(wasm_i64x2_const(0,0), inc_sum, 1, 2); inc_sum = wasm_i8x16_add(inc_sum, shfl); total_bits = wasm_u8x16_extract_lane(inc_sum, 15); @@ -2069,14 +2069,14 @@ namespace ojph { v128_t cwd_vec = wasm_i16x8_splat((si16)cwd); cwd_vec = wasm_i8x16_swizzle(cwd_vec, wasm_i8x16_const(0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1)); - cwd_vec = wasm_v128_and(cwd_vec, + cwd_vec = wasm_v128_and(cwd_vec, wasm_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); - cwd_vec = wasm_i8x16_eq(cwd_vec, + cwd_vec = wasm_i8x16_eq(cwd_vec, wasm_u64x2_const(OJPH_REPEAT2(0x8040201008040201))); - cwd_vec = + cwd_vec = wasm_i8x16_add(cwd_vec, wasm_i8x16_const(OJPH_REPEAT16(1))); cwd_vec = wasm_i8x16_add(cwd_vec, cwd_vec); - cwd_vec = + cwd_vec = wasm_v128_or(cwd_vec, wasm_i8x16_const(OJPH_REPEAT16(1))); // load data and insert the mrp bit @@ -2085,7 +2085,7 @@ namespace ojph { ui32 *dp = dpp; for (int c = 0; c < 4; ++c) { v128_t s0, s0_sig, s0_idx, s0_val; - // load coefficients + // load coefficients s0 = wasm_v128_load(dp); // find significant samples in this row s0_sig = wasm_i8x16_swizzle(sig_vec, m); diff --git a/external/OpenJPH/src/core/coding/ojph_block_encoder_avx2.cpp b/external/OpenJPH/src/core/coding/ojph_block_encoder_avx2.cpp index 91d0195de..e2ab2310a 100644 --- a/external/OpenJPH/src/core/coding/ojph_block_encoder_avx2.cpp +++ b/external/OpenJPH/src/core/coding/ojph_block_encoder_avx2.cpp @@ -6,6 +6,7 @@ // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia // Copyright (c) 2024, Intel Corporation +// Copyright (c) 2026, Osamu Watanabe // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -34,6 +35,14 @@ // File: ojph_block_encoder_avx2.cpp //***************************************************************************/ +// Apple Clang on Intel produces corrupt bitstreams with several of the +// scalar optimizations below (branchless VLC drain, 64-bit MagSgn, etc.) +// for reasons not yet identified. Since this file is x86-only, the guard +// does not affect Apple Silicon builds (which use NEON, not AVX2). +#if defined(__apple_build_version__) +#include "ojph_block_encoder_avx2_apple.h" +#else + #include "ojph_arch.h" #if defined(OJPH_ARCH_I386) || defined(OJPH_ARCH_X86_64) @@ -72,6 +81,8 @@ namespace ojph { static ui32 vlc_tbl1[2048]; //UVLC encoding + static ui32 uvlc_tbl_pair1[33 * 33]; + static ui32 uvlc_tbl_pair2[33 * 33]; static ui32 ulvc_cwd_pre[33]; static int ulvc_cwd_pre_len[33]; static ui32 ulvc_cwd_suf[33]; @@ -221,6 +232,56 @@ namespace ojph { return true; } + ///////////////////////////////////////////////////////////////////////// + static void uvlc_init_pair_tables() + { + for (int uq0 = 0; uq0 < 33; ++uq0) { + for (int uq1 = 0; uq1 < 33; ++uq1) { + ui32 cwd; int len; + + cwd = 0; len = 0; + if (uq0 > 2 && uq1 > 2) { + cwd |= ulvc_cwd_pre[uq0 - 2]; + len += ulvc_cwd_pre_len[uq0 - 2]; + cwd |= ulvc_cwd_pre[uq1 - 2] << len; + len += ulvc_cwd_pre_len[uq1 - 2]; + cwd |= ulvc_cwd_suf[uq0 - 2] << len; + len += ulvc_cwd_suf_len[uq0 - 2]; + cwd |= ulvc_cwd_suf[uq1 - 2] << len; + len += ulvc_cwd_suf_len[uq1 - 2]; + } else if (uq0 > 2 && uq1 > 0) { + cwd |= ulvc_cwd_pre[uq0]; + len += ulvc_cwd_pre_len[uq0]; + cwd |= (ui32)(uq1 - 1) << len; + len += 1; + cwd |= ulvc_cwd_suf[uq0] << len; + len += ulvc_cwd_suf_len[uq0]; + } else { + cwd |= ulvc_cwd_pre[uq0]; + len += ulvc_cwd_pre_len[uq0]; + cwd |= ulvc_cwd_pre[uq1] << len; + len += ulvc_cwd_pre_len[uq1]; + cwd |= ulvc_cwd_suf[uq0] << len; + len += ulvc_cwd_suf_len[uq0]; + cwd |= ulvc_cwd_suf[uq1] << len; + len += ulvc_cwd_suf_len[uq1]; + } + uvlc_tbl_pair1[uq0 * 33 + uq1] = (cwd << 5) | (ui32)len; + + cwd = 0; len = 0; + cwd |= ulvc_cwd_pre[uq0]; + len += ulvc_cwd_pre_len[uq0]; + cwd |= ulvc_cwd_pre[uq1] << len; + len += ulvc_cwd_pre_len[uq1]; + cwd |= ulvc_cwd_suf[uq0] << len; + len += ulvc_cwd_suf_len[uq0]; + cwd |= ulvc_cwd_suf[uq1] << len; + len += ulvc_cwd_suf_len[uq1]; + uvlc_tbl_pair2[uq0 * 33 + uq1] = (cwd << 5) | (ui32)len; + } + } + } + ///////////////////////////////////////////////////////////////////////// bool initialize_block_encoder_tables_avx2() { static bool tables_initialized = false; @@ -230,6 +291,7 @@ namespace ojph { memset(vlc_tbl1, 0, 2048 * sizeof(ui32)); tables_initialized = vlc_init_tables(); tables_initialized = tables_initialized && uvlc_init_tables(); + uvlc_init_pair_tables(); }); return tables_initialized; } @@ -265,16 +327,20 @@ namespace ojph { melp->threshold = 1; // this is 1 << mel_exp[melp->k]; } + static const int mel_exp[13] = {0,0,0,1,1,1,2,2,2,3,3,4,5}; + ////////////////////////////////////////////////////////////////////////// static inline void - mel_emit_bit(mel_struct* melp, int v) + mel_emit_bits(mel_struct* melp, ui32 bits, int num_bits) { - melp->tmp = (melp->tmp << 1) + v; - melp->remaining_bits--; - if (melp->remaining_bits == 0) { - melp->buf[melp->pos++] = (ui8)melp->tmp; - melp->remaining_bits = (melp->tmp == 0xFF ? 7 : 8); - melp->tmp = 0; + melp->tmp = (melp->tmp << num_bits) | (int)bits; + melp->remaining_bits -= num_bits; + if (melp->remaining_bits <= 0) { + int excess = -melp->remaining_bits; + ui8 byte = (ui8)(melp->tmp >> excess); + melp->buf[melp->pos++] = byte; + melp->tmp &= (1 << excess) - 1; + melp->remaining_bits += 8 - (byte == 0xFF); } } @@ -282,33 +348,59 @@ namespace ojph { static inline void mel_encode(mel_struct* melp, bool bit) { - //MEL exponent - static const int mel_exp[13] = {0,0,0,1,1,1,2,2,2,3,3,4,5}; - if (bit == false) { ++melp->run; if (melp->run >= melp->threshold) { - mel_emit_bit(melp, 1); + mel_emit_bits(melp, 1, 1); melp->run = 0; melp->k = ojph_min(12, melp->k + 1); melp->threshold = 1 << mel_exp[melp->k]; } } else { - mel_emit_bit(melp, 0); int t = mel_exp[melp->k]; - while (t > 0) { - mel_emit_bit(melp, (melp->run >> --t) & 1); - } + mel_emit_bits(melp, (ui32)melp->run & ((1u << t) - 1), t + 1); melp->run = 0; melp->k = ojph_max(0, melp->k - 1); melp->threshold = 1 << mel_exp[melp->k]; } } + ////////////////////////////////////////////////////////////////////////// + // static inline void + // mel_advance_run(mel_struct* melp, ui32 n) + // { + // ui32 remaining = n; + // while (remaining > 0) { + // ui32 space = (ui32)melp->threshold - (ui32)melp->run; + // if (remaining >= space) { + // remaining -= space; + // mel_emit_bits(melp, 1, 1); + // melp->run = 0; + // melp->k = ojph_min(12, melp->k + 1); + // melp->threshold = 1 << mel_exp[melp->k]; + // } else { + // melp->run += (int)remaining; + // remaining = 0; + // } + // } + // } + + ////////////////////////////////////////////////////////////////////////// + // static inline void + // mel_encode_significance(mel_struct* melp) + // { + // int t = mel_exp[melp->k]; + // mel_emit_bits(melp, melp->run & ((1u << t) - 1), t + 1); + // melp->run = 0; + // melp->k = ojph_max(0, melp->k - 1); + // melp->threshold = 1 << mel_exp[melp->k]; + // } + ///////////////////////////////////////////////////////////////////////// // ///////////////////////////////////////////////////////////////////////// - struct vlc_struct_avx2 { + + struct vlc_struct { //storage ui8* buf; //pointer to data buffer ui32 pos; //position of next writing within buf @@ -321,7 +413,7 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// static inline void - vlc_init(vlc_struct_avx2* vlcp, ui32 buffer_size, ui8* data) + vlc_init(vlc_struct* vlcp, ui32 buffer_size, ui8* data) { vlcp->buf = data + buffer_size - 1; //points to last byte vlcp->pos = 1; //locations will be all -pos @@ -335,39 +427,40 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// static inline void - vlc_encode(vlc_struct_avx2* vlcp, ui32 cwd, int cwd_len) + vlc_drain(vlc_struct* vlcp) { - vlcp->tmp |= (ui64)cwd << vlcp->used_bits; - vlcp->used_bits += cwd_len; - while (vlcp->used_bits >= 8) { - ui8 tmp; - - if (unlikely(vlcp->last_greater_than_8F)) { - tmp = vlcp->tmp & 0x7F; - - if (likely(tmp != 0x7F)) { - tmp = vlcp->tmp & 0xFF; - *(vlcp->buf - vlcp->pos) = tmp; - vlcp->last_greater_than_8F = tmp > 0x8F; - vlcp->tmp >>= 8; - vlcp->used_bits -= 8; - } else { - *(vlcp->buf - vlcp->pos) = tmp; - vlcp->last_greater_than_8F = false; - vlcp->tmp >>= 7; - vlcp->used_bits -= 7; - } + int escape = (int)vlcp->last_greater_than_8F; + int is_7f = (int)((vlcp->tmp & 0x7F) == 0x7F); + int need_stuff = (escape & is_7f) != 0 ? 1 : 0; + int bits = 8 - need_stuff; - } else { - tmp = vlcp->tmp & 0xFF; - *(vlcp->buf - vlcp->pos) = tmp; - vlcp->last_greater_than_8F = tmp > 0x8F; - vlcp->tmp >>= 8; - vlcp->used_bits -= 8; - } + ui8 byte = (ui8)(vlcp->tmp & ((1u << bits) - 1)); + *(vlcp->buf - vlcp->pos) = byte; + vlcp->pos++; + vlcp->tmp >>= bits; + vlcp->used_bits -= bits; + vlcp->last_greater_than_8F = byte > 0x8F; + } + } - vlcp->pos++; + ////////////////////////////////////////////////////////////////////////// + static inline void + vlc_encode(vlc_struct* vlcp, ui64 cwd, int cwd_len) + { + while (true) { + int avail = 64 - vlcp->used_bits; + if (likely(avail > 0 && cwd_len <= avail)) { + vlcp->tmp |= cwd << vlcp->used_bits; + vlcp->used_bits += cwd_len; + return; + } + if (likely(avail > 0)) // available space smaller than needed + vlcp->tmp |= cwd << vlcp->used_bits; + vlcp->used_bits = 64; + vlc_drain(vlcp); + cwd >>= avail; + cwd_len -= avail; } } @@ -375,10 +468,10 @@ namespace ojph { // ////////////////////////////////////////////////////////////////////////// static inline void - terminate_mel_vlc(mel_struct* melp, vlc_struct_avx2* vlcp) + terminate_mel_vlc(mel_struct* melp, vlc_struct* vlcp) { if (melp->run > 0) - mel_emit_bit(melp, 1); + mel_emit_bits(melp, 1, 1); if (vlcp->last_greater_than_8F && (vlcp->tmp & 0x7f) == 0x7f) { *(vlcp->buf - vlcp->pos) = 0x7f; @@ -416,15 +509,16 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// // ///////////////////////////////////////////////////////////////////////// + struct ms_struct { //storage - ui8* buf; //pointer to data buffer - ui32 pos; //position of next writing within buf - ui32 buf_size; //size of buffer, which we must not exceed + ui8* buf; //pointer to data buffer + ui32 pos; //position of next writing within buf + ui32 buf_size; //size of buffer, which we must not exceed - int max_bits; //maximum number of bits that can be store in tmp - int used_bits; //number of occupied bits in tmp - ui32 tmp; //temporary storage of coded bits + int used_bits; //number of occupied bits in tmp + ui64 tmp; //temporary storage of coded bits (64-bit accumulator) + bool last_was_ff;//true if the last written byte was 0xFF }; ////////////////////////////////////////////////////////////////////////// @@ -434,51 +528,129 @@ namespace ojph { msp->buf = data; msp->pos = 0; msp->buf_size = buffer_size; - msp->max_bits = 8; msp->used_bits = 0; msp->tmp = 0; + msp->last_was_ff = false; } ////////////////////////////////////////////////////////////////////////// static inline void - ms_encode(ms_struct* msp, ui64 cwd, int cwd_len) + ms_drain(ms_struct* msp) { - while (cwd_len > 0) - { - if (msp->pos >= msp->buf_size) - OJPH_ERROR(0x00020005, "magnitude sign encoder's buffer is full"); - int t = ojph_min(msp->max_bits - msp->used_bits, cwd_len); - msp->tmp |= ((ui32)(cwd & ((1U << t) - 1))) << msp->used_bits; - msp->used_bits += t; - cwd >>= t; - cwd_len -= t; - if (msp->used_bits >= msp->max_bits) - { - msp->buf[msp->pos++] = (ui8)msp->tmp; - msp->max_bits = (msp->tmp == 0xFF) ? 7 : 8; - msp->tmp = 0; - msp->used_bits = 0; + if (msp->last_was_ff) { + if (msp->used_bits < 7) + return; + msp->buf[msp->pos++] = (ui8)(msp->tmp & 0x7F); + msp->tmp >>= 7; + msp->used_bits -= 7; + msp->last_was_ff = false; + } + + while (msp->used_bits >= 8) { + int n_bytes = msp->used_bits >> 3; + if (n_bytes > 8) n_bytes = 8; + + ui64 word = msp->tmp; + ui64 valid_mask = (n_bytes < 8) + ? (1ULL << (n_bytes * 8)) - 1 : ~(ui64)0; + + ui64 w = ~word; + ui64 ff_detect = (w - 0x0101010101010101ULL) & ~w + & 0x8080808080808080ULL; + ff_detect &= valid_mask; + + if (likely(ff_detect == 0)) { + memcpy(msp->buf + msp->pos, &word, (size_t)n_bytes); + msp->pos += (ui32)n_bytes; + if (n_bytes < 8) + msp->tmp >>= (n_bytes * 8); + else + msp->tmp = 0; + msp->used_bits -= n_bytes * 8; + } else { + int ff_pos = (int)(count_trailing_zeros(ff_detect) >> 3); + int safe = ff_pos + 1; + memcpy(msp->buf + msp->pos, &word, (size_t)safe); + msp->pos += (ui32)safe; + int bits = safe * 8; + if (bits < 64) + msp->tmp >>= bits; + else + msp->tmp = 0; + msp->used_bits -= bits; + + if (msp->used_bits >= 7) { + msp->buf[msp->pos++] = (ui8)(msp->tmp & 0x7F); + msp->tmp >>= 7; + msp->used_bits -= 7; + msp->last_was_ff = false; + } else { + msp->last_was_ff = true; + return; + } } } } + ////////////////////////////////////////////////////////////////////////// + static inline void + ms_encode_nodefer(ms_struct* msp, ui64 cwd, int cwd_len) + { + while (true) { + int avail = 64 - msp->used_bits; + if (likely(avail > 0 && cwd_len <= avail)) { + msp->tmp |= cwd << msp->used_bits; + msp->used_bits += cwd_len; + return; + } + if (likely(avail > 0)) // available space smaller than needed + msp->tmp |= cwd << msp->used_bits; + msp->used_bits = 64; + ms_drain(msp); + cwd >>= avail; + cwd_len -= avail; + } + } + + ////////////////////////////////////////////////////////////////////////// + // static inline void + // ms_encode(ms_struct* msp, ui64 cwd, int cwd_len) + // { + // int avail = 64 - msp->used_bits; + // if (likely(cwd_len <= avail)) { + // msp->tmp |= cwd << msp->used_bits; + // msp->used_bits += cwd_len; + // } else { + // msp->tmp |= (cwd & ((1ULL << avail) - 1)) << msp->used_bits; + // msp->used_bits = 64; + // ms_drain(msp); + // cwd >>= avail; + // cwd_len -= avail; + // msp->tmp |= cwd << msp->used_bits; + // msp->used_bits += cwd_len; + // } + // ms_drain(msp); + // } + ////////////////////////////////////////////////////////////////////////// static inline void ms_terminate(ms_struct* msp) { + ms_drain(msp); if (msp->used_bits) { - int t = msp->max_bits - msp->used_bits; //unused bits - msp->tmp |= (0xFF & ((1U << t) - 1)) << msp->used_bits; - msp->used_bits += t; - if (msp->tmp != 0xFF) + int max_bits = msp->last_was_ff ? 7 : 8; + int t = max_bits - msp->used_bits; + ui32 byte = (ui32)(msp->tmp & ((1ULL << msp->used_bits) - 1)); + byte |= (0xFFu & ((1u << t) - 1)) << msp->used_bits; + if (byte != 0xFF) { if (msp->pos >= msp->buf_size) OJPH_ERROR(0x00020006, "magnitude sign encoder's buffer is full"); - msp->buf[msp->pos++] = (ui8)msp->tmp; + msp->buf[msp->pos++] = (ui8)byte; } } - else if (msp->max_bits == 7) + else if (msp->last_was_ff) msp->pos--; } @@ -665,23 +837,10 @@ static void proc_ms_encode(ms_struct *msp, m_vec[3] = _mm256_and_si256(mask, tmp); rotate_matrix(m_vec); - /* s_vec from - * s_vec[0]:[0, 0], [0, 2] ... [0,14], [0, 16], [0, 18] ... [0,30] - * s_vec[1]:[1, 0], [1, 2] ... [1,14], [1, 16], [1, 18] ... [1,30] - * s_vec[2]:[0, 1], [0, 3] ... [0,15], [0, 17], [0, 19] ... [0,31] - * s_vec[3]:[1, 1], [1, 3] ... [1,15], [1, 17], [1, 19] ... [1,31] - * to - * s_vec[0]:[0, 0], [1, 0], [0, 1], [1, 1], [0, 2], [1, 2]...[0, 7], [1, 7] - * s_vec[1]:[0, 8], [1, 8], [0, 9], [1, 9], [0,10], [1,10]...[0,15], [1,15] - * s_vec[2]:[0,16], [1,16], [0,17], [1,17], [0,18], [1,18]...[0,23], [1,23] - * s_vec[3]:[0,24], [1,24], [0,25], [1,25], [0,26], [1,26]...[0,31], [1,31] - */ rotate_matrix(s_vec); ui32 cwd[8]; int cwd_len[8]; - ui64 _cwd = 0; - int _cwd_len = 0; /* Each iteration process 8 bytes * 2 lines */ for (ui32 i = 0; i < 4; ++i) { @@ -694,15 +853,32 @@ static void proc_ms_encode(ms_struct *msp, tmp = _mm256_and_si256(tmp, s_vec[i]); _mm256_storeu_si256((__m256i*)cwd, tmp); - for (ui32 j = 0; j < 4; ++j) { - ui32 idx = j * 2; - _cwd = cwd[idx]; - _cwd_len = cwd_len[idx]; - _cwd |= ((ui64)cwd[idx + 1]) << _cwd_len; - _cwd_len += cwd_len[idx + 1]; - ms_encode(msp, _cwd, _cwd_len); + for (ui32 j = 0; j < 4; j += 2) { + ui32 idx0 = j * 2; + ui64 _cwd = cwd[idx0]; + int _cwd_len = cwd_len[idx0]; + _cwd |= ((ui64)cwd[idx0 + 1]) << _cwd_len; + _cwd_len += cwd_len[idx0 + 1]; + + ui32 idx1 = (j + 1) * 2; + int len1 = cwd_len[idx1] + cwd_len[idx1 + 1]; + if (likely(_cwd_len + len1 <= 64)) { + _cwd |= ((ui64)cwd[idx1]) << _cwd_len; + _cwd_len += cwd_len[idx1]; + _cwd |= ((ui64)cwd[idx1 + 1]) << _cwd_len; + _cwd_len += cwd_len[idx1 + 1]; + ms_encode_nodefer(msp, _cwd, _cwd_len); + } else { + ms_encode_nodefer(msp, _cwd, _cwd_len); + _cwd = cwd[idx1]; + _cwd_len = cwd_len[idx1]; + _cwd |= ((ui64)cwd[idx1 + 1]) << _cwd_len; + _cwd_len += cwd_len[idx1 + 1]; + ms_encode_nodefer(msp, _cwd, _cwd_len); + } } } + ms_drain(msp); } static __m256i cal_eps_vec(__m256i *eq_vec, __m256i &u_q_vec, @@ -806,7 +982,7 @@ static __m256i proc_cq2(ui32 x, __m256i *cx_val_vec, __m256i &rho_vec, auto tmp = _mm256_permutevar8x32_epi32(lcxp1_vec, right_shift); #ifdef OJPH_ARCH_X86_64 - tmp = _mm256_insert_epi64(tmp, + tmp = _mm256_insert_epi64(tmp, _mm_cvtsi128_si64(_mm256_castsi256_si128(cx_val_vec[x + 1])), 3); #elif (defined OJPH_ARCH_I386) int lsb = _mm_cvtsi128_si32(_mm256_castsi256_si128(cx_val_vec[x + 1])); @@ -817,7 +993,7 @@ static __m256i proc_cq2(ui32 x, __m256i *cx_val_vec, __m256i &rho_vec, #error Error unsupport compiler #endif tmp = _mm256_slli_epi32(tmp, 2); - auto tmp1 = _mm256_insert_epi32(lcxp1_vec, + auto tmp1 = _mm256_insert_epi32(lcxp1_vec, _mm_cvtsi128_si32(_mm256_castsi256_si128(cx_val_vec[x + 1])), 7); tmp = _mm256_add_epi32(tmp1, tmp); @@ -831,8 +1007,6 @@ static __m256i proc_cq2(ui32 x, __m256i *cx_val_vec, __m256i &rho_vec, return _mm256_or_si256(tmp, tmp1); } -using fn_proc_cq = __m256i (*)(ui32, __m256i *, __m256i &, const __m256i); - static void proc_mel_encode1(mel_struct *melp, __m256i &cq_vec, __m256i &rho_vec, __m256i u_q_vec, ui32 ignore, const __m256i right_shift) @@ -882,134 +1056,184 @@ static void proc_mel_encode2(mel_struct *melp, __m256i &cq_vec, { ojph_unused(u_q_vec); ojph_unused(right_shift); - int32_t mel_need_encode[8]; - int32_t mel_bit[8]; - /* Prepare mel_encode params */ - /* if (c_q[i] == 0) { */ - _mm256_storeu_si256((__m256i*)mel_need_encode, _mm256_cmpeq_epi32(cq_vec, ZERO)); - /* mel_encode(&mel, rho[i] != 0); */ - _mm256_storeu_si256((__m256i*)mel_bit, _mm256_srli_epi32(avx2_cmpneq_epi32(rho_vec, ZERO), 31)); - /* } */ + __m256i need = _mm256_cmpeq_epi32(cq_vec, ZERO); + ui32 mask = (ui32)_mm256_movemask_epi8(need); + mask &= 0x88888888; ui32 i_max = 8 - (ignore / 2); + if (i_max < 8) + mask &= (1u << (i_max * 4)) - 1; - for (ui32 i = 0; i < i_max; ++i) { - if (mel_need_encode[i]) { - mel_encode(melp, mel_bit[i]); - } + if (mask == 0) + return; + + int32_t mel_bit[8]; + _mm256_storeu_si256((__m256i*)mel_bit, + _mm256_srli_epi32(avx2_cmpneq_epi32(rho_vec, ZERO), 31)); + + while (mask) { + ui32 bit_pos = (ui32)count_trailing_zeros(mask); + ui32 i = bit_pos / 4; + mel_encode(melp, mel_bit[i]); + mask &= mask - 1; } } using fn_proc_mel_encode = void (*)(mel_struct *, __m256i &, __m256i &, __m256i, ui32, const __m256i); -static void proc_vlc_encode1(vlc_struct_avx2 *vlcp, ui32 *tuple, - ui32 *u_q, ui32 ignore) +static inline void +build_vlc_uvlc_pair(ui32 *tuple, ui32 *u_q, ui32 i, + const ui32 *uvlc_tbl, ui64 &val, int &size) { - ui32 i_max = 8 - (ignore / 2); + val = tuple[i + 0] >> 4; + size = tuple[i + 0] & 7; - for (ui32 i = 0; i < i_max; i += 2) { - /* 7 bits */ - ui32 val = tuple[i + 0] >> 4; - int size = tuple[i + 0] & 7; + val |= (ui64)(tuple[i + 1] >> 4) << size; + size += tuple[i + 1] & 7; - if (i + 1 < i_max) { - /* 7 bits */ - val |= (tuple[i + 1] >> 4) << size; - size += tuple[i + 1] & 7; - } + ui32 entry = uvlc_tbl[u_q[i] * 33 + u_q[i + 1]]; + val |= (ui64)(entry >> 5) << size; + size += entry & 0x1F; +} - if (u_q[i] > 2 && u_q[i + 1] > 2) { - /* 3 bits */ - val |= (ulvc_cwd_pre[u_q[i] - 2]) << size; - size += ulvc_cwd_pre_len[u_q[i] - 2]; +static void proc_vlc_encode(vlc_struct *vlcp, ui32 *tuple, + ui32 *u_q, ui32 ignore, const ui32 *uvlc_tbl) +{ + ui32 i_max = 8 - (ignore / 2); - /* 3 bits */ - val |= (ulvc_cwd_pre[u_q[i + 1] - 2]) << size; - size += ulvc_cwd_pre_len[u_q[i + 1] - 2]; + ui32 i = 0; + for (; i + 2 < i_max; i += 4) { + ui64 val1; int size1; + build_vlc_uvlc_pair(tuple, u_q, i, uvlc_tbl, val1, size1); + ui64 val2; int size2; + build_vlc_uvlc_pair(tuple, u_q, i + 2, uvlc_tbl, val2, size2); + vlc_encode(vlcp, val1 | (val2 << size1), size1 + size2); + } + if (i < i_max) { + ui64 val; int size; + build_vlc_uvlc_pair(tuple, u_q, i, uvlc_tbl, val, size); + vlc_encode(vlcp, val, size); + } +} - /* 5 bits */ - val |= (ulvc_cwd_suf[u_q[i] - 2]) << size; - size += ulvc_cwd_suf_len[u_q[i] - 2]; +template +OJPH_FORCE_INLINE void encode_x_loop( + ui32 *sp, ui32 stride, ui32 height, ui32 y, + ui32 n_loop, ui32 _width, ui32 ignore, ui32 p, + mel_struct &mel, vlc_struct &vlc, ms_struct &ms, + __m256i *e_val_vec, __m256i &prev_e_val_vec, + __m256i *cx_val_vec, __m256i &prev_cx_val_vec, + ui32 &prev_cq, + const __m256i &right_shift, const __m256i &left_shift) +{ + ui32 *vlc_tbl = (PASS == 1) ? vlc_tbl0 : vlc_tbl1; - /* 5 bits */ - val |= (ulvc_cwd_suf[u_q[i + 1] - 2]) << size; - size += ulvc_cwd_suf_len[u_q[i + 1] - 2]; + __m256i tmp, tmp1; + __m256i eq_vec[4]; + __m256i s_vec[4]; + __m256i src_vec[4]; - } else if (u_q[i] > 2 && u_q[i + 1] > 0) { - /* 3 bits */ - val |= (ulvc_cwd_pre[u_q[i]]) << size; - size += ulvc_cwd_pre_len[u_q[i]]; + /* 16 bytes per iteration */ + for (ui32 x = 0; x < n_loop; ++x) { + + /* t = sp[i]; */ + if ((x == (n_loop - 1)) && (_width % 16)) { + ui32 tmp_buf[16] = { 0 }; + memcpy(tmp_buf, sp, (_width % 16) * sizeof(ui32)); + src_vec[0] = _mm256_loadu_si256((__m256i*)(tmp_buf)); + src_vec[2] = _mm256_loadu_si256((__m256i*)(tmp_buf + 8)); + if (y + 1 < height) { + memcpy(tmp_buf, sp + stride, (_width % 16) * sizeof(ui32)); + src_vec[1] = _mm256_loadu_si256((__m256i*)(tmp_buf)); + src_vec[3] = _mm256_loadu_si256((__m256i*)(tmp_buf + 8)); + } + else { + src_vec[1] = ZERO; + src_vec[3] = ZERO; + } + } + else { + src_vec[0] = _mm256_loadu_si256((__m256i*)(sp)); + src_vec[2] = _mm256_loadu_si256((__m256i*)(sp + 8)); - /* 1 bit */ - val |= (u_q[i + 1] - 1) << size; - size += 1; + if (y + 1 < height) { + src_vec[1] = _mm256_loadu_si256((__m256i*)(sp + stride)); + src_vec[3] = _mm256_loadu_si256((__m256i*)(sp + 8 + stride)); + } + else { + src_vec[1] = ZERO; + src_vec[3] = ZERO; + } + sp += 16; + } - /* 5 bits */ - val |= (ulvc_cwd_suf[u_q[i]]) << size; - size += ulvc_cwd_suf_len[u_q[i]]; + __m256i rho_vec, e_qmax_vec; + proc_pixel(src_vec, p, eq_vec, s_vec, rho_vec, e_qmax_vec); - } else { - /* 3 bits */ - val |= (ulvc_cwd_pre[u_q[i]]) << size; - size += ulvc_cwd_pre_len[u_q[i]]; + // max_e[(i + 1) % num] = ojph_max(lep[i + 1], lep[i + 2]) - 1; + tmp = _mm256_permutevar8x32_epi32(e_val_vec[x], right_shift); + tmp = _mm256_insert_epi32(tmp, _mm_cvtsi128_si32(_mm256_castsi256_si128(e_val_vec[x + 1])), 7); - /* 3 bits */ - val |= (ulvc_cwd_pre[u_q[i + 1]]) << size; - size += ulvc_cwd_pre_len[u_q[i + 1]]; + auto max_e_vec = _mm256_max_epi32(tmp, e_val_vec[x]); + max_e_vec = _mm256_sub_epi32(max_e_vec, ONE); - /* 5 bits */ - val |= (ulvc_cwd_suf[u_q[i]]) << size; - size += ulvc_cwd_suf_len[u_q[i]]; + // kappa[i] = (rho[i] & (rho[i] - 1)) ? ojph_max(1, max_e[i]) : 1; + tmp = _mm256_max_epi32(max_e_vec, ONE); + tmp1 = _mm256_sub_epi32(rho_vec, ONE); + tmp1 = _mm256_and_si256(rho_vec, tmp1); - /* 5 bits */ - val |= (ulvc_cwd_suf[u_q[i + 1]]) << size; - size += ulvc_cwd_suf_len[u_q[i + 1]]; - } + auto cmp = _mm256_cmpeq_epi32(tmp1, ZERO); + auto kappa_vec1_ = _mm256_and_si256(cmp, ONE); + auto kappa_vec2_ = _mm256_and_si256(_mm256_xor_si256(cmp, _mm256_set1_epi32((int32_t)0xffffffff)), tmp); + const __m256i kappa_vec = _mm256_max_epi32(kappa_vec1_, kappa_vec2_); - vlc_encode(vlcp, val, size); - } -} - -static void proc_vlc_encode2(vlc_struct_avx2 *vlcp, ui32 *tuple, - ui32 *u_q, ui32 ignore) -{ - ui32 i_max = 8 - (ignore / 2); + if (PASS == 1) + tmp = proc_cq1(x, cx_val_vec, rho_vec, right_shift); + else + tmp = proc_cq2(x, cx_val_vec, rho_vec, right_shift); - for (ui32 i = 0; i < i_max; i += 2) { - /* 7 bits */ - ui32 val = tuple[i + 0] >> 4; - int size = tuple[i + 0] & 7; + auto cq_vec = _mm256_permutevar8x32_epi32(tmp, left_shift); + cq_vec = _mm256_insert_epi32(cq_vec, prev_cq, 0); + prev_cq = (ui32)_mm256_extract_epi32(tmp, 7); - if (i + 1 < i_max) { - /* 7 bits */ - val |= (tuple[i + 1] >> 4) << size; - size += tuple[i + 1] & 7; - } + update_lep(x, prev_e_val_vec, eq_vec, e_val_vec, left_shift); + update_lcxp(x, prev_cx_val_vec, rho_vec, cx_val_vec, left_shift); - /* 3 bits */ - val |= ulvc_cwd_pre[u_q[i]] << size; - size += ulvc_cwd_pre_len[u_q[i]]; + /* Uq[i] = ojph_max(e_qmax[i], kappa[i]); */ + /* u_q[i] = Uq[i] - kappa[i]; */ + auto uq_vec = _mm256_max_epi32(kappa_vec, e_qmax_vec); + auto u_q_vec = _mm256_sub_epi32(uq_vec, kappa_vec); - /* 3 bits */ - val |= (ulvc_cwd_pre[u_q[i + 1]]) << size; - size += ulvc_cwd_pre_len[u_q[i + 1]]; + auto eps_vec = cal_eps_vec(eq_vec, u_q_vec, e_qmax_vec); + __m256i tuple_vec = cal_tuple(cq_vec, rho_vec, eps_vec, vlc_tbl); + ui32 _ignore = ((n_loop - 1) == x) ? ignore : 0; - /* 5 bits */ - val |= (ulvc_cwd_suf[u_q[i + 0]]) << size; - size += ulvc_cwd_suf_len[u_q[i + 0]]; + if (PASS == 1) + proc_mel_encode1(&mel, cq_vec, rho_vec, u_q_vec, _ignore, + right_shift); + else + proc_mel_encode2(&mel, cq_vec, rho_vec, u_q_vec, _ignore, + right_shift); - /* 5 bits */ - val |= (ulvc_cwd_suf[u_q[i + 1]]) << size; - size += ulvc_cwd_suf_len[u_q[i + 1]]; + proc_ms_encode(&ms, tuple_vec, uq_vec, rho_vec, s_vec); - vlc_encode(vlcp, val, size); + ui32 u_q[10]; + ui32 tuple[10]; + tuple_vec = _mm256_srli_epi32(tuple_vec, 4); + _mm256_storeu_si256((__m256i*)tuple, tuple_vec); + _mm256_storeu_si256((__m256i*)u_q, u_q_vec); + { + ui32 i_max = 8 - (_ignore / 2); + if (i_max & 1) { tuple[i_max] = 0; u_q[i_max] = 0; } + tuple[8] = 0; u_q[8] = 0; + } + proc_vlc_encode(&vlc, tuple, u_q, _ignore, + (PASS == 1) ? uvlc_tbl_pair1 : uvlc_tbl_pair2); } } -using fn_proc_vlc_encode = void (*)(vlc_struct_avx2 *, ui32 *, ui32 *, ui32); - void ojph_encode_codeblock_avx2(ui32* buf, ui32 missing_msbs, ui32 num_passes, ui32 _width, ui32 height, ui32 stride, ui32* lengths, @@ -1032,7 +1256,7 @@ void ojph_encode_codeblock_avx2(ui32* buf, ui32 missing_msbs, mel_struct mel; mel_init(&mel, mel_size, mel_buf); - vlc_struct_avx2 vlc; + vlc_struct vlc; vlc_init(&vlc, vlc_size, vlc_buf); ms_struct ms; ms_init(&ms, ms_size, ms_buf); @@ -1059,9 +1283,9 @@ void ojph_encode_codeblock_avx2(ui32* buf, ui32 missing_msbs, ui32 n_loop = (width + 15) / 16; __m256i e_val_vec[65]; - for (ui32 i = 0; i > 3); */ - __m256i tmp = _mm256_and_si256(prev_cx_val_vec, _mm256_set1_epi32(8)); + tmp = _mm256_and_si256(prev_cx_val_vec, _mm256_set1_epi32(8)); cx_val_vec[n_loop] = _mm256_srli_epi32(tmp, 3); prev_e_val_vec = ZERO; @@ -1091,119 +1308,27 @@ void ojph_encode_codeblock_avx2(ui32* buf, ui32 missing_msbs, ui32 *sp = buf + y * stride; - /* 16 bytes per iteration */ - for (ui32 x = 0; x < n_loop; ++x) { - - /* t = sp[i]; */ - if ((x == (n_loop - 1)) && (_width % 16)) { - ui32 tmp_buf[16] = { 0 }; - memcpy(tmp_buf, sp, (_width % 16) * sizeof(ui32)); - src_vec[0] = _mm256_loadu_si256((__m256i*)(tmp_buf)); - src_vec[2] = _mm256_loadu_si256((__m256i*)(tmp_buf + 8)); - if (y + 1 < height) { - memcpy(tmp_buf, sp + stride, (_width % 16) * sizeof(ui32)); - src_vec[1] = _mm256_loadu_si256((__m256i*)(tmp_buf)); - src_vec[3] = _mm256_loadu_si256((__m256i*)(tmp_buf + 8)); - } - else { - src_vec[1] = ZERO; - src_vec[3] = ZERO; - } - } - else { - src_vec[0] = _mm256_loadu_si256((__m256i*)(sp)); - src_vec[2] = _mm256_loadu_si256((__m256i*)(sp + 8)); - - if (y + 1 < height) { - src_vec[1] = _mm256_loadu_si256((__m256i*)(sp + stride)); - src_vec[3] = _mm256_loadu_si256((__m256i*)(sp + 8 + stride)); - } - else { - src_vec[1] = ZERO; - src_vec[3] = ZERO; - } - sp += 16; - } - - /* src_vec layout: - * src_vec[0]:[0, 0],[0, 1],[0, 2],[0, 3],[0, 4],[0, 5],.[0, 6],.[0, 7] - * src_vec[1]:[1, 0],[1, 1],[1, 2],[1, 3],[1, 4],[1, 5],.[1, 6],.[1, 7] - * src_vec[2]:[0, 8],[0, 9],[0,10],[0,11],[0,12],[0,13],.[0,14], [0,15] - * src_vec[3]:[1, 8],[1, 9],[1,10],[1,11],[1,12],[1,13],.[1,14], [1,15] - */ - __m256i rho_vec, e_qmax_vec; - proc_pixel(src_vec, p, eq_vec, s_vec, rho_vec, e_qmax_vec); - - // max_e[(i + 1) % num] = ojph_max(lep[i + 1], lep[i + 2]) - 1; - tmp = _mm256_permutevar8x32_epi32(e_val_vec[x], right_shift); - tmp = _mm256_insert_epi32(tmp, _mm_cvtsi128_si32(_mm256_castsi256_si128(e_val_vec[x + 1])), 7); - - auto max_e_vec = _mm256_max_epi32(tmp, e_val_vec[x]); - max_e_vec = _mm256_sub_epi32(max_e_vec, ONE); - - // kappa[i] = (rho[i] & (rho[i] - 1)) ? ojph_max(1, max_e[i]) : 1; - tmp = _mm256_max_epi32(max_e_vec, ONE); - __m256i tmp1 = _mm256_sub_epi32(rho_vec, ONE); - tmp1 = _mm256_and_si256(rho_vec, tmp1); - - auto cmp = _mm256_cmpeq_epi32(tmp1, ZERO); - auto kappa_vec1_ = _mm256_and_si256(cmp, ONE); - auto kappa_vec2_ = _mm256_and_si256(_mm256_xor_si256(cmp, _mm256_set1_epi32((int32_t)0xffffffff)), tmp); - const __m256i kappa_vec = _mm256_max_epi32(kappa_vec1_, kappa_vec2_); - - /* cq[1 - 16] = cq_vec - * cq[0] = prev_cq_vec[0] - */ - tmp = proc_cq(x, cx_val_vec, rho_vec, right_shift); - - auto cq_vec = _mm256_permutevar8x32_epi32(tmp, left_shift); - cq_vec = _mm256_insert_epi32(cq_vec, prev_cq, 0); - prev_cq = (ui32)_mm256_extract_epi32(tmp, 7); - - update_lep(x, prev_e_val_vec, eq_vec, e_val_vec, left_shift); - update_lcxp(x, prev_cx_val_vec, rho_vec, cx_val_vec, left_shift); - - /* Uq[i] = ojph_max(e_qmax[i], kappa[i]); */ - /* u_q[i] = Uq[i] - kappa[i]; */ - auto uq_vec = _mm256_max_epi32(kappa_vec, e_qmax_vec); - auto u_q_vec = _mm256_sub_epi32(uq_vec, kappa_vec); - - auto eps_vec = cal_eps_vec(eq_vec, u_q_vec, e_qmax_vec); - __m256i tuple_vec = cal_tuple(cq_vec, rho_vec, eps_vec, vlc_tbl); - ui32 _ignore = ((n_loop - 1) == x) ? ignore : 0; - - proc_mel_encode(&mel, cq_vec, rho_vec, u_q_vec, _ignore, - right_shift); - - proc_ms_encode(&ms, tuple_vec, uq_vec, rho_vec, s_vec); - - // vlc_encode(&vlc, tuple[i*2+0] >> 8, (tuple[i*2+0] >> 4) & 7); - // vlc_encode(&vlc, tuple[i*2+1] >> 8, (tuple[i*2+1] >> 4) & 7); - ui32 u_q[8]; - ui32 tuple[8]; - /* The tuple is scaled by 4 due to: - * vlc_encode(&vlc, tuple0 >> 8, (tuple0 >> 4) & 7, true); - * So in the vlc_encode, the tuple will only be scaled by 2. - */ - tuple_vec = _mm256_srli_epi32(tuple_vec, 4); - _mm256_storeu_si256((__m256i*)tuple, tuple_vec); - _mm256_storeu_si256((__m256i*)u_q, u_q_vec); - - proc_vlc_encode(&vlc, tuple, u_q, _ignore); - } + if (y == 0) + encode_x_loop<1>(sp, stride, height, y, n_loop, _width, + ignore, p, mel, vlc, ms, + e_val_vec, prev_e_val_vec, + cx_val_vec, prev_cx_val_vec, prev_cq, + right_shift, left_shift); + else + encode_x_loop<2>(sp, stride, height, y, n_loop, _width, + ignore, p, mel, vlc, ms, + e_val_vec, prev_e_val_vec, + cx_val_vec, prev_cx_val_vec, prev_cq, + right_shift, left_shift); tmp = _mm256_permutevar8x32_epi32(cx_val_vec[0], right_shift); tmp = _mm256_slli_epi32(tmp, 2); tmp = _mm256_add_epi32(tmp, cx_val_vec[0]); prev_cq = (ui32)_mm_cvtsi128_si32(_mm256_castsi256_si128(tmp)); - - proc_cq = proc_cq2; - vlc_tbl = vlc_tbl1; - proc_mel_encode = proc_mel_encode2; - proc_vlc_encode = proc_vlc_encode2; } ms_terminate(&ms); + vlc_drain(&vlc); terminate_mel_vlc(&mel, &vlc); //copy to elastic @@ -1227,3 +1352,4 @@ void ojph_encode_codeblock_avx2(ui32* buf, ui32 missing_msbs, } /* namespace ojph */ #endif +#endif // !defined(__apple_build_version__) diff --git a/external/OpenJPH/src/core/coding/ojph_block_encoder_avx2_apple.h b/external/OpenJPH/src/core/coding/ojph_block_encoder_avx2_apple.h new file mode 100644 index 000000000..cf12c2cbd --- /dev/null +++ b/external/OpenJPH/src/core/coding/ojph_block_encoder_avx2_apple.h @@ -0,0 +1,1230 @@ +//***************************************************************************/ +// This software is released under the 2-Clause BSD license, included +// below. +// +// Copyright (c) 2019, Aous Naman +// Copyright (c) 2019, Kakadu Software Pty Ltd, Australia +// Copyright (c) 2019, The University of New South Wales, Australia +// Copyright (c) 2024, Intel Corporation +// Copyright (c) 2026, Osamu Watanabe +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************/ +// This file is part of the OpenJPH software implementation. +// File: ojph_block_encoder_avx2.cpp +//***************************************************************************/ + +#include "ojph_arch.h" +#if defined(OJPH_ARCH_I386) || defined(OJPH_ARCH_X86_64) + +#include +#include +#include +#include +#include +#include + +#include "ojph_mem.h" +#include "ojph_arch.h" +#include "ojph_block_encoder.h" +#include "ojph_message.h" + +#ifdef OJPH_COMPILER_MSVC + #define likely(x) (x) + #define unlikely(x) (x) +#else + #define likely(x) __builtin_expect((x), 1) + #define unlikely(x) __builtin_expect((x), 0) +#endif + +namespace ojph { + namespace local { + + ///////////////////////////////////////////////////////////////////////// + // tables + ///////////////////////////////////////////////////////////////////////// + + //VLC encoding + // index is (c_q << 8) + (rho << 4) + eps + // data is (cwd << 8) + (cwd_len << 4) + eps + // table 0 is for the initial line of quads + static ui32 vlc_tbl0[2048]; + static ui32 vlc_tbl1[2048]; + + //UVLC encoding + static ui32 ulvc_cwd_pre[33]; + static int ulvc_cwd_pre_len[33]; + static ui32 ulvc_cwd_suf[33]; + static int ulvc_cwd_suf_len[33]; + + ///////////////////////////////////////////////////////////////////////// + static bool vlc_init_tables() + { + struct vlc_src_table { int c_q, rho, u_off, e_k, e_1, cwd, cwd_len; }; + vlc_src_table tbl0[] = { + #include "table0.h" + }; + size_t tbl0_size = sizeof(tbl0) / sizeof(vlc_src_table); + + si32 pattern_popcnt[16]; + for (ui32 i = 0; i < 16; ++i) + pattern_popcnt[i] = (si32)population_count(i); + + vlc_src_table* src_tbl = tbl0; + ui32 *tgt_tbl = vlc_tbl0; + size_t tbl_size = tbl0_size; + for (int i = 0; i < 2048; ++i) + { + int c_q = i >> 8, rho = (i >> 4) & 0xF, emb = i & 0xF; + if (((emb & rho) != emb) || (rho == 0 && c_q == 0)) + tgt_tbl[i] = 0; + else + { + vlc_src_table *best_entry = NULL; + if (emb) // u_off = 1 + { + int best_e_k = -1; + for (size_t j = 0; j < tbl_size; ++j) + { + if (src_tbl[j].c_q == c_q && src_tbl[j].rho == rho) + if (src_tbl[j].u_off == 1) + if ((emb & src_tbl[j].e_k) == src_tbl[j].e_1) + { + //now we need to find the smallest cwd with the highest + // number of bits set in e_k + int ones_count = pattern_popcnt[src_tbl[j].e_k]; + if (ones_count >= best_e_k) + { + best_entry = src_tbl + j; + best_e_k = ones_count; + } + } + } + } + else // u_off = 0 + { + for (size_t j = 0; j < tbl_size; ++j) + { + if (src_tbl[j].c_q == c_q && src_tbl[j].rho == rho) + if (src_tbl[j].u_off == 0) + { + best_entry = src_tbl + j; + break; + } + } + } + assert(best_entry); + tgt_tbl[i] = (ui16)((best_entry->cwd<<8) + (best_entry->cwd_len<<4) + + best_entry->e_k); + } + } + + vlc_src_table tbl1[] = { + #include "table1.h" + }; + size_t tbl1_size = sizeof(tbl1) / sizeof(vlc_src_table); + + src_tbl = tbl1; + tgt_tbl = vlc_tbl1; + tbl_size = tbl1_size; + for (int i = 0; i < 2048; ++i) + { + int c_q = i >> 8, rho = (i >> 4) & 0xF, emb = i & 0xF; + if (((emb & rho) != emb) || (rho == 0 && c_q == 0)) + tgt_tbl[i] = 0; + else + { + vlc_src_table *best_entry = NULL; + if (emb) // u_off = 1 + { + int best_e_k = -1; + for (size_t j = 0; j < tbl_size; ++j) + { + if (src_tbl[j].c_q == c_q && src_tbl[j].rho == rho) + if (src_tbl[j].u_off == 1) + if ((emb & src_tbl[j].e_k) == src_tbl[j].e_1) + { + //now we need to find the smallest cwd with the highest + // number of bits set in e_k + int ones_count = pattern_popcnt[src_tbl[j].e_k]; + if (ones_count >= best_e_k) + { + best_entry = src_tbl + j; + best_e_k = ones_count; + } + } + } + } + else // u_off = 0 + { + for (size_t j = 0; j < tbl_size; ++j) + { + if (src_tbl[j].c_q == c_q && src_tbl[j].rho == rho) + if (src_tbl[j].u_off == 0) + { + best_entry = src_tbl + j; + break; + } + } + } + assert(best_entry); + tgt_tbl[i] = (ui16)((best_entry->cwd<<8) + (best_entry->cwd_len<<4) + + best_entry->e_k); + } + } + + + return true; + } + + ///////////////////////////////////////////////////////////////////////// + static bool uvlc_init_tables() + { + //code goes from 0 to 31, extension and 32 are not supported here + ulvc_cwd_pre[0] = 0; ulvc_cwd_pre[1] = 1; ulvc_cwd_pre[2] = 2; + ulvc_cwd_pre[3] = 4; ulvc_cwd_pre[4] = 4; + ulvc_cwd_pre_len[0] = 0; ulvc_cwd_pre_len[1] = 1; + ulvc_cwd_pre_len[2] = 2; + ulvc_cwd_pre_len[3] = 3; ulvc_cwd_pre_len[4] = 3; + ulvc_cwd_suf[0] = 0; ulvc_cwd_suf[1] = 0; ulvc_cwd_suf[2] = 0; + ulvc_cwd_suf[3] = 0; ulvc_cwd_suf[4] = 1; + ulvc_cwd_suf_len[0] = 0; ulvc_cwd_suf_len[1] = 0; + ulvc_cwd_suf_len[2] = 0; + ulvc_cwd_suf_len[3] = 1; ulvc_cwd_suf_len[4] = 1; + for (int i = 5; i < 33; ++i) + { + ulvc_cwd_pre[i] = 0; + ulvc_cwd_pre_len[i] = 3; + ulvc_cwd_suf[i] = (ui32)(i-5); + ulvc_cwd_suf_len[i] = 5; + } + return true; + } + + ///////////////////////////////////////////////////////////////////////// + bool initialize_block_encoder_tables_avx2() { + static bool tables_initialized = false; + static std::once_flag tables_initialized_flag; + std::call_once(tables_initialized_flag, []() { + memset(vlc_tbl0, 0, 2048 * sizeof(ui32)); + memset(vlc_tbl1, 0, 2048 * sizeof(ui32)); + tables_initialized = vlc_init_tables(); + tables_initialized = tables_initialized && uvlc_init_tables(); + }); + return tables_initialized; + } + + ///////////////////////////////////////////////////////////////////////// + // + ///////////////////////////////////////////////////////////////////////// + struct mel_struct { + //storage + ui8* buf; //pointer to data buffer + ui32 pos; //position of next writing within buf + ui32 buf_size; //size of buffer, which we must not exceed + + // all these can be replaced by bytes + int remaining_bits; //number of empty bits in tmp + int tmp; //temporary storage of coded bits + int run; //number of 0 run + int k; //state + int threshold; //threshold where one bit must be coded + }; + + ////////////////////////////////////////////////////////////////////////// + static inline void + mel_init(mel_struct* melp, ui32 buffer_size, ui8* data) + { + melp->buf = data; + melp->pos = 0; + melp->buf_size = buffer_size; + melp->remaining_bits = 8; + melp->tmp = 0; + melp->run = 0; + melp->k = 0; + melp->threshold = 1; // this is 1 << mel_exp[melp->k]; + } + + ////////////////////////////////////////////////////////////////////////// + static inline void + mel_emit_bit(mel_struct* melp, int v) + { + melp->tmp = (melp->tmp << 1) + v; + melp->remaining_bits--; + if (melp->remaining_bits == 0) { + melp->buf[melp->pos++] = (ui8)melp->tmp; + melp->remaining_bits = (melp->tmp == 0xFF ? 7 : 8); + melp->tmp = 0; + } + } + + ////////////////////////////////////////////////////////////////////////// + static inline void + mel_encode(mel_struct* melp, bool bit) + { + //MEL exponent + static const int mel_exp[13] = {0,0,0,1,1,1,2,2,2,3,3,4,5}; + + if (bit == false) { + ++melp->run; + if (melp->run >= melp->threshold) { + mel_emit_bit(melp, 1); + melp->run = 0; + melp->k = ojph_min(12, melp->k + 1); + melp->threshold = 1 << mel_exp[melp->k]; + } + } else { + mel_emit_bit(melp, 0); + int t = mel_exp[melp->k]; + while (t > 0) { + mel_emit_bit(melp, (melp->run >> --t) & 1); + } + melp->run = 0; + melp->k = ojph_max(0, melp->k - 1); + melp->threshold = 1 << mel_exp[melp->k]; + } + } + + ///////////////////////////////////////////////////////////////////////// + // + ///////////////////////////////////////////////////////////////////////// + struct vlc_struct_avx2 { + //storage + ui8* buf; //pointer to data buffer + ui32 pos; //position of next writing within buf + ui32 buf_size; //size of buffer, which we must not exceed + + int used_bits; //number of occupied bits in tmp + ui64 tmp; //temporary storage of coded bits + bool last_greater_than_8F; //true if last byte us greater than 0x8F + }; + + ////////////////////////////////////////////////////////////////////////// + static inline void + vlc_init(vlc_struct_avx2* vlcp, ui32 buffer_size, ui8* data) + { + vlcp->buf = data + buffer_size - 1; //points to last byte + vlcp->pos = 1; //locations will be all -pos + vlcp->buf_size = buffer_size; + + vlcp->buf[0] = 0xFF; + vlcp->used_bits = 4; + vlcp->tmp = 0xF; + vlcp->last_greater_than_8F = true; + } + + ////////////////////////////////////////////////////////////////////////// + static inline void + vlc_encode(vlc_struct_avx2* vlcp, ui32 cwd, int cwd_len) + { + vlcp->tmp |= (ui64)cwd << vlcp->used_bits; + vlcp->used_bits += cwd_len; + + while (vlcp->used_bits >= 8) { + ui8 tmp; + + if (unlikely(vlcp->last_greater_than_8F)) { + tmp = vlcp->tmp & 0x7F; + + if (likely(tmp != 0x7F)) { + tmp = vlcp->tmp & 0xFF; + *(vlcp->buf - vlcp->pos) = tmp; + vlcp->last_greater_than_8F = tmp > 0x8F; + vlcp->tmp >>= 8; + vlcp->used_bits -= 8; + } else { + *(vlcp->buf - vlcp->pos) = tmp; + vlcp->last_greater_than_8F = false; + vlcp->tmp >>= 7; + vlcp->used_bits -= 7; + } + + } else { + tmp = vlcp->tmp & 0xFF; + *(vlcp->buf - vlcp->pos) = tmp; + vlcp->last_greater_than_8F = tmp > 0x8F; + vlcp->tmp >>= 8; + vlcp->used_bits -= 8; + } + + vlcp->pos++; + } + } + + ////////////////////////////////////////////////////////////////////////// + // + ////////////////////////////////////////////////////////////////////////// + static inline void + terminate_mel_vlc(mel_struct* melp, vlc_struct_avx2* vlcp) + { + if (melp->run > 0) + mel_emit_bit(melp, 1); + + if (vlcp->last_greater_than_8F && (vlcp->tmp & 0x7f) == 0x7f) { + *(vlcp->buf - vlcp->pos) = 0x7f; + vlcp->pos++; + vlcp->tmp >>= 7; + vlcp->used_bits -= 7; + } + + melp->tmp = melp->tmp << melp->remaining_bits; + int mel_mask = (0xFF << melp->remaining_bits) & 0xFF; + int vlc_mask = 0xFF >> (8 - vlcp->used_bits); + if ((mel_mask | vlc_mask) == 0) + return; //last mel byte cannot be 0xFF, since then + //melp->remaining_bits would be < 8 + if (melp->pos >= melp->buf_size) + OJPH_ERROR(0x00020003, "mel encoder's buffer is full"); + ui8 vlcp_tmp = (ui8)vlcp->tmp; + int fuse = melp->tmp | vlcp_tmp; + if ( ( ((fuse ^ melp->tmp) & mel_mask) + | ((fuse ^ vlcp_tmp) & vlc_mask) ) == 0 + && (fuse != 0xFF) && vlcp->pos > 1) + { + melp->buf[melp->pos++] = (ui8)fuse; + } + else + { + if (vlcp->pos >= vlcp->buf_size) + OJPH_ERROR(0x00020004, "vlc encoder's buffer is full"); + melp->buf[melp->pos++] = (ui8)melp->tmp; //melp->tmp cannot be 0xFF + *(vlcp->buf - vlcp->pos) = (ui8)vlcp_tmp; + vlcp->pos++; + } + } + +///////////////////////////////////////////////////////////////////////// +// +///////////////////////////////////////////////////////////////////////// + struct ms_struct { + //storage + ui8* buf; //pointer to data buffer + ui32 pos; //position of next writing within buf + ui32 buf_size; //size of buffer, which we must not exceed + + int max_bits; //maximum number of bits that can be store in tmp + int used_bits; //number of occupied bits in tmp + ui32 tmp; //temporary storage of coded bits + }; + + ////////////////////////////////////////////////////////////////////////// + static inline void + ms_init(ms_struct* msp, ui32 buffer_size, ui8* data) + { + msp->buf = data; + msp->pos = 0; + msp->buf_size = buffer_size; + msp->max_bits = 8; + msp->used_bits = 0; + msp->tmp = 0; + } + + ////////////////////////////////////////////////////////////////////////// + static inline void + ms_encode(ms_struct* msp, ui64 cwd, int cwd_len) + { + while (cwd_len > 0) + { + if (msp->pos >= msp->buf_size) + OJPH_ERROR(0x00020005, "magnitude sign encoder's buffer is full"); + int t = ojph_min(msp->max_bits - msp->used_bits, cwd_len); + msp->tmp |= ((ui32)(cwd & ((1U << t) - 1))) << msp->used_bits; + msp->used_bits += t; + cwd >>= t; + cwd_len -= t; + if (msp->used_bits >= msp->max_bits) + { + msp->buf[msp->pos++] = (ui8)msp->tmp; + msp->max_bits = (msp->tmp == 0xFF) ? 7 : 8; + msp->tmp = 0; + msp->used_bits = 0; + } + } + } + + ////////////////////////////////////////////////////////////////////////// + static inline void + ms_terminate(ms_struct* msp) + { + if (msp->used_bits) + { + int t = msp->max_bits - msp->used_bits; //unused bits + msp->tmp |= (0xFF & ((1U << t) - 1)) << msp->used_bits; + msp->used_bits += t; + if (msp->tmp != 0xFF) + { + if (msp->pos >= msp->buf_size) + OJPH_ERROR(0x00020006, "magnitude sign encoder's buffer is full"); + msp->buf[msp->pos++] = (ui8)msp->tmp; + } + } + else if (msp->max_bits == 7) + msp->pos--; + } + +#define ZERO _mm256_setzero_si256() +#define ONE _mm256_set1_epi32(1) + +// https://stackoverflow.com/a/58827596 +inline __m256i avx2_lzcnt_epi32(__m256i v) { + // prevent value from being rounded up to the next power of two + v = _mm256_andnot_si256(_mm256_srli_epi32(v, 8), v); // keep 8 MSB + + v = _mm256_castps_si256(_mm256_cvtepi32_ps(v)); // convert an integer to float + v = _mm256_srli_epi32(v, 23); // shift down the exponent + v = _mm256_subs_epu16(_mm256_set1_epi32(158), v); // undo bias + v = _mm256_min_epi16(v, _mm256_set1_epi32(32)); // clamp at 32 + + return v; +} + +inline __m256i avx2_cmpneq_epi32(__m256i v, __m256i v2) { + return _mm256_xor_si256(_mm256_cmpeq_epi32(v, v2), _mm256_set1_epi32((int32_t)0xffffffff)); +} + +static void proc_pixel(__m256i *src_vec, ui32 p, + __m256i *eq_vec, __m256i *s_vec, + __m256i &rho_vec, __m256i &e_qmax_vec) +{ + __m256i val_vec[4]; + __m256i _eq_vec[4]; + __m256i _s_vec[4]; + __m256i _rho_vec[4]; + + for (ui32 i = 0; i < 4; ++i) { + /* val = t + t; //multiply by 2 and get rid of sign */ + val_vec[i] = _mm256_add_epi32(src_vec[i], src_vec[i]); + + /* val >>= p; // 2 \mu_p + x */ + val_vec[i] = _mm256_srli_epi32(val_vec[i], (int)p); + + /* val &= ~1u; // 2 \mu_p */ + val_vec[i] = _mm256_and_si256(val_vec[i], _mm256_set1_epi32((int)~1u)); + + /* if (val) { */ + const __m256i val_notmask = avx2_cmpneq_epi32(val_vec[i], ZERO); + + /* rho[i] = 1 << i; + * rho is processed below. + */ + + /* e_q[i] = 32 - (int)count_leading_ZEROs(--val); //2\mu_p - 1 */ + val_vec[i] = _mm256_sub_epi32(val_vec[i], ONE); + _eq_vec[i] = avx2_lzcnt_epi32(val_vec[i]); + _eq_vec[i] = _mm256_sub_epi32(_mm256_set1_epi32(32), _eq_vec[i]); + + /* e_qmax[i] = ojph_max(e_qmax[i], e_q[j]); + * e_qmax is processed below + */ + + /* s[0] = --val + (t >> 31); //v_n = 2(\mu_p-1) + s_n */ + val_vec[i] = _mm256_sub_epi32(val_vec[i], ONE); + _s_vec[i] = _mm256_srli_epi32(src_vec[i], 31); + _s_vec[i] = _mm256_add_epi32(_s_vec[i], val_vec[i]); + + _eq_vec[i] = _mm256_and_si256(_eq_vec[i], val_notmask); + _s_vec[i] = _mm256_and_si256(_s_vec[i], val_notmask); + val_vec[i] = _mm256_srli_epi32(val_notmask, 31); + /* } */ + } + + const __m256i idx = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); + + /* Reorder from + * *_vec[0]:[0, 0], [0, 1], [0, 2], [0, 3], [0, 4], [0, 5], [0, 6], [0, 7] + * *_vec[1]:[1, 0], [1, 1], [1, 2], [1, 3], [1, 4], [1, 5],.[1, 6], [1, 7] + * *_vec[2]:[0, 8], [0, 9], [0,10], [0,11], [0,12], [0,13], [0,14], [0,15] + * *_vec[3]:[1, 8], [1, 9], [1,10], [1,11], [1,12], [1,13], [1,14], [1,15] + * to + * *_vec[0]:[0, 0], [0, 2], [0, 4], [0, 6], [0, 8], [0,10], [0,12], [0,14] + * *_vec[1]:[1, 0], [1, 2], [1, 4], [1, 6], [1, 8], [1,10], [1,12], [1,14] + * *_vec[2]:[0, 1], [0, 3], [0, 5], [0, 7], [0, 9], [0,11], [0,13], [0,15] + * *_vec[3]:[1, 1], [1, 3], [1, 5], [1, 7], [1, 9], [1,11], [1,13], [1,15] + */ + __m256i tmp1, tmp2; + for (ui32 i = 0; i < 2; ++i) { + tmp1 = _mm256_permutevar8x32_epi32(_eq_vec[0 + i], idx); + tmp2 = _mm256_permutevar8x32_epi32(_eq_vec[2 + i], idx); + eq_vec[0 + i] = _mm256_permute2x128_si256(tmp1, tmp2, (0 << 0) + (2 << 4)); + eq_vec[2 + i] = _mm256_permute2x128_si256(tmp1, tmp2, (1 << 0) + (3 << 4)); + + tmp1 = _mm256_permutevar8x32_epi32(_s_vec[0 + i], idx); + tmp2 = _mm256_permutevar8x32_epi32(_s_vec[2 + i], idx); + s_vec[0 + i] = _mm256_permute2x128_si256(tmp1, tmp2, (0 << 0) + (2 << 4)); + s_vec[2 + i] = _mm256_permute2x128_si256(tmp1, tmp2, (1 << 0) + (3 << 4)); + + tmp1 = _mm256_permutevar8x32_epi32(val_vec[0 + i], idx); + tmp2 = _mm256_permutevar8x32_epi32(val_vec[2 + i], idx); + _rho_vec[0 + i] = _mm256_permute2x128_si256(tmp1, tmp2, (0 << 0) + (2 << 4)); + _rho_vec[2 + i] = _mm256_permute2x128_si256(tmp1, tmp2, (1 << 0) + (3 << 4)); + } + + e_qmax_vec = _mm256_max_epi32(eq_vec[0], eq_vec[1]); + e_qmax_vec = _mm256_max_epi32(e_qmax_vec, eq_vec[2]); + e_qmax_vec = _mm256_max_epi32(e_qmax_vec, eq_vec[3]); + _rho_vec[1] = _mm256_slli_epi32(_rho_vec[1], 1); + _rho_vec[2] = _mm256_slli_epi32(_rho_vec[2], 2); + _rho_vec[3] = _mm256_slli_epi32(_rho_vec[3], 3); + rho_vec = _mm256_or_si256(_rho_vec[0], _rho_vec[1]); + rho_vec = _mm256_or_si256(rho_vec, _rho_vec[2]); + rho_vec = _mm256_or_si256(rho_vec, _rho_vec[3]); +} + +/* from [0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, ...] + * [0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, ...] + * [0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, ...] + * [0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, ...] + * + * to [0x00, 0x10, 0x20, 0x30, 0x01, 0x11, 0x21, 0x31, + * 0x02, 0x12, 0x22, 0x32, 0x03, 0x13, 0x23, 0x33] + * + * [0x04, 0x14, 0x24, 0x34, 0x05, 0x15, 0x25, 0x35, + * 0x06, 0x16, 0x26, 0x36, 0x07, 0x17, 0x27, 0x37] + * + * [..] + */ +static void rotate_matrix(__m256i *matrix) +{ + __m256i tmp1 = _mm256_unpacklo_epi32(matrix[0], matrix[1]); + __m256i tmp2 = _mm256_unpacklo_epi32(matrix[2], matrix[3]); + __m256i tmp3 = _mm256_unpackhi_epi32(matrix[0], matrix[1]); + __m256i tmp4 = _mm256_unpackhi_epi32(matrix[2], matrix[3]); + + matrix[0] = _mm256_unpacklo_epi64(tmp1, tmp2); + matrix[1] = _mm256_unpacklo_epi64(tmp3, tmp4); + matrix[2] = _mm256_unpackhi_epi64(tmp1, tmp2); + matrix[3] = _mm256_unpackhi_epi64(tmp3, tmp4); + + tmp1 = _mm256_permute2x128_si256(matrix[0], matrix[2], 0x20); + matrix[2] = _mm256_permute2x128_si256(matrix[0], matrix[2], 0x31); + matrix[0] = tmp1; + + tmp1 = _mm256_permute2x128_si256(matrix[1], matrix[3], 0x20); + matrix[3] = _mm256_permute2x128_si256(matrix[1], matrix[3], 0x31); + matrix[1] = tmp1; +} + +static void proc_ms_encode(ms_struct *msp, + __m256i &tuple_vec, + __m256i &uq_vec, + __m256i &rho_vec, + __m256i *s_vec) +{ + __m256i m_vec[4]; + + /* Prepare parameters for ms_encode */ + /* m = (rho[i] & 1) ? Uq[i] - ((tuple[i] & 1) >> 0) : 0; */ + auto tmp = _mm256_and_si256(tuple_vec, ONE); + tmp = _mm256_sub_epi32(uq_vec, tmp); + auto tmp1 = _mm256_and_si256(rho_vec, ONE); + auto mask = avx2_cmpneq_epi32(tmp1, ZERO); + m_vec[0] = _mm256_and_si256(mask, tmp); + + /* m = (rho[i] & 2) ? Uq[i] - ((tuple[i] & 2) >> 1) : 0; */ + tmp = _mm256_and_si256(tuple_vec, _mm256_set1_epi32(2)); + tmp = _mm256_srli_epi32(tmp, 1); + tmp = _mm256_sub_epi32(uq_vec, tmp); + tmp1 = _mm256_and_si256(rho_vec, _mm256_set1_epi32(2)); + mask = avx2_cmpneq_epi32(tmp1, ZERO); + m_vec[1] = _mm256_and_si256(mask, tmp); + + /* m = (rho[i] & 4) ? Uq[i] - ((tuple[i] & 4) >> 2) : 0; */ + tmp = _mm256_and_si256(tuple_vec, _mm256_set1_epi32(4)); + tmp = _mm256_srli_epi32(tmp, 2); + tmp = _mm256_sub_epi32(uq_vec, tmp); + tmp1 = _mm256_and_si256(rho_vec, _mm256_set1_epi32(4)); + mask = avx2_cmpneq_epi32(tmp1, ZERO); + m_vec[2] = _mm256_and_si256(mask, tmp); + + /* m = (rho[i] & 8) ? Uq[i] - ((tuple[i] & 8) >> 3) : 0; */ + tmp = _mm256_and_si256(tuple_vec, _mm256_set1_epi32(8)); + tmp = _mm256_srli_epi32(tmp, 3); + tmp = _mm256_sub_epi32(uq_vec, tmp); + tmp1 = _mm256_and_si256(rho_vec, _mm256_set1_epi32(8)); + mask = avx2_cmpneq_epi32(tmp1, ZERO); + m_vec[3] = _mm256_and_si256(mask, tmp); + + rotate_matrix(m_vec); + /* s_vec from + * s_vec[0]:[0, 0], [0, 2] ... [0,14], [0, 16], [0, 18] ... [0,30] + * s_vec[1]:[1, 0], [1, 2] ... [1,14], [1, 16], [1, 18] ... [1,30] + * s_vec[2]:[0, 1], [0, 3] ... [0,15], [0, 17], [0, 19] ... [0,31] + * s_vec[3]:[1, 1], [1, 3] ... [1,15], [1, 17], [1, 19] ... [1,31] + * to + * s_vec[0]:[0, 0], [1, 0], [0, 1], [1, 1], [0, 2], [1, 2]...[0, 7], [1, 7] + * s_vec[1]:[0, 8], [1, 8], [0, 9], [1, 9], [0,10], [1,10]...[0,15], [1,15] + * s_vec[2]:[0,16], [1,16], [0,17], [1,17], [0,18], [1,18]...[0,23], [1,23] + * s_vec[3]:[0,24], [1,24], [0,25], [1,25], [0,26], [1,26]...[0,31], [1,31] + */ + rotate_matrix(s_vec); + + ui32 cwd[8]; + int cwd_len[8]; + ui64 _cwd = 0; + int _cwd_len = 0; + + /* Each iteration process 8 bytes * 2 lines */ + for (ui32 i = 0; i < 4; ++i) { + /* cwd = s[i * 4 + 0] & ((1U << m) - 1) + * cwd_len = m + */ + _mm256_storeu_si256((__m256i *)cwd_len, m_vec[i]); + tmp = _mm256_sllv_epi32(ONE, m_vec[i]); + tmp = _mm256_sub_epi32(tmp, ONE); + tmp = _mm256_and_si256(tmp, s_vec[i]); + _mm256_storeu_si256((__m256i*)cwd, tmp); + + for (ui32 j = 0; j < 4; ++j) { + ui32 idx = j * 2; + _cwd = cwd[idx]; + _cwd_len = cwd_len[idx]; + _cwd |= ((ui64)cwd[idx + 1]) << _cwd_len; + _cwd_len += cwd_len[idx + 1]; + ms_encode(msp, _cwd, _cwd_len); + } + } +} + +static __m256i cal_eps_vec(__m256i *eq_vec, __m256i &u_q_vec, + __m256i &e_qmax_vec) +{ + /* if (u_q[i] > 0) { + * eps[i] |= (e_q[i * 4 + 0] == e_qmax[i]); + * eps[i] |= (e_q[i * 4 + 1] == e_qmax[i]) << 1; + * eps[i] |= (e_q[i * 4 + 2] == e_qmax[i]) << 2; + * eps[i] |= (e_q[i * 4 + 3] == e_qmax[i]) << 3; + * } + */ + auto u_q_mask = _mm256_cmpgt_epi32(u_q_vec, ZERO); + + auto mask = _mm256_cmpeq_epi32(eq_vec[0], e_qmax_vec); + auto eps_vec = _mm256_srli_epi32(mask, 31); + + mask = _mm256_cmpeq_epi32(eq_vec[1], e_qmax_vec); + auto tmp = _mm256_srli_epi32(mask, 31); + tmp = _mm256_slli_epi32(tmp, 1); + eps_vec = _mm256_or_si256(eps_vec, tmp); + + mask = _mm256_cmpeq_epi32(eq_vec[2], e_qmax_vec); + tmp = _mm256_srli_epi32(mask, 31); + tmp = _mm256_slli_epi32(tmp, 2); + eps_vec = _mm256_or_si256(eps_vec, tmp); + + mask = _mm256_cmpeq_epi32(eq_vec[3], e_qmax_vec); + tmp = _mm256_srli_epi32(mask, 31); + tmp = _mm256_slli_epi32(tmp, 3); + eps_vec = _mm256_or_si256(eps_vec, tmp); + + return _mm256_and_si256(u_q_mask, eps_vec); +} + +static void update_lep(ui32 x, __m256i &prev_e_val_vec, + __m256i *eq_vec, __m256i *e_val_vec, + const __m256i left_shift) +{ + /* lep[0] = ojph_max(lep[0], (ui8)e_q[1]); lep++; + * lep[0] = (ui8)e_q[3]; + * Compare e_q[1] with e_q[3] of the prevous round. + */ + auto tmp = _mm256_permutevar8x32_epi32(eq_vec[3], left_shift); + tmp = _mm256_insert_epi32(tmp, _mm_cvtsi128_si32(_mm256_castsi256_si128(prev_e_val_vec)), 0); + prev_e_val_vec = _mm256_insert_epi32(ZERO, _mm256_extract_epi32(eq_vec[3], 7), 0); + e_val_vec[x] = _mm256_max_epi32(eq_vec[1], tmp); +} + + +static void update_lcxp(ui32 x, __m256i &prev_cx_val_vec, + __m256i &rho_vec, __m256i *cx_val_vec, + const __m256i left_shift) +{ + /* lcxp[0] = (ui8)(lcxp[0] | (ui8)((rho[0] & 2) >> 1)); lcxp++; + * lcxp[0] = (ui8)((rho[0] & 8) >> 3); + * Or (rho[0] & 2) and (rho[0] of the previous round & 8). + */ + auto tmp = _mm256_permutevar8x32_epi32(rho_vec, left_shift); + tmp = _mm256_insert_epi32(tmp, _mm_cvtsi128_si32(_mm256_castsi256_si128(prev_cx_val_vec)), 0); + prev_cx_val_vec = _mm256_insert_epi32(ZERO, _mm256_extract_epi32(rho_vec, 7), 0); + + tmp = _mm256_and_si256(tmp, _mm256_set1_epi32(8)); + tmp = _mm256_srli_epi32(tmp, 3); + + auto tmp1 = _mm256_and_si256(rho_vec, _mm256_set1_epi32(2)); + tmp1 = _mm256_srli_epi32(tmp1, 1); + cx_val_vec[x] = _mm256_or_si256(tmp, tmp1); +} + +static __m256i cal_tuple(__m256i &cq_vec, __m256i &rho_vec, + __m256i &eps_vec, ui32 *vlc_tbl) +{ + /* tuple[i] = vlc_tbl1[(c_q[i] << 8) + (rho[i] << 4) + eps[i]]; */ + auto tmp = _mm256_slli_epi32(cq_vec, 8); + auto tmp1 = _mm256_slli_epi32(rho_vec, 4); + tmp = _mm256_add_epi32(tmp, tmp1); + tmp = _mm256_add_epi32(tmp, eps_vec); + return _mm256_i32gather_epi32((const int *)vlc_tbl, tmp, 4); +} + +static __m256i proc_cq1(ui32 x, __m256i *cx_val_vec, __m256i &rho_vec, + const __m256i right_shift) +{ + ojph_unused(x); + ojph_unused(cx_val_vec); + ojph_unused(right_shift); + + /* c_q[i + 1] = (rho[i] >> 1) | (rho[i] & 1); */ + auto tmp = _mm256_srli_epi32(rho_vec, 1); + auto tmp1 = _mm256_and_si256(rho_vec, ONE); + return _mm256_or_si256(tmp, tmp1); +} + +static __m256i proc_cq2(ui32 x, __m256i *cx_val_vec, __m256i &rho_vec, + const __m256i right_shift) +{ + // c_q[i + 1] = (lcxp[i + 1] + (lcxp[i + 2] << 2)) + // | (((rho[i] & 4) >> 1) | ((rho[i] & 8) >> 2)); + auto lcxp1_vec = _mm256_permutevar8x32_epi32(cx_val_vec[x], right_shift); + auto tmp = _mm256_permutevar8x32_epi32(lcxp1_vec, right_shift); + +#ifdef OJPH_ARCH_X86_64 + tmp = _mm256_insert_epi64(tmp, + _mm_cvtsi128_si64(_mm256_castsi256_si128(cx_val_vec[x + 1])), 3); +#elif (defined OJPH_ARCH_I386) + int lsb = _mm_cvtsi128_si32(_mm256_castsi256_si128(cx_val_vec[x + 1])); + tmp = _mm256_insert_epi32(tmp, lsb, 6); + int msb = _mm_extract_epi32(_mm256_castsi256_si128(cx_val_vec[x + 1]), 1); + tmp = _mm256_insert_epi32(tmp, msb, 7); +#else + #error Error unsupport compiler +#endif + tmp = _mm256_slli_epi32(tmp, 2); + auto tmp1 = _mm256_insert_epi32(lcxp1_vec, + _mm_cvtsi128_si32(_mm256_castsi256_si128(cx_val_vec[x + 1])), 7); + tmp = _mm256_add_epi32(tmp1, tmp); + + tmp1 = _mm256_and_si256(rho_vec, _mm256_set1_epi32(4)); + tmp1 = _mm256_srli_epi32(tmp1, 1); + tmp = _mm256_or_si256(tmp, tmp1); + + tmp1 = _mm256_and_si256(rho_vec, _mm256_set1_epi32(8)); + tmp1 = _mm256_srli_epi32(tmp1, 2); + + return _mm256_or_si256(tmp, tmp1); +} + +using fn_proc_cq = __m256i (*)(ui32, __m256i *, __m256i &, const __m256i); + +static void proc_mel_encode1(mel_struct *melp, __m256i &cq_vec, + __m256i &rho_vec, __m256i u_q_vec, ui32 ignore, + const __m256i right_shift) +{ + int32_t mel_need_encode[8]; + int32_t mel_need_encode2[8]; + int32_t mel_bit[8]; + int32_t mel_bit2[8]; + /* Prepare mel_encode params */ + /* if (c_q[i] == 0) { */ + _mm256_storeu_si256((__m256i *)mel_need_encode, _mm256_cmpeq_epi32(cq_vec, ZERO)); + /* mel_encode(&mel, rho[i] != 0); */ + _mm256_storeu_si256((__m256i*)mel_bit, _mm256_srli_epi32(avx2_cmpneq_epi32(rho_vec, ZERO), 31)); + /* } */ + + /* mel_encode(&mel, ojph_min(u_q[i], u_q[i + 1]) > 2); */ + auto tmp = _mm256_permutevar8x32_epi32(u_q_vec, right_shift); + auto tmp1 = _mm256_min_epi32(u_q_vec, tmp); + _mm256_storeu_si256((__m256i*)mel_bit2, _mm256_srli_epi32(_mm256_cmpgt_epi32(tmp1, _mm256_set1_epi32(2)), 31)); + + /* if (u_q[i] > 0 && u_q[i + 1] > 0) { } */ + auto need_encode2 = _mm256_cmpgt_epi32(u_q_vec, ZERO); + _mm256_storeu_si256((__m256i*)mel_need_encode2, _mm256_and_si256(need_encode2, _mm256_cmpgt_epi32(tmp, ZERO))); + + ui32 i_max = 8 - (ignore / 2); + + for (ui32 i = 0; i < i_max; i += 2) { + if (mel_need_encode[i]) { + mel_encode(melp, mel_bit[i]); + } + + if (i + 1 < i_max) { + if (mel_need_encode[i + 1]) { + mel_encode(melp, mel_bit[i + 1]); + } + } + + if (mel_need_encode2[i]) { + mel_encode(melp, mel_bit2[i]); + } + } +} + +static void proc_mel_encode2(mel_struct *melp, __m256i &cq_vec, + __m256i &rho_vec, __m256i u_q_vec, ui32 ignore, + const __m256i right_shift) +{ + ojph_unused(u_q_vec); + ojph_unused(right_shift); + int32_t mel_need_encode[8]; + int32_t mel_bit[8]; + + /* Prepare mel_encode params */ + /* if (c_q[i] == 0) { */ + _mm256_storeu_si256((__m256i*)mel_need_encode, _mm256_cmpeq_epi32(cq_vec, ZERO)); + /* mel_encode(&mel, rho[i] != 0); */ + _mm256_storeu_si256((__m256i*)mel_bit, _mm256_srli_epi32(avx2_cmpneq_epi32(rho_vec, ZERO), 31)); + /* } */ + + ui32 i_max = 8 - (ignore / 2); + + for (ui32 i = 0; i < i_max; ++i) { + if (mel_need_encode[i]) { + mel_encode(melp, mel_bit[i]); + } + } +} + +using fn_proc_mel_encode = void (*)(mel_struct *, __m256i &, __m256i &, + __m256i, ui32, const __m256i); + +static void proc_vlc_encode1(vlc_struct_avx2 *vlcp, ui32 *tuple, + ui32 *u_q, ui32 ignore) +{ + ui32 i_max = 8 - (ignore / 2); + + for (ui32 i = 0; i < i_max; i += 2) { + /* 7 bits */ + ui32 val = tuple[i + 0] >> 4; + int size = tuple[i + 0] & 7; + + if (i + 1 < i_max) { + /* 7 bits */ + val |= (tuple[i + 1] >> 4) << size; + size += tuple[i + 1] & 7; + } + + if (u_q[i] > 2 && u_q[i + 1] > 2) { + /* 3 bits */ + val |= (ulvc_cwd_pre[u_q[i] - 2]) << size; + size += ulvc_cwd_pre_len[u_q[i] - 2]; + + /* 3 bits */ + val |= (ulvc_cwd_pre[u_q[i + 1] - 2]) << size; + size += ulvc_cwd_pre_len[u_q[i + 1] - 2]; + + /* 5 bits */ + val |= (ulvc_cwd_suf[u_q[i] - 2]) << size; + size += ulvc_cwd_suf_len[u_q[i] - 2]; + + /* 5 bits */ + val |= (ulvc_cwd_suf[u_q[i + 1] - 2]) << size; + size += ulvc_cwd_suf_len[u_q[i + 1] - 2]; + + } else if (u_q[i] > 2 && u_q[i + 1] > 0) { + /* 3 bits */ + val |= (ulvc_cwd_pre[u_q[i]]) << size; + size += ulvc_cwd_pre_len[u_q[i]]; + + /* 1 bit */ + val |= (u_q[i + 1] - 1) << size; + size += 1; + + /* 5 bits */ + val |= (ulvc_cwd_suf[u_q[i]]) << size; + size += ulvc_cwd_suf_len[u_q[i]]; + + } else { + /* 3 bits */ + val |= (ulvc_cwd_pre[u_q[i]]) << size; + size += ulvc_cwd_pre_len[u_q[i]]; + + /* 3 bits */ + val |= (ulvc_cwd_pre[u_q[i + 1]]) << size; + size += ulvc_cwd_pre_len[u_q[i + 1]]; + + /* 5 bits */ + val |= (ulvc_cwd_suf[u_q[i]]) << size; + size += ulvc_cwd_suf_len[u_q[i]]; + + /* 5 bits */ + val |= (ulvc_cwd_suf[u_q[i + 1]]) << size; + size += ulvc_cwd_suf_len[u_q[i + 1]]; + } + + vlc_encode(vlcp, val, size); + } +} + +static void proc_vlc_encode2(vlc_struct_avx2 *vlcp, ui32 *tuple, + ui32 *u_q, ui32 ignore) +{ + ui32 i_max = 8 - (ignore / 2); + + for (ui32 i = 0; i < i_max; i += 2) { + /* 7 bits */ + ui32 val = tuple[i + 0] >> 4; + int size = tuple[i + 0] & 7; + + if (i + 1 < i_max) { + /* 7 bits */ + val |= (tuple[i + 1] >> 4) << size; + size += tuple[i + 1] & 7; + } + + /* 3 bits */ + val |= ulvc_cwd_pre[u_q[i]] << size; + size += ulvc_cwd_pre_len[u_q[i]]; + + /* 3 bits */ + val |= (ulvc_cwd_pre[u_q[i + 1]]) << size; + size += ulvc_cwd_pre_len[u_q[i + 1]]; + + /* 5 bits */ + val |= (ulvc_cwd_suf[u_q[i + 0]]) << size; + size += ulvc_cwd_suf_len[u_q[i + 0]]; + + /* 5 bits */ + val |= (ulvc_cwd_suf[u_q[i + 1]]) << size; + size += ulvc_cwd_suf_len[u_q[i + 1]]; + + vlc_encode(vlcp, val, size); + } +} + +using fn_proc_vlc_encode = void (*)(vlc_struct_avx2 *, ui32 *, ui32 *, ui32); + +void ojph_encode_codeblock_avx2(ui32* buf, ui32 missing_msbs, + ui32 num_passes, ui32 _width, ui32 height, + ui32 stride, ui32* lengths, + ojph::mem_elastic_allocator *elastic, + ojph::coded_lists *& coded) +{ + ojph_unused(num_passes); //currently not used + + ui32 width = (_width + 15) & ~15u; + ui32 ignore = width - _width; + const int ms_size = (16384 * 16 + 14) / 15; //more than enough + const int mel_vlc_size = 3072; //more than enough + const int mel_size = 192; + const int vlc_size = mel_vlc_size - mel_size; + + ui8 ms_buf[ms_size]; + ui8 mel_vlc_buf[mel_vlc_size]; + ui8 *mel_buf = mel_vlc_buf; + ui8 *vlc_buf = mel_vlc_buf + mel_size; + + mel_struct mel; + mel_init(&mel, mel_size, mel_buf); + vlc_struct_avx2 vlc; + vlc_init(&vlc, vlc_size, vlc_buf); + ms_struct ms; + ms_init(&ms, ms_size, ms_buf); + + const ui32 p = 30 - missing_msbs; + + //e_val: E values for a line (these are the highest set bit) + //cx_val: is the context values + //Each byte stores the info for the 2 sample. For E, it is maximum + // of the two samples, while for cx, it is the OR of these two samples. + //The maximum is between the pixel at the bottom left of one quad + // and the bottom right of the earlier quad. The same is true for cx. + //For a 1024 pixels, we need 512 bytes, the 2 extra, + // one for the non-existing earlier quad, and one for beyond the + // the end + const __m256i right_shift = _mm256_set_epi32( + 0, 7, 6, 5, 4, 3, 2, 1 + ); + + const __m256i left_shift = _mm256_set_epi32( + 6, 5, 4, 3, 2, 1, 0, 7 + ); + + ui32 n_loop = (width + 15) / 16; + + __m256i e_val_vec[65]; + for (ui32 i = 0; i > 3); */ + __m256i tmp = _mm256_and_si256(prev_cx_val_vec, _mm256_set1_epi32(8)); + cx_val_vec[n_loop] = _mm256_srli_epi32(tmp, 3); + + prev_e_val_vec = ZERO; + prev_cx_val_vec = ZERO; + + ui32 *sp = buf + y * stride; + + /* 16 bytes per iteration */ + for (ui32 x = 0; x < n_loop; ++x) { + + /* t = sp[i]; */ + if ((x == (n_loop - 1)) && (_width % 16)) { + ui32 tmp_buf[16] = { 0 }; + memcpy(tmp_buf, sp, (_width % 16) * sizeof(ui32)); + src_vec[0] = _mm256_loadu_si256((__m256i*)(tmp_buf)); + src_vec[2] = _mm256_loadu_si256((__m256i*)(tmp_buf + 8)); + if (y + 1 < height) { + memcpy(tmp_buf, sp + stride, (_width % 16) * sizeof(ui32)); + src_vec[1] = _mm256_loadu_si256((__m256i*)(tmp_buf)); + src_vec[3] = _mm256_loadu_si256((__m256i*)(tmp_buf + 8)); + } + else { + src_vec[1] = ZERO; + src_vec[3] = ZERO; + } + } + else { + src_vec[0] = _mm256_loadu_si256((__m256i*)(sp)); + src_vec[2] = _mm256_loadu_si256((__m256i*)(sp + 8)); + + if (y + 1 < height) { + src_vec[1] = _mm256_loadu_si256((__m256i*)(sp + stride)); + src_vec[3] = _mm256_loadu_si256((__m256i*)(sp + 8 + stride)); + } + else { + src_vec[1] = ZERO; + src_vec[3] = ZERO; + } + sp += 16; + } + + /* src_vec layout: + * src_vec[0]:[0, 0],[0, 1],[0, 2],[0, 3],[0, 4],[0, 5],.[0, 6],.[0, 7] + * src_vec[1]:[1, 0],[1, 1],[1, 2],[1, 3],[1, 4],[1, 5],.[1, 6],.[1, 7] + * src_vec[2]:[0, 8],[0, 9],[0,10],[0,11],[0,12],[0,13],.[0,14], [0,15] + * src_vec[3]:[1, 8],[1, 9],[1,10],[1,11],[1,12],[1,13],.[1,14], [1,15] + */ + __m256i rho_vec, e_qmax_vec; + proc_pixel(src_vec, p, eq_vec, s_vec, rho_vec, e_qmax_vec); + + // max_e[(i + 1) % num] = ojph_max(lep[i + 1], lep[i + 2]) - 1; + tmp = _mm256_permutevar8x32_epi32(e_val_vec[x], right_shift); + tmp = _mm256_insert_epi32(tmp, _mm_cvtsi128_si32(_mm256_castsi256_si128(e_val_vec[x + 1])), 7); + + auto max_e_vec = _mm256_max_epi32(tmp, e_val_vec[x]); + max_e_vec = _mm256_sub_epi32(max_e_vec, ONE); + + // kappa[i] = (rho[i] & (rho[i] - 1)) ? ojph_max(1, max_e[i]) : 1; + tmp = _mm256_max_epi32(max_e_vec, ONE); + __m256i tmp1 = _mm256_sub_epi32(rho_vec, ONE); + tmp1 = _mm256_and_si256(rho_vec, tmp1); + + auto cmp = _mm256_cmpeq_epi32(tmp1, ZERO); + auto kappa_vec1_ = _mm256_and_si256(cmp, ONE); + auto kappa_vec2_ = _mm256_and_si256(_mm256_xor_si256(cmp, _mm256_set1_epi32((int32_t)0xffffffff)), tmp); + const __m256i kappa_vec = _mm256_max_epi32(kappa_vec1_, kappa_vec2_); + + /* cq[1 - 16] = cq_vec + * cq[0] = prev_cq_vec[0] + */ + tmp = proc_cq(x, cx_val_vec, rho_vec, right_shift); + + auto cq_vec = _mm256_permutevar8x32_epi32(tmp, left_shift); + cq_vec = _mm256_insert_epi32(cq_vec, prev_cq, 0); + prev_cq = (ui32)_mm256_extract_epi32(tmp, 7); + + update_lep(x, prev_e_val_vec, eq_vec, e_val_vec, left_shift); + update_lcxp(x, prev_cx_val_vec, rho_vec, cx_val_vec, left_shift); + + /* Uq[i] = ojph_max(e_qmax[i], kappa[i]); */ + /* u_q[i] = Uq[i] - kappa[i]; */ + auto uq_vec = _mm256_max_epi32(kappa_vec, e_qmax_vec); + auto u_q_vec = _mm256_sub_epi32(uq_vec, kappa_vec); + + auto eps_vec = cal_eps_vec(eq_vec, u_q_vec, e_qmax_vec); + __m256i tuple_vec = cal_tuple(cq_vec, rho_vec, eps_vec, vlc_tbl); + ui32 _ignore = ((n_loop - 1) == x) ? ignore : 0; + + proc_mel_encode(&mel, cq_vec, rho_vec, u_q_vec, _ignore, + right_shift); + + proc_ms_encode(&ms, tuple_vec, uq_vec, rho_vec, s_vec); + + // vlc_encode(&vlc, tuple[i*2+0] >> 8, (tuple[i*2+0] >> 4) & 7); + // vlc_encode(&vlc, tuple[i*2+1] >> 8, (tuple[i*2+1] >> 4) & 7); + ui32 u_q[8]; + ui32 tuple[8]; + /* The tuple is scaled by 4 due to: + * vlc_encode(&vlc, tuple0 >> 8, (tuple0 >> 4) & 7, true); + * So in the vlc_encode, the tuple will only be scaled by 2. + */ + tuple_vec = _mm256_srli_epi32(tuple_vec, 4); + _mm256_storeu_si256((__m256i*)tuple, tuple_vec); + _mm256_storeu_si256((__m256i*)u_q, u_q_vec); + + proc_vlc_encode(&vlc, tuple, u_q, _ignore); + } + + tmp = _mm256_permutevar8x32_epi32(cx_val_vec[0], right_shift); + tmp = _mm256_slli_epi32(tmp, 2); + tmp = _mm256_add_epi32(tmp, cx_val_vec[0]); + prev_cq = (ui32)_mm_cvtsi128_si32(_mm256_castsi256_si128(tmp)); + + proc_cq = proc_cq2; + vlc_tbl = vlc_tbl1; + proc_mel_encode = proc_mel_encode2; + proc_vlc_encode = proc_vlc_encode2; + } + + ms_terminate(&ms); + terminate_mel_vlc(&mel, &vlc); + + //copy to elastic + lengths[0] = mel.pos + vlc.pos + ms.pos; + elastic->get_buffer(mel.pos + vlc.pos + ms.pos, coded); + memcpy(coded->buf, ms.buf, ms.pos); + memcpy(coded->buf + ms.pos, mel.buf, mel.pos); + memcpy(coded->buf + ms.pos + mel.pos, vlc.buf - vlc.pos + 1, vlc.pos); + + // put in the interface locator word + ui32 num_bytes = mel.pos + vlc.pos; + coded->buf[lengths[0]-1] = (ui8)(num_bytes >> 4); + coded->buf[lengths[0]-2] = coded->buf[lengths[0]-2] & 0xF0; + coded->buf[lengths[0]-2] = + (ui8)(coded->buf[lengths[0]-2] | (num_bytes & 0xF)); + + coded->avail_size -= lengths[0]; +} + +} /* namespace local */ +} /* namespace ojph */ + +#endif diff --git a/external/OpenJPH/src/core/openjph/ojph_arch.h b/external/OpenJPH/src/core/openjph/ojph_arch.h index 097c46f15..f8ba7992d 100644 --- a/external/OpenJPH/src/core/openjph/ojph_arch.h +++ b/external/OpenJPH/src/core/openjph/ojph_arch.h @@ -5,6 +5,7 @@ // Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia +// Copyright (c) 2026, Osamu Watanabe // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -39,6 +40,7 @@ #ifndef OJPH_ARCH_H #define OJPH_ARCH_H +#include #include #include #include @@ -63,6 +65,17 @@ #include #endif + ///////////////////////////////////////////////////////////////////////////// + // portable force-inline / no-inline function qualifiers + ///////////////////////////////////////////////////////////////////////////// +#ifdef OJPH_COMPILER_MSVC + #define OJPH_FORCE_INLINE static __forceinline + #define OJPH_NO_INLINE static __declspec(noinline) +#else + #define OJPH_FORCE_INLINE static inline __attribute__((always_inline)) + #define OJPH_NO_INLINE static __attribute__((noinline)) +#endif + /////////////////////////////////////////////////////////////////////////////// // preprocessor directives for architecture /////////////////////////////////////////////////////////////////////////////// @@ -88,12 +101,20 @@ #define OJPH_ARCH_UNKNOWN #endif +// Only little-endian POWER (ppc64le) is supported for SIMD +#if defined(OJPH_ARCH_PPC64) && \ + (defined(__LITTLE_ENDIAN__) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + #define OJPH_ARCH_PPC64LE +#endif + namespace ojph { //////////////////////////////////////////////////////////////////////////// // disable SIMD for unknown architecture //////////////////////////////////////////////////////////////////////////// #if !defined(OJPH_ARCH_X86_64) && !defined(OJPH_ARCH_I386) && \ - !defined(OJPH_ARCH_ARM) && !defined(OJPH_DISABLE_SIMD) + !defined(OJPH_ARCH_ARM) && !defined(OJPH_ARCH_PPC64LE) && \ + !defined(OJPH_DISABLE_SIMD) #define OJPH_DISABLE_SIMD #endif // !OJPH_ARCH_UNKNOWN @@ -152,6 +173,14 @@ namespace ojph { ARM_CPU_EXT_LEVEL_SVE2 = 3, }; + // POWER9 (ISA 3.0) is the minimum supported SIMD level; older CPUs + // (POWER8 and earlier) use the generic code paths + enum : int { + PPC_CPU_EXT_LEVEL_GENERIC = 0, + PPC_CPU_EXT_LEVEL_ARCH_3_00 = 1, // ISA 3.0 (POWER9) + PPC_CPU_EXT_LEVEL_ARCH_3_1 = 2, // ISA 3.1 (POWER10) + }; + ///////////////////////////////////////////////////////////////////////////// static inline ui32 population_count(ui32 val) { @@ -255,6 +284,35 @@ namespace ojph { #endif } + ///////////////////////////////////////////////////////////////////////////// +#ifdef OJPH_COMPILER_MSVC + #pragma intrinsic(_BitScanForward64) +#endif + static inline ui32 count_trailing_zeros(ui64 val) + { + #ifdef OJPH_COMPILER_MSVC + unsigned long result = 0; + #if (defined OJPH_ARCH_X86_64) || (defined OJPH_ARCH_ARM) + _BitScanForward64(&result, val); + #elif (defined OJPH_ARCH_I386) + ui32 lsb = (ui32)val, msb = (ui32)(val >> 32); + if (lsb != 0) + _BitScanForward(&result, lsb); + else { + _BitScanForward(&result, msb); + result += 32; + } + #endif + return (ui32)result; + #elif (defined OJPH_COMPILER_GNUC) + return (ui32)__builtin_ctzll(val); + #else + if ((ui32)val != 0) + return count_trailing_zeros((ui32)val); + return 32 + count_trailing_zeros((ui32)(val >> 32)); + #endif + } + //////////////////////////////////////////////////////////////////////////// static inline si32 ojph_round(float val) { @@ -316,6 +374,124 @@ namespace ojph { return reinterpret_cast(p); } + //////////////////////////////////////////////////////////////////////////// + // Determine the byte order of the target at compile time when possible, + // so that the compiler can remove the branches for the other byte order. + // __BYTE_ORDER__ is a predefined macro that describes the target + // architecture, not the machine running the compiler, so it is also + // correct when cross-compiling. + // All MSVC targets (x86, x64, ARM64 Windows) are little endian. +#if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + constexpr bool is_machine_little_endian = false; +#elif defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + constexpr bool is_machine_little_endian = true; +#elif defined(OJPH_COMPILER_MSVC) + constexpr bool is_machine_little_endian = true; +#else + // fallback in case macro __BYTE_ORDER__ is not defined + // If the first byte in memory is 0x01, the machine is Little Endian. + // If the first byte in memory is 0x00, the machine is Big Endian. + static bool check_if_machine_is_little_endian() + { + const uint16_t n = 0x0001; + bool is_machine_little_endian = (*((uint8_t *)&n) == 0x01); + return is_machine_little_endian; + } + const bool is_machine_little_endian = check_if_machine_is_little_endian(); +#endif + + //////////////////////////////////////////////////////////////////////////// + // swap bytes 1 2 --> 2 1 on big-endian machines + static inline ui16 swap_bytes_if_be(ui16 t) + { + if (is_machine_little_endian) + return t; + else + return (ui16)((t << 8) | (t >> 8)); + } + //////////////////////////////////////////////////////////////////////////// + // swap bytes 1 2 --> 2 1 on little-endian machines + static inline ui16 swap_bytes_if_le(ui16 t) + { + if (is_machine_little_endian) + return (ui16)((t << 8) | (t >> 8)); + else + return t; + } + //////////////////////////////////////////////////////////////////////////// + // swap bytes 1 2 3 4 --> 4 3 2 1 on big-endian machines + static inline ui32 swap_bytes_if_be(ui32 t) + { + if (is_machine_little_endian) + return t; + else + { + ui32 u = swap_bytes_if_be((ui16)(t & 0xFFFFu)); + u <<= 16; + u |= swap_bytes_if_be((ui16)(t >> 16)); + return u; + } + } + //////////////////////////////////////////////////////////////////////////// + // swap bytes 1 2 3 4 --> 4 3 2 1 on little-endian machines + static inline ui32 swap_bytes_if_le(ui32 t) + { + if (is_machine_little_endian) + { + ui32 u = swap_bytes_if_le((ui16)(t & 0xFFFFu)); + u <<= 16; + u |= swap_bytes_if_le((ui16)(t >> 16)); + return u; + } + else + return t; + } + //////////////////////////////////////////////////////////////////////////// + // swap bytes 1 2 3 4 5 6 7 8 --> 8 7 6 5 4 3 2 1 on little-endian machines + static inline ui64 swap_bytes_if_le(ui64 t) + { + if (is_machine_little_endian) + { + ui64 u = + swap_bytes_if_le((ui32)(t & 0xFFFFFFFFu)); + u <<= 32; + u |= swap_bytes_if_le((ui32)(t >> 32)); + return u; + } + else + return t; + } + + //////////////////////////////////////////////////////////////////////////// + // loads 4 bytes from p as a little-endian 32-bit integer; that is, the + // byte at the lowest address goes into the least-significant byte of the + // result, irrespective of the machine's endianness + static inline ui32 load_le_ui32(const ui8 *p) + { + if (is_machine_little_endian) { + ui32 val; + std::memcpy(&val, p, sizeof(val)); + return val; + } + else + return (ui32)p[0] | ((ui32)p[1] << 8) + | ((ui32)p[2] << 16) | ((ui32)p[3] << 24); + } + + //////////////////////////////////////////////////////////////////////////// + // loads two consecutive ui16 values from p, placing the one at the lower + // address in the least-significant 16 bits of the result, irrespective + // of the machine's endianness + static inline ui32 load_le_ui16x2(const ui16 *p) + { + if (is_machine_little_endian) { + ui32 val; + std::memcpy(&val, p, sizeof(val)); + return val; + } + else + return (ui32)p[0] | ((ui32)p[1] << 16); + } } #endif // !OJPH_ARCH_H diff --git a/external/OpenJPH/src/core/openjph/ojph_base.h b/external/OpenJPH/src/core/openjph/ojph_base.h index f71e90a61..d8f380ec4 100644 --- a/external/OpenJPH/src/core/openjph/ojph_base.h +++ b/external/OpenJPH/src/core/openjph/ojph_base.h @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2019, Aous Naman +// Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -65,8 +65,18 @@ namespace ojph { { point org; size siz; + + bool operator==(const rect& a) const { + return a.org.x == org.x && a.org.y == org.y + && a.siz.w == siz.w && a.siz.h == siz.h; + } + + bool operator!=(const rect& a) const + { return !(a == *this); } }; - + + ///////////////////////////////////////////////////////////////////////////// + } #endif // !OJPH_BASE_H diff --git a/external/OpenJPH/src/core/openjph/ojph_params.h b/external/OpenJPH/src/core/openjph/ojph_params.h index c55048db0..4c7dafbf9 100644 --- a/external/OpenJPH/src/core/openjph/ojph_params.h +++ b/external/OpenJPH/src/core/openjph/ojph_params.h @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2019, Aous Naman +// Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -48,7 +48,6 @@ namespace ojph { // defined here class param_siz; class param_cod; - class param_coc; class param_qcd; class param_cap; class param_nlt; @@ -59,7 +58,6 @@ namespace ojph { namespace local { struct param_siz; struct param_cod; - struct param_coc; struct param_qcd; struct param_cap; struct param_nlt; @@ -99,19 +97,37 @@ namespace ojph { local::param_siz* state; }; - /***************************************************************************/ + /***************************************************************************** + * @brief An interface to COD and COC marker segments. + * + * The param_cod object uses Pimpl design. + * The top set of functions give access to the COD marker segment, while + * the lower set, the ones that have comp_idx as the first parameter, + * gives access to COC marker segment. + * The functions: + * - set_num_decomposition(ui32 comp_idx, ...) + * - set_block_dims(ui32 comp_idx, ...) + * - set_precinct_size(ui32 comp_idx, ...) + * - set_reversible(ui32 comp_idx, ...) + * create a COC segment on first call; subsequent calls to these + * functions on the same component index will use the COC segment + * created by the first call. On first creation, the COC segment is + * initialized to the default COD settings; in particular, 5 levels of + * decomposition, 64x64 codeblocks, reversible 5/3 transform and no + * precinct size is defined, which gives 32768x32768 precincts. + */ class OJPH_EXPORT param_cod { public: param_cod(local::param_cod* p) : state(p) {} + // COD marker segment interface void set_num_decomposition(ui32 num_decompositions); void set_block_dims(ui32 width, ui32 height); void set_precinct_size(int num_levels, size* precinct_size); void set_progression_order(const char *name); void set_color_transform(bool color_transform); void set_reversible(bool reversible); - param_coc get_coc(ui32 component_idx); ui32 get_num_decompositions() const; size get_block_dims() const; @@ -127,28 +143,19 @@ namespace ojph { bool packets_use_eph() const; bool get_block_vertical_causality() const; - private: - local::param_cod* state; - }; - - /***************************************************************************/ - class OJPH_EXPORT param_coc - { - public: - param_coc(local::param_cod* p) : state(p) {} - - void set_num_decomposition(ui32 num_decompositions); - void set_block_dims(ui32 width, ui32 height); - void set_precinct_size(int num_levels, size* precinct_size); - void set_reversible(bool reversible); + // COC marker segment interface + void set_num_decomposition(ui32 comp_idx, ui32 num_decompositions); + void set_block_dims(ui32 comp_idx, ui32 width, ui32 height); + void set_precinct_size(ui32 comp_idx, int num_levels, size* precinct_size); + void set_reversible(ui32 comp_idx, bool reversible); - ui32 get_num_decompositions() const; - size get_block_dims() const; - size get_log_block_dims() const; - bool is_reversible() const; - size get_precinct_size(ui32 level_num) const; - size get_log_precinct_size(ui32 level_num) const; - bool get_block_vertical_causality() const; + ui32 get_num_decompositions(ui32 comp_idx) const; + size get_block_dims(ui32 comp_idx) const; + size get_log_block_dims(ui32 comp_idx) const; + bool is_reversible(ui32 comp_idx) const; + size get_precinct_size(ui32 comp_idx, ui32 level_num) const; + size get_log_precinct_size(ui32 comp_idx, ui32 level_num) const; + bool get_block_vertical_causality(ui32 comp_idx) const; private: local::param_cod* state; @@ -157,36 +164,53 @@ namespace ojph { /***************************************************************************/ /** * @brief Quantization parameters object - * + * */ class OJPH_EXPORT param_qcd { public: + enum comp_type : ui8 { + OJPH_COMP_Y, + OJPH_COMP_CB, + OJPH_COMP_CR + }; + param_qcd(local::param_qcd* p) : state(p) {} /** - * @brief Set the irreversible quantization base delta. - * - * This represents the default base delta and influences QCD marker + * @brief Set the irreversible quantization base delta. + * + * This represents the default base delta and influences QCD marker * segment - * - * @param delta + * + * @param delta */ void set_irrev_quant(float delta); /** - * @brief Set the irreversible quantization base delta for a specific + * @brief Set the irreversible quantization base delta for a specific * component - * - * This represents the default base delta for component comp_idx, and + * + * This represents the default base delta for component comp_idx, and * influences QCC marker segment for the component, inserting one * if needed, which is usually the case. - * - * @param comp_idx - * @param delta + * + * @param comp_idx + * @param delta */ void set_irrev_quant(ui32 comp_idx, float delta); + /** + * @brief Sets Qfactor for a specific component. + * + * Setting Qfactor takes precedence over setting an irreversible quantization base delta + * + * @param comp_idx Component index + * @param ctype Indicates whether the component is a Y, Cb or Cr channel, after the ICT if present + * @param qfactor Compression quality as an integer between 1 (worst quality) and 100 (best quality) + */ + void set_qfactor(ui32 comp_idx, comp_type ctype, ui8 qfactor); + private: local::param_qcd* state; }; @@ -195,82 +219,82 @@ namespace ojph { /** * @brief non-linearity point transformation object * (implements NLT marker segment) - * - * There are a few things to know here. - * The NLT marker segment contains the nonlinearity type and the + * + * There are a few things to know here. + * The NLT marker segment contains the nonlinearity type and the * bit depth and signedness of the component to which it applies. - * There is the default component ALL_COMPS which applies to all + * There is the default component ALL_COMPS which applies to all * components unless it is overridden by another NLT segment marker. * The library checks that the settings make sense, and also make * sure that bit depth and signedness are correct, creating any missing * NLT marker segments in the process. * If all components have the same bit depth and signedness, and need - * nonlinearity type 3 (Binary Complement to Sign Magnitude Conversion), + * nonlinearity type 3 (Binary Complement to Sign Magnitude Conversion), * then the best option is to set ALL_COMPS to type 3. - * Otherwise, the best option is to set type 3 only to components that + * Otherwise, the best option is to set type 3 only to components that * need it, leaving out the default ALL_COMPS nonlinearity not set. - * Another option is for the end-user can set the ALL_COMPS to type 3, - * and then put exception for the components that does not need type 3, + * Another option is for the end-user can set the ALL_COMPS to type 3, + * and then put exception for the components that does not need type 3, * by setting them to type 0. - * + * * The library, during validity check, which is run when the codestream * is created for writing, will do the following: - * -- If ALL_COMPS is set to type 0, it will be ignored, and the + * -- If ALL_COMPS is set to type 0, it will be ignored, and the * codestream will NOT have the corresponding NLT marker segment. * -- If ALL_COMPS is set to type 3, then the following will happen: - * - If all the components (except those with type 0 set for them) have - * the same bit depth and signedness, then the ALL_COMPS NLT marker + * - If all the components (except those with type 0 set for them) have + * the same bit depth and signedness, then the ALL_COMPS NLT marker * segment will be respected and inserted into the codestream. * Of course, components with NLT 0 will also have the corresponding * NLT marker segment inserted. * - If components, for which no NTL type 0 is specified, have differing - * bit depth or signedness, then the ALL_COMPS will be ignored, and + * bit depth or signedness, then the ALL_COMPS will be ignored, and * NLT markers are inserted for each component that needs type 3. * Components that have their component field larger than the number of * components in the codestream are removed. - * - * It also worth noting that type 3 nonlinearity has no effect on - * positive image samples. It is also not recommended for integer-valued - * types. It is only recommended for floating-point image samples, for - * which some of the samples are negative, where type 3 nonlinearity - * should be beneficial. This is because the encoding engine expects - * two-complement representation for negative values while floating point - * numbers have a sign bit followed by an exponent, which has a biased + * + * It also worth noting that type 3 nonlinearity has no effect on + * positive image samples. It is also not recommended for integer-valued + * types. It is only recommended for floating-point image samples, for + * which some of the samples are negative, where type 3 nonlinearity + * should be beneficial. This is because the encoding engine expects + * two-complement representation for negative values while floating point + * numbers have a sign bit followed by an exponent, which has a biased * integer representation. The core idea is to make floating-point * representation more compatible with integer representation. - * + * */ class OJPH_EXPORT param_nlt { public: enum special_comp_num : ui16 { ALL_COMPS = 65535 }; - enum nonlinearity : ui8 { + enum nonlinearity : ui8 { OJPH_NLT_NO_NLT = 0, // supported OJPH_NLT_GAMMA_STYLE_NLT = 1, // not supported OJPH_NLT_LUT_STYLE_NLT = 2, // not supported OJPH_NLT_BINARY_COMPLEMENT_NLT = 3, // supported - OJPH_NLT_UNDEFINED = 255 // This is used internally and is - // not part of the standard + OJPH_NLT_UNDEFINED = 255 // This is used internally and is + // not part of the standard }; public: param_nlt(local::param_nlt* p) : state(p) {} /** - * @brief enables or disables type 3 nonlinearity for a component + * @brief enables or disables type 3 nonlinearity for a component * or the default setting - * + * * When creating a codestream for writing, call this function before * you call codestream::write_headers. - * - * + * + * * @param comp_num: component number, or 65535 for the default setting * @param type: desired non-linearity from enum nonlinearity */ void set_nonlinear_transform(ui32 comp_num, ui8 nl_type); /** - * @brief get the nonlinearity type associated with comp_num, which + * @brief get the nonlinearity type associated with comp_num, which * should be one from enum nonlinearity * * @param comp_num: component number, or 65535 for the default setting @@ -279,7 +303,7 @@ namespace ojph { * @param type: nonlinearity type * @return true if the nonlinearity for comp_num is set */ - bool get_nonlinear_transform(ui32 comp_num, ui8& bit_depth, + bool get_nonlinear_transform(ui32 comp_num, ui8& bit_depth, bool& is_signed, ui8& nl_type) const; private: diff --git a/external/OpenJPH/src/core/openjph/ojph_simd_vsx.h b/external/OpenJPH/src/core/openjph/ojph_simd_vsx.h new file mode 100644 index 000000000..5bdf2c207 --- /dev/null +++ b/external/OpenJPH/src/core/openjph/ojph_simd_vsx.h @@ -0,0 +1,318 @@ +//***************************************************************************/ +// This software is released under the 2-Clause BSD license, included +// below. +// +// Copyright (c) 2026, Aous Naman +// Copyright (c) 2026, Kakadu Software Pty Ltd, Australia +// Copyright (c) 2026, The University of New South Wales, Australia +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************/ +// This file is part of the OpenJPH software implementation. +// File: ojph_simd_vsx.h +// +// 128-bit SIMD helpers for POWER VSX, used by the ojph_*_vsx.cpp +// kernels. Lane numbering and operation semantics follow the same +// conventions as the other 128-bit kernels in this codebase (lane 0 +// is the lowest memory address). Supported targets are POWER9 +// (ISA 3.0) and newer, little-endian only (ppc64le). +//***************************************************************************/ + +#ifndef OJPH_SIMD_VSX_H +#define OJPH_SIMD_VSX_H + +#if !defined(__powerpc64__) && !defined(__PPC64__) + #error "this header is for 64-bit POWER targets only" +#endif +#if !defined(__LITTLE_ENDIAN__) && \ + !(defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #error "this header assumes a little-endian target (ppc64le)" +#endif + +#include +#include + +// altivec.h leaks these context-sensitive keywords as macros under GNU C; +// they break standard headers and the codebase (e.g. std::vector) +#undef vector +#undef pixel +#undef bool + +typedef __vector unsigned char v128_t; + +typedef __vector signed char vsx_v_i8; +typedef __vector unsigned char vsx_v_u8; +typedef __vector signed short vsx_v_i16; +typedef __vector unsigned short vsx_v_u16; +typedef __vector signed int vsx_v_i32; +typedef __vector unsigned int vsx_v_u32; +typedef __vector signed long long vsx_v_i64; +typedef __vector unsigned long long vsx_v_u64; +typedef __vector float vsx_v_f32; + +//--------------------------------------------------------------------------- +// load/store (alignment-agnostic; lxv/stxv handle unaligned addresses) +//--------------------------------------------------------------------------- +static inline v128_t vsx_v128_load(const void *p) +{ return vec_xl(0, (const unsigned char *)p); } + +static inline void vsx_v128_store(void *p, v128_t a) +{ vec_xst(a, 0, (unsigned char *)p); } + +#define vsx_v128_store32_lane(p, a, i) \ + do { vsx_v_i32 t_ = (vsx_v_i32)(a); int v_ = t_[(i)]; \ + std::memcpy((p), &v_, 4); } while (0) + +//--------------------------------------------------------------------------- +// constants, splats, makes +//--------------------------------------------------------------------------- +// functions, not macros, so that an argument that is itself a macro +// expanding to an argument list (e.g. OJPH_REPEAT4) works +static inline v128_t vsx_i8x16_const( + signed char c0, signed char c1, signed char c2, signed char c3, + signed char c4, signed char c5, signed char c6, signed char c7, + signed char c8, signed char c9, signed char c10, signed char c11, + signed char c12, signed char c13, signed char c14, signed char c15) +{ vsx_v_i8 v = {c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15}; + return (v128_t)v; } +static inline v128_t vsx_i16x8_const(short c0, short c1, short c2, + short c3, short c4, short c5, + short c6, short c7) +{ vsx_v_i16 v = {c0,c1,c2,c3,c4,c5,c6,c7}; return (v128_t)v; } +static inline v128_t vsx_u16x8_const(unsigned short c0, unsigned short c1, + unsigned short c2, unsigned short c3, + unsigned short c4, unsigned short c5, + unsigned short c6, unsigned short c7) +{ vsx_v_u16 v = {c0,c1,c2,c3,c4,c5,c6,c7}; return (v128_t)v; } +static inline v128_t vsx_i32x4_const(int c0, int c1, int c2, int c3) +{ vsx_v_i32 v = {c0,c1,c2,c3}; return (v128_t)v; } +static inline v128_t vsx_u32x4_const(unsigned int c0, unsigned int c1, + unsigned int c2, unsigned int c3) +{ vsx_v_u32 v = {c0,c1,c2,c3}; return (v128_t)v; } +static inline v128_t vsx_i64x2_const(long long c0, long long c1) +{ vsx_v_i64 v = {c0,c1}; return (v128_t)v; } +static inline v128_t vsx_u64x2_const(unsigned long long c0, + unsigned long long c1) +{ vsx_v_u64 v = {c0,c1}; return (v128_t)v; } + +static inline v128_t vsx_i8x16_splat(signed char x) +{ return (v128_t)vec_splats(x); } +static inline v128_t vsx_i16x8_splat(short x) +{ return (v128_t)vec_splats(x); } +static inline v128_t vsx_i32x4_splat(int x) +{ return (v128_t)vec_splats(x); } +static inline v128_t vsx_u32x4_splat(unsigned int x) +{ return (v128_t)vec_splats(x); } +static inline v128_t vsx_i64x2_splat(long long x) +{ return (v128_t)vec_splats((signed long long)x); } +static inline v128_t vsx_f32x4_splat(float x) +{ return (v128_t)vec_splats(x); } + +static inline v128_t vsx_i32x4_make(int a, int b, int c, int d) +{ return (v128_t)(vsx_v_i32){a, b, c, d}; } + +//--------------------------------------------------------------------------- +// lane extraction (subscript is little-endian lane order) +//--------------------------------------------------------------------------- +#define vsx_u8x16_extract_lane(a, i) (((vsx_v_u8)(a))[(i)]) +#define vsx_u16x8_extract_lane(a, i) (((vsx_v_u16)(a))[(i)]) +#define vsx_i32x4_extract_lane(a, i) (((vsx_v_i32)(a))[(i)]) +#define vsx_u32x4_extract_lane(a, i) (((vsx_v_u32)(a))[(i)]) +#define vsx_i64x2_extract_lane(a, i) (((vsx_v_i64)(a))[(i)]) + +//--------------------------------------------------------------------------- +// bitwise +//--------------------------------------------------------------------------- +static inline v128_t vsx_v128_and(v128_t a, v128_t b) +{ return vec_and(a, b); } +static inline v128_t vsx_v128_or(v128_t a, v128_t b) +{ return vec_or(a, b); } +static inline v128_t vsx_v128_xor(v128_t a, v128_t b) +{ return vec_xor(a, b); } +// a & ~b (same operand order as vec_andc) +static inline v128_t vsx_v128_andnot(v128_t a, v128_t b) +{ return vec_andc(a, b); } + +//--------------------------------------------------------------------------- +// integer arithmetic +//--------------------------------------------------------------------------- +static inline v128_t vsx_i8x16_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_i8)a, (vsx_v_i8)b); } +static inline v128_t vsx_i16x8_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_i16)a, (vsx_v_i16)b); } +static inline v128_t vsx_i32x4_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_i32)a, (vsx_v_i32)b); } +static inline v128_t vsx_i64x2_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_i64)a, (vsx_v_i64)b); } + +static inline v128_t vsx_i16x8_sub(v128_t a, v128_t b) +{ return (v128_t)vec_sub((vsx_v_i16)a, (vsx_v_i16)b); } +static inline v128_t vsx_i32x4_sub(v128_t a, v128_t b) +{ return (v128_t)vec_sub((vsx_v_i32)a, (vsx_v_i32)b); } +static inline v128_t vsx_i64x2_sub(v128_t a, v128_t b) +{ return (v128_t)vec_sub((vsx_v_i64)a, (vsx_v_i64)b); } + +// low half of products; vmladduhm / vmuluwm; i64x2 is lowered by the +// compiler (mulld on ISA 3.0, vmulld on ISA 3.1) +static inline v128_t vsx_i16x8_mul(v128_t a, v128_t b) +{ return (v128_t)((vsx_v_i16)a * (vsx_v_i16)b); } +static inline v128_t vsx_i32x4_mul(v128_t a, v128_t b) +{ return (v128_t)((vsx_v_i32)a * (vsx_v_i32)b); } +static inline v128_t vsx_i64x2_mul(v128_t a, v128_t b) +{ return (v128_t)((vsx_v_i64)a * (vsx_v_i64)b); } + +static inline v128_t vsx_i8x16_abs(v128_t a) +{ return (v128_t)vec_abs((vsx_v_i8)a); } +static inline v128_t vsx_u8x16_min(v128_t a, v128_t b) +{ return (v128_t)vec_min((vsx_v_u8)a, (vsx_v_u8)b); } +static inline v128_t vsx_i16x8_max(v128_t a, v128_t b) +{ return (v128_t)vec_max((vsx_v_i16)a, (vsx_v_i16)b); } + +//--------------------------------------------------------------------------- +// shifts (scalar count, modulo lane width) +//--------------------------------------------------------------------------- +static inline v128_t vsx_i16x8_shl(v128_t a, int n) +{ return (v128_t)vec_sl((vsx_v_i16)a, vec_splats((unsigned short)n)); } +static inline v128_t vsx_i32x4_shl(v128_t a, int n) +{ return (v128_t)vec_sl((vsx_v_i32)a, vec_splats((unsigned int)n)); } +static inline v128_t vsx_i64x2_shl(v128_t a, int n) +{ return (v128_t)vec_sl((vsx_v_i64)a, + vec_splats((unsigned long long)n)); } + +static inline v128_t vsx_i32x4_shr(v128_t a, int n) // arithmetic +{ return (v128_t)vec_sra((vsx_v_i32)a, vec_splats((unsigned int)n)); } +static inline v128_t vsx_i64x2_shr(v128_t a, int n) // arithmetic +{ return (v128_t)vec_sra((vsx_v_i64)a, + vec_splats((unsigned long long)n)); } + +static inline v128_t vsx_u16x8_shr(v128_t a, int n) // logical +{ return (v128_t)vec_sr((vsx_v_u16)a, vec_splats((unsigned short)n)); } +static inline v128_t vsx_u32x4_shr(v128_t a, int n) // logical +{ return (v128_t)vec_sr((vsx_v_u32)a, vec_splats((unsigned int)n)); } +static inline v128_t vsx_u64x2_shr(v128_t a, int n) // logical +{ return (v128_t)vec_sr((vsx_v_u64)a, + vec_splats((unsigned long long)n)); } + +//--------------------------------------------------------------------------- +// comparisons (true lanes -> all-ones, false lanes -> all-zeros) +//--------------------------------------------------------------------------- +static inline v128_t vsx_i8x16_eq(v128_t a, v128_t b) +{ return (v128_t)vec_cmpeq((vsx_v_i8)a, (vsx_v_i8)b); } +static inline v128_t vsx_i16x8_eq(v128_t a, v128_t b) +{ return (v128_t)vec_cmpeq((vsx_v_i16)a, (vsx_v_i16)b); } +static inline v128_t vsx_i32x4_eq(v128_t a, v128_t b) +{ return (v128_t)vec_cmpeq((vsx_v_i32)a, (vsx_v_i32)b); } + +static inline v128_t vsx_i8x16_gt(v128_t a, v128_t b) +{ return (v128_t)vec_cmpgt((vsx_v_i8)a, (vsx_v_i8)b); } +static inline v128_t vsx_i32x4_gt(v128_t a, v128_t b) +{ return (v128_t)vec_cmpgt((vsx_v_i32)a, (vsx_v_i32)b); } +static inline v128_t vsx_i32x4_lt(v128_t a, v128_t b) +{ return (v128_t)vec_cmplt((vsx_v_i32)a, (vsx_v_i32)b); } +static inline v128_t vsx_i64x2_lt(v128_t a, v128_t b) +{ return (v128_t)vec_cmplt((vsx_v_i64)a, (vsx_v_i64)b); } + +static inline v128_t vsx_f32x4_ge(v128_t a, v128_t b) +{ return (v128_t)vec_cmpge((vsx_v_f32)a, (vsx_v_f32)b); } +static inline v128_t vsx_f32x4_lt(v128_t a, v128_t b) +{ return (v128_t)vec_cmplt((vsx_v_f32)a, (vsx_v_f32)b); } + +//--------------------------------------------------------------------------- +// float arithmetic and conversions +//--------------------------------------------------------------------------- +static inline v128_t vsx_f32x4_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_f32)a, (vsx_v_f32)b); } +static inline v128_t vsx_f32x4_sub(v128_t a, v128_t b) +{ return (v128_t)vec_sub((vsx_v_f32)a, (vsx_v_f32)b); } +static inline v128_t vsx_f32x4_mul(v128_t a, v128_t b) +{ return (v128_t)vec_mul((vsx_v_f32)a, (vsx_v_f32)b); } + +// xvcvspsxws: truncating, saturating (NaN gives 0x80000000; the +// callers never pass NaN) +static inline v128_t vsx_i32x4_trunc_sat_f32x4(v128_t a) +{ return (v128_t)vec_cts((vsx_v_f32)a, 0); } +static inline v128_t vsx_f32x4_convert_i32x4(v128_t a) +{ return (v128_t)vec_ctf((vsx_v_i32)a, 0); } + +//--------------------------------------------------------------------------- +// widening +//--------------------------------------------------------------------------- +static inline v128_t vsx_i64x2_extend_low_i32x4(v128_t a) +{ + vsx_v_i32 v = (vsx_v_i32)a; + return (v128_t)__builtin_convertvector( + __builtin_shufflevector(v, v, 0, 1), vsx_v_i64); +} +static inline v128_t vsx_i64x2_extend_high_i32x4(v128_t a) +{ + vsx_v_i32 v = (vsx_v_i32)a; + return (v128_t)__builtin_convertvector( + __builtin_shufflevector(v, v, 2, 3), vsx_v_i64); +} + +//--------------------------------------------------------------------------- +// shuffles (immediate lane indices; 0..N-1 from a, N..2N-1 from b) +//--------------------------------------------------------------------------- +#define vsx_i8x16_shuffle(a, b, c0,c1,c2,c3,c4,c5,c6,c7, \ + c8,c9,c10,c11,c12,c13,c14,c15) \ + ((v128_t)__builtin_shufflevector((vsx_v_u8)(a), (vsx_v_u8)(b), \ + c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15)) +#define vsx_i16x8_shuffle(a, b, c0,c1,c2,c3,c4,c5,c6,c7) \ + ((v128_t)__builtin_shufflevector((vsx_v_i16)(a), (vsx_v_i16)(b), \ + c0,c1,c2,c3,c4,c5,c6,c7)) +#define vsx_i32x4_shuffle(a, b, c0,c1,c2,c3) \ + ((v128_t)__builtin_shufflevector((vsx_v_i32)(a), (vsx_v_i32)(b), \ + c0,c1,c2,c3)) +#define vsx_i64x2_shuffle(a, b, c0,c1) \ + ((v128_t)__builtin_shufflevector((vsx_v_i64)(a), (vsx_v_i64)(b), c0,c1)) + +//--------------------------------------------------------------------------- +// swizzle: runtime byte-table lookup; lanes with index > 15 give 0 +//--------------------------------------------------------------------------- +static inline v128_t vsx_i8x16_swizzle(v128_t a, v128_t idx) +{ + v128_t r = vec_perm(a, a, idx); + v128_t oob = (v128_t)vec_cmpgt((vsx_v_u8)idx, + vec_splats((unsigned char)15)); + return vec_andc(r, oob); +} + +//--------------------------------------------------------------------------- +// bitmask: MSB of each byte lane -> bit of result, lane 0 -> bit 0 +// (vbpermq gathers the 16 selected bits into bits 48..63 of the +// big-endian first doubleword, which is doubleword 1 on ppc64le) +//--------------------------------------------------------------------------- +static inline int vsx_i8x16_bitmask(v128_t a) +{ +#if defined(__POWER10_VECTOR__) + return (int)vec_extractm(a); // ISA 3.1 native movemask +#else + const vsx_v_u8 perm = { 120, 112, 104, 96, 88, 80, 72, 64, + 56, 48, 40, 32, 24, 16, 8, 0 }; + vsx_v_u64 r = (vsx_v_u64)vec_bperm(a, perm); + return (int)r[1]; +#endif +} + +#endif // OJPH_SIMD_VSX_H diff --git a/external/OpenJPH/src/core/openjph/ojph_version.h b/external/OpenJPH/src/core/openjph/ojph_version.h index 77b692bc0..ee0f2a38e 100644 --- a/external/OpenJPH/src/core/openjph/ojph_version.h +++ b/external/OpenJPH/src/core/openjph/ojph_version.h @@ -34,5 +34,6 @@ //***************************************************************************/ #define OPENJPH_VERSION_MAJOR 0 -#define OPENJPH_VERSION_MINOR 27 -#define OPENJPH_VERSION_PATCH 3 +#define OPENJPH_VERSION_MINOR 30 +#define OPENJPH_VERSION_PATCH 1 +#define OPENJPH_VERSION_SHA 2d0a033a135fb58dab87ea9551db8870e5b68548 diff --git a/external/OpenJPH/src/core/others/ojph_arch.cpp b/external/OpenJPH/src/core/others/ojph_arch.cpp index b44aa1cdb..0ab12b8a0 100644 --- a/external/OpenJPH/src/core/others/ojph_arch.cpp +++ b/external/OpenJPH/src/core/others/ojph_arch.cpp @@ -5,7 +5,8 @@ // Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia -// +// Copyright (c) 2026, Osamu Watanabe +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -143,8 +144,12 @@ namespace ojph { osxsave_avail && ((xcr_val & 0xE0) == 0xE0); bool avx512f_avail = (avx2_abcd[1] & 0x10000) != 0; bool avx512cd_avail = (avx2_abcd[1] & 0x10000000) != 0; - bool avx512_avail = - zmm_avail && avx512f_avail && avx512cd_avail; + bool avx512bw_avail = (avx2_abcd[1] & 0x40000000) != 0; + bool avx512vl_avail = + (avx2_abcd[1] & 0x80000000u) != 0; + bool avx512_avail = zmm_avail && avx512f_avail + && avx512cd_avail && avx512bw_avail + && avx512vl_avail; if (avx512_avail) level = X86_CPU_EXT_LEVEL_AVX512; } @@ -224,7 +229,38 @@ namespace ojph { #endif - #else // architectures other than Intel/AMD and ARM + #elif defined(OJPH_ARCH_PPC64LE) + + #if defined(OJPH_OS_LINUX) + + #include + #include + + bool init_cpu_ext_level(int& level) { + unsigned long hwcap = getauxval(AT_HWCAP); + unsigned long hwcap2 = getauxval(AT_HWCAP2); + level = PPC_CPU_EXT_LEVEL_GENERIC; + if ((hwcap & PPC_FEATURE_HAS_VSX) && + (hwcap2 & PPC_FEATURE2_ARCH_3_00)) { + level = PPC_CPU_EXT_LEVEL_ARCH_3_00; + #ifdef PPC_FEATURE2_ARCH_3_1 + if (hwcap2 & PPC_FEATURE2_ARCH_3_1) + level = PPC_CPU_EXT_LEVEL_ARCH_3_1; + #endif + } + return true; + } + + #else // !OJPH_OS_LINUX + + bool init_cpu_ext_level(int& level) { + level = PPC_CPU_EXT_LEVEL_GENERIC; + return true; + } + + #endif + + #else // architectures other than Intel/AMD, ARM, and PPC64LE //////////////////////////////////////////////////////////////////////////// bool init_cpu_ext_level(int& level) { diff --git a/external/OpenJPH/src/core/shared/ojph_simd_vsx.h b/external/OpenJPH/src/core/shared/ojph_simd_vsx.h new file mode 100644 index 000000000..fae483d86 --- /dev/null +++ b/external/OpenJPH/src/core/shared/ojph_simd_vsx.h @@ -0,0 +1,361 @@ +//***************************************************************************/ +// This software is released under the 2-Clause BSD license, included +// below. +// +// Copyright (c) 2026, Aous Naman +// Copyright (c) 2026, Kakadu Software Pty Ltd, Australia +// Copyright (c) 2026, The University of New South Wales, Australia +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************/ +// This file is part of the OpenJPH software implementation. +// File: ojph_simd_vsx.h +// +// 128-bit SIMD helpers for POWER VSX, used by the ojph_*_vsx.cpp +// kernels. Lane numbering and operation semantics follow the same +// conventions as the other 128-bit kernels in this codebase (lane 0 +// is the lowest memory address). Supported targets are POWER9 +// (ISA 3.0) and newer, little-endian only (ppc64le). +//***************************************************************************/ + +#ifndef OJPH_SIMD_VSX_H +#define OJPH_SIMD_VSX_H + +#if !defined(__powerpc64__) && !defined(__PPC64__) + #error "this header is for 64-bit POWER targets only" +#endif +#if !defined(__LITTLE_ENDIAN__) && \ + !(defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #error "this header assumes a little-endian target (ppc64le)" +#endif + +#include +#include + +#include "ojph_defs.h" + +// altivec.h leaks these context-sensitive keywords as macros under GNU C; +// they break standard headers and the codebase (e.g. std::vector) +#undef vector +#undef pixel +#undef bool + +typedef __vector unsigned char v128_t; + +typedef __vector signed char vsx_v_i8; +typedef __vector unsigned char vsx_v_u8; +typedef __vector signed short vsx_v_i16; +typedef __vector unsigned short vsx_v_u16; +typedef __vector signed int vsx_v_i32; +typedef __vector unsigned int vsx_v_u32; +typedef __vector signed long long vsx_v_i64; +typedef __vector unsigned long long vsx_v_u64; +typedef __vector float vsx_v_f32; + +//--------------------------------------------------------------------------- +// load/store (alignment-agnostic; lxv/stxv handle unaligned addresses) +//--------------------------------------------------------------------------- +static inline v128_t vsx_v128_load(const void *p) +{ return vec_xl(0, (const unsigned char *)p); } + +static inline void vsx_v128_store(void *p, v128_t a) +{ vec_xst(a, 0, (unsigned char *)p); } + +#define vsx_v128_store32_lane(p, a, i) \ + do { vsx_v_i32 t_ = (vsx_v_i32)(a); int v_ = t_[(i)]; \ + std::memcpy((p), &v_, 4); } while (0) + +//--------------------------------------------------------------------------- +// constants, splats, makes +//--------------------------------------------------------------------------- +// functions, not macros, so that an argument that is itself a macro +// expanding to an argument list (e.g. OJPH_REPEAT4) works +static inline v128_t vsx_i8x16_const( + signed char c0, signed char c1, signed char c2, signed char c3, + signed char c4, signed char c5, signed char c6, signed char c7, + signed char c8, signed char c9, signed char c10, signed char c11, + signed char c12, signed char c13, signed char c14, signed char c15) +{ vsx_v_i8 v = {c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15}; + return (v128_t)v; } +static inline v128_t vsx_i16x8_const(short c0, short c1, short c2, + short c3, short c4, short c5, + short c6, short c7) +{ vsx_v_i16 v = {c0,c1,c2,c3,c4,c5,c6,c7}; return (v128_t)v; } +static inline v128_t vsx_u16x8_const(unsigned short c0, unsigned short c1, + unsigned short c2, unsigned short c3, + unsigned short c4, unsigned short c5, + unsigned short c6, unsigned short c7) +{ vsx_v_u16 v = {c0,c1,c2,c3,c4,c5,c6,c7}; return (v128_t)v; } +static inline v128_t vsx_i32x4_const(int c0, int c1, int c2, int c3) +{ vsx_v_i32 v = {c0,c1,c2,c3}; return (v128_t)v; } +static inline v128_t vsx_u32x4_const(unsigned int c0, unsigned int c1, + unsigned int c2, unsigned int c3) +{ vsx_v_u32 v = {c0,c1,c2,c3}; return (v128_t)v; } +static inline v128_t vsx_i64x2_const(long long c0, long long c1) +{ vsx_v_i64 v = {c0,c1}; return (v128_t)v; } +static inline v128_t vsx_u64x2_const(unsigned long long c0, + unsigned long long c1) +{ vsx_v_u64 v = {c0,c1}; return (v128_t)v; } + +static inline v128_t vsx_i8x16_splat(signed char x) +{ ojph_unused(x); return (v128_t)vec_splats(x); } +static inline v128_t vsx_i16x8_splat(short x) +{ ojph_unused(x); return (v128_t)vec_splats(x); } +static inline v128_t vsx_i32x4_splat(int x) +{ ojph_unused(x); return (v128_t)vec_splats(x); } +static inline v128_t vsx_u32x4_splat(unsigned int x) +{ ojph_unused(x); return (v128_t)vec_splats(x); } +static inline v128_t vsx_i64x2_splat(long long x) +{ ojph_unused(x); return (v128_t)vec_splats((signed long long)x); } +static inline v128_t vsx_f32x4_splat(float x) +{ ojph_unused(x); return (v128_t)vec_splats(x); } + +static inline v128_t vsx_i32x4_make(int a, int b, int c, int d) +{ return (v128_t)(vsx_v_i32){a, b, c, d}; } + +//--------------------------------------------------------------------------- +// lane extraction (subscript is little-endian lane order) +//--------------------------------------------------------------------------- +#define vsx_u8x16_extract_lane(a, i) (((vsx_v_u8)(a))[(i)]) +#define vsx_u16x8_extract_lane(a, i) (((vsx_v_u16)(a))[(i)]) +#define vsx_i32x4_extract_lane(a, i) (((vsx_v_i32)(a))[(i)]) +#define vsx_u32x4_extract_lane(a, i) (((vsx_v_u32)(a))[(i)]) +#define vsx_i64x2_extract_lane(a, i) (((vsx_v_i64)(a))[(i)]) + +//--------------------------------------------------------------------------- +// bitwise +//--------------------------------------------------------------------------- +static inline v128_t vsx_v128_and(v128_t a, v128_t b) +{ return vec_and(a, b); } +static inline v128_t vsx_v128_or(v128_t a, v128_t b) +{ return vec_or(a, b); } +static inline v128_t vsx_v128_xor(v128_t a, v128_t b) +{ return vec_xor(a, b); } +// a & ~b (same operand order as vec_andc) +static inline v128_t vsx_v128_andnot(v128_t a, v128_t b) +{ return vec_andc(a, b); } + +//--------------------------------------------------------------------------- +// integer arithmetic +//--------------------------------------------------------------------------- +static inline v128_t vsx_i8x16_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_i8)a, (vsx_v_i8)b); } +static inline v128_t vsx_i16x8_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_i16)a, (vsx_v_i16)b); } +static inline v128_t vsx_i32x4_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_i32)a, (vsx_v_i32)b); } +static inline v128_t vsx_i64x2_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_i64)a, (vsx_v_i64)b); } + +static inline v128_t vsx_i16x8_sub(v128_t a, v128_t b) +{ return (v128_t)vec_sub((vsx_v_i16)a, (vsx_v_i16)b); } +static inline v128_t vsx_i32x4_sub(v128_t a, v128_t b) +{ return (v128_t)vec_sub((vsx_v_i32)a, (vsx_v_i32)b); } +static inline v128_t vsx_i64x2_sub(v128_t a, v128_t b) +{ return (v128_t)vec_sub((vsx_v_i64)a, (vsx_v_i64)b); } + +// low half of products; vmladduhm / vmuluwm; i64x2 is lowered by the +// compiler (mulld on ISA 3.0, vmulld on ISA 3.1) +static inline v128_t vsx_i16x8_mul(v128_t a, v128_t b) +{ return (v128_t)((vsx_v_i16)a * (vsx_v_i16)b); } +static inline v128_t vsx_i32x4_mul(v128_t a, v128_t b) +{ return (v128_t)((vsx_v_i32)a * (vsx_v_i32)b); } +static inline v128_t vsx_i64x2_mul(v128_t a, v128_t b) +{ return (v128_t)((vsx_v_i64)a * (vsx_v_i64)b); } + +static inline v128_t vsx_i8x16_abs(v128_t a) +{ return (v128_t)vec_abs((vsx_v_i8)a); } +static inline v128_t vsx_u8x16_min(v128_t a, v128_t b) +{ return (v128_t)vec_min((vsx_v_u8)a, (vsx_v_u8)b); } +static inline v128_t vsx_i16x8_max(v128_t a, v128_t b) +{ return (v128_t)vec_max((vsx_v_i16)a, (vsx_v_i16)b); } + +//--------------------------------------------------------------------------- +// shifts (scalar count, modulo lane width) +//--------------------------------------------------------------------------- +static inline v128_t vsx_i16x8_shl(v128_t a, int n) +{ return (v128_t)vec_sl((vsx_v_i16)a, vec_splats((unsigned short)n)); } +static inline v128_t vsx_i32x4_shl(v128_t a, int n) +{ return (v128_t)vec_sl((vsx_v_i32)a, vec_splats((unsigned int)n)); } +static inline v128_t vsx_i64x2_shl(v128_t a, int n) +{ return (v128_t)vec_sl((vsx_v_i64)a, + vec_splats((unsigned long long)n)); } + +static inline v128_t vsx_i32x4_shr(v128_t a, int n) // arithmetic +{ return (v128_t)vec_sra((vsx_v_i32)a, vec_splats((unsigned int)n)); } +static inline v128_t vsx_i64x2_shr(v128_t a, int n) // arithmetic +{ return (v128_t)vec_sra((vsx_v_i64)a, + vec_splats((unsigned long long)n)); } + +static inline v128_t vsx_u16x8_shr(v128_t a, int n) // logical +{ return (v128_t)vec_sr((vsx_v_u16)a, vec_splats((unsigned short)n)); } +static inline v128_t vsx_u32x4_shr(v128_t a, int n) // logical +{ return (v128_t)vec_sr((vsx_v_u32)a, vec_splats((unsigned int)n)); } +static inline v128_t vsx_u64x2_shr(v128_t a, int n) // logical +{ return (v128_t)vec_sr((vsx_v_u64)a, + vec_splats((unsigned long long)n)); } + +//--------------------------------------------------------------------------- +// comparisons (true lanes -> all-ones, false lanes -> all-zeros) +//--------------------------------------------------------------------------- +static inline v128_t vsx_i8x16_eq(v128_t a, v128_t b) +{ return (v128_t)vec_cmpeq((vsx_v_i8)a, (vsx_v_i8)b); } +static inline v128_t vsx_i16x8_eq(v128_t a, v128_t b) +{ return (v128_t)vec_cmpeq((vsx_v_i16)a, (vsx_v_i16)b); } +static inline v128_t vsx_i32x4_eq(v128_t a, v128_t b) +{ return (v128_t)vec_cmpeq((vsx_v_i32)a, (vsx_v_i32)b); } + +static inline v128_t vsx_i8x16_gt(v128_t a, v128_t b) +{ return (v128_t)vec_cmpgt((vsx_v_i8)a, (vsx_v_i8)b); } +static inline v128_t vsx_i32x4_gt(v128_t a, v128_t b) +{ return (v128_t)vec_cmpgt((vsx_v_i32)a, (vsx_v_i32)b); } +static inline v128_t vsx_i32x4_lt(v128_t a, v128_t b) +{ return (v128_t)vec_cmplt((vsx_v_i32)a, (vsx_v_i32)b); } +static inline v128_t vsx_i64x2_lt(v128_t a, v128_t b) +{ return (v128_t)vec_cmplt((vsx_v_i64)a, (vsx_v_i64)b); } + +static inline v128_t vsx_f32x4_ge(v128_t a, v128_t b) +{ return (v128_t)vec_cmpge((vsx_v_f32)a, (vsx_v_f32)b); } +static inline v128_t vsx_f32x4_lt(v128_t a, v128_t b) +{ return (v128_t)vec_cmplt((vsx_v_f32)a, (vsx_v_f32)b); } + +//--------------------------------------------------------------------------- +// float arithmetic and conversions +//--------------------------------------------------------------------------- +static inline v128_t vsx_f32x4_add(v128_t a, v128_t b) +{ return (v128_t)vec_add((vsx_v_f32)a, (vsx_v_f32)b); } +static inline v128_t vsx_f32x4_sub(v128_t a, v128_t b) +{ return (v128_t)vec_sub((vsx_v_f32)a, (vsx_v_f32)b); } +static inline v128_t vsx_f32x4_mul(v128_t a, v128_t b) +{ return (v128_t)vec_mul((vsx_v_f32)a, (vsx_v_f32)b); } + +// xvcvspsxws: truncating, saturating (NaN gives 0x80000000; the +// callers never pass NaN) +static inline v128_t vsx_i32x4_trunc_sat_f32x4(v128_t a) +{ return (v128_t)vec_cts((vsx_v_f32)a, 0); } +static inline v128_t vsx_f32x4_convert_i32x4(v128_t a) +{ return (v128_t)vec_ctf((vsx_v_i32)a, 0); } + +//--------------------------------------------------------------------------- +// widening +//--------------------------------------------------------------------------- +static inline v128_t vsx_i64x2_extend_low_i32x4(v128_t a) +{ + // vsx_v_i32 v = (vsx_v_i32)a; + // return (v128_t)__builtin_convertvector( + // __builtin_shufflevector(v, v, 0, 1), vsx_v_i64); + + // Unpacks and sign-extends elements 0 and 1 on Little Endian + return (v128_t)vec_unpackl((vsx_v_i32)a); +} +static inline v128_t vsx_i64x2_extend_high_i32x4(v128_t a) +{ + // vsx_v_i32 v = (vsx_v_i32)a; + // return (v128_t)__builtin_convertvector( + // __builtin_shufflevector(v, v, 2, 3), vsx_v_i64); + + // Unpacks and sign-extends elements 2 and 3 on Little Endian + return (v128_t)vec_unpackh((vsx_v_i32)a); +} + +//--------------------------------------------------------------------------- +// shuffles (immediate lane indices; 0..N-1 from a, N..2N-1 from b) +//--------------------------------------------------------------------------- +// #define vsx_i8x16_shuffle(a, b, c0,c1,c2,c3,c4,c5,c6,c7, +// c8,c9,c10,c11,c12,c13,c14,c15) +// ((v128_t)__builtin_shufflevector((vsx_v_u8)(a), (vsx_v_u8)(b), +// c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15)) +// #define vsx_i16x8_shuffle(a, b, c0,c1,c2,c3,c4,c5,c6,c7) +// ((v128_t)__builtin_shufflevector((vsx_v_i16)(a), (vsx_v_i16)(b), +// c0,c1,c2,c3,c4,c5,c6,c7)) +// #define vsx_i32x4_shuffle(a, b, c0,c1,c2,c3) +// ((v128_t)__builtin_shufflevector((vsx_v_i32)(a), (vsx_v_i32)(b), +// c0,c1,c2,c3)) +// #define vsx_i64x2_shuffle(a, b, c0,c1) +// ((v128_t)__builtin_shufflevector((vsx_v_i64)(a), (vsx_v_i64)(b), c0,c1)) + +// 8-bit Shuffle (Maps direct element indices to raw byte indices) +#define vsx_i8x16_shuffle(a, b, c0,c1,c2,c3,c4,c5,c6,c7, \ + c8,c9,c10,c11,c12,c13,c14,c15) \ + ((v128_t)vec_perm((vsx_v_u8)(a), (vsx_v_u8)(b), (vsx_v_u8){ \ + (c0), (c1), (c2), (c3), (c4), (c5), (c6), (c7), \ + (c8), (c9), (c10),(c11),(c12),(c13),(c14),(c15) \ + })) + +// 16-bit Shuffle (Multiplies element index by 2 to get byte offsets) +#define vsx_i16x8_shuffle(a, b, c0,c1,c2,c3,c4,c5,c6,c7) \ + ((v128_t)vec_perm((vsx_v_u8)(a), (vsx_v_u8)(b), (vsx_v_u8){ \ + (c0)*2, (c0)*2+1, (c1)*2, (c1)*2+1, \ + (c2)*2, (c2)*2+1, (c3)*2, (c3)*2+1, \ + (c4)*2, (c4)*2+1, (c5)*2, (c5)*2+1, \ + (c6)*2, (c6)*2+1, (c7)*2, (c7)*2+1 \ + })) + +// 32-bit Shuffle (Multiplies element index by 4 to get byte offsets) +#define vsx_i32x4_shuffle(a, b, c0,c1,c2,c3) \ + ((v128_t)vec_perm((vsx_v_u8)(a), (vsx_v_u8)(b), (vsx_v_u8){ \ + (c0)*4, (c0)*4+1, (c0)*4+2, (c0)*4+3, \ + (c1)*4, (c1)*4+1, (c1)*4+2, (c1)*4+3, \ + (c2)*4, (c2)*4+1, (c2)*4+2, (c2)*4+3, \ + (c3)*4, (c3)*4+1, (c3)*4+2, (c3)*4+3 \ + })) + +// 64-bit Shuffle (Multiplies element index by 8 to get byte offsets) +#define vsx_i64x2_shuffle(a, b, c0,c1) \ + ((v128_t)vec_perm((vsx_v_u8)(a), (vsx_v_u8)(b), (vsx_v_u8){ \ + (c0)*8, (c0)*8+1, (c0)*8+2, (c0)*8+3, \ + (c0)*8+4, (c0)*8+5, (c0)*8+6, (c0)*8+7, \ + (c1)*8, (c1)*8+1, (c1)*8+2, (c1)*8+3, \ + (c1)*8+4, (c1)*8+5, (c1)*8+6, (c1)*8+7 \ + })) + +//--------------------------------------------------------------------------- +// swizzle: runtime byte-table lookup; lanes with index > 15 give 0 +//--------------------------------------------------------------------------- +static inline v128_t vsx_i8x16_swizzle(v128_t a, v128_t idx) +{ + v128_t r = vec_perm(a, a, idx); + v128_t oob = (v128_t)vec_cmpgt((vsx_v_u8)idx, + vec_splats((unsigned char)15)); + return vec_andc(r, oob); +} + +//--------------------------------------------------------------------------- +// bitmask: MSB of each byte lane -> bit of result, lane 0 -> bit 0 +// (vbpermq gathers the 16 selected bits into bits 48..63 of the +// big-endian first doubleword, which is doubleword 1 on ppc64le) +//--------------------------------------------------------------------------- +static inline int vsx_i8x16_bitmask(v128_t a) +{ +#if defined(__POWER10_VECTOR__) + return (int)vec_extractm(a); // ISA 3.1 native movemask +#else + const vsx_v_u8 perm = { 120, 112, 104, 96, 88, 80, 72, 64, + 56, 48, 40, 32, 24, 16, 8, 0 }; + vsx_v_u64 r = (vsx_v_u64)vec_bperm(a, perm); + return (int)r[1]; +#endif +} + +#endif // OJPH_SIMD_VSX_H diff --git a/external/OpenJPH/src/core/transform/ojph_colour.cpp b/external/OpenJPH/src/core/transform/ojph_colour.cpp index ef147adeb..6e9d9709e 100644 --- a/external/OpenJPH/src/core/transform/ojph_colour.cpp +++ b/external/OpenJPH/src/core/transform/ojph_colour.cpp @@ -177,6 +177,25 @@ namespace ojph { #elif defined(OJPH_ARCH_ARM) + #elif defined(OJPH_ARCH_PPC64LE) + + if (get_cpu_ext_level() >= PPC_CPU_EXT_LEVEL_ARCH_3_00) + { + // 128-bit VSX kernels; see ojph_simd_vsx.h + rev_convert = vsx_rev_convert; + rev_convert_nlt_type3 = vsx_rev_convert_nlt_type3; + irv_convert_to_integer = vsx_irv_convert_to_integer; + irv_convert_to_float = vsx_irv_convert_to_float; + irv_convert_to_integer_nlt_type3 = + vsx_irv_convert_to_integer_nlt_type3; + irv_convert_to_float_nlt_type3 = + vsx_irv_convert_to_float_nlt_type3; + rct_forward = vsx_rct_forward; + rct_backward = vsx_rct_backward; + ict_forward = vsx_ict_forward; + ict_backward = vsx_ict_backward; + } + #endif // !(defined(OJPH_ARCH_X86_64) || defined(OJPH_ARCH_I386)) #endif // !OJPH_DISABLE_SIMD diff --git a/external/OpenJPH/src/core/transform/ojph_colour_local.h b/external/OpenJPH/src/core/transform/ojph_colour_local.h index a85bf8bd1..abe7eea0d 100644 --- a/external/OpenJPH/src/core/transform/ojph_colour_local.h +++ b/external/OpenJPH/src/core/transform/ojph_colour_local.h @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2019, Aous Naman +// Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -66,14 +66,14 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void gen_rev_convert( - const line_buf *src_line, const ui32 src_line_offset, - line_buf *dst_line, const ui32 dst_line_offset, + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width); ////////////////////////////////////////////////////////////////////////// void gen_rev_convert_nlt_type3( - const line_buf *src_line, const ui32 src_line_offset, - line_buf *dst_line, const ui32 dst_line_offset, + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width); ////////////////////////////////////////////////////////////////////////// @@ -158,14 +158,14 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void sse2_rev_convert( - const line_buf *src_line, const ui32 src_line_offset, - line_buf *dst_line, const ui32 dst_line_offset, + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width); ////////////////////////////////////////////////////////////////////////// void sse2_rev_convert_nlt_type3( - const line_buf *src_line, const ui32 src_line_offset, - line_buf *dst_line, const ui32 dst_line_offset, + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width); ////////////////////////////////////////////////////////////////////////// @@ -214,14 +214,14 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void avx2_rev_convert( - const line_buf *src_line, const ui32 src_line_offset, - line_buf *dst_line, const ui32 dst_line_offset, + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width); ////////////////////////////////////////////////////////////////////////// void avx2_rev_convert_nlt_type3( - const line_buf *src_line, const ui32 src_line_offset, - line_buf *dst_line, const ui32 dst_line_offset, + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width); ////////////////////////////////////////////////////////////////////////// @@ -257,7 +257,7 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// // // - // WASM Functions + // WASM Functions // // ////////////////////////////////////////////////////////////////////////// @@ -274,14 +274,14 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// void wasm_rev_convert( - const line_buf *src_line, const ui32 src_line_offset, - line_buf *dst_line, const ui32 dst_line_offset, + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width); ////////////////////////////////////////////////////////////////////////// void wasm_rev_convert_nlt_type3( - const line_buf *src_line, const ui32 src_line_offset, - line_buf *dst_line, const ui32 dst_line_offset, + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, si64 shift, ui32 width); ////////////////////////////////////////////////////////////////////////// @@ -312,6 +312,64 @@ namespace ojph { void wasm_ict_backward(const float *y, const float *cb, const float *cr, float *r, float *g, float *b, ui32 repeat); + ////////////////////////////////////////////////////////////////////////// + // + // + // VSX Functions + // + // + ////////////////////////////////////////////////////////////////////////// + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_convert_to_integer( + const line_buf *src_line, line_buf *dst_line, ui32 dst_line_offset, + ui32 bit_depth, bool is_signed, ui32 width); + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_convert_to_float( + const line_buf *src_line, ui32 src_line_offset, + line_buf *dst_line, ui32 bit_depth, bool is_signed, ui32 width); + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_convert( + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, + si64 shift, ui32 width); + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_convert_nlt_type3( + const line_buf *src_line, const ui32 src_line_offset, + line_buf *dst_line, const ui32 dst_line_offset, + si64 shift, ui32 width); + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_convert_to_integer_nlt_type3( + const line_buf *src_line, line_buf *dst_line, ui32 dst_line_offset, + ui32 bit_depth, bool is_signed, ui32 width); + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_convert_to_float_nlt_type3( + const line_buf *src_line, ui32 src_line_offset, + line_buf *dst_line, ui32 bit_depth, bool is_signed, ui32 width); + + ////////////////////////////////////////////////////////////////////////// + void vsx_rct_forward( + const line_buf *r, const line_buf *g, const line_buf *b, + line_buf *y, line_buf *cb, line_buf *cr, ui32 repeat); + + ////////////////////////////////////////////////////////////////////////// + void vsx_rct_backward( + const line_buf *y, const line_buf *cb, const line_buf *cr, + line_buf *r, line_buf *g, line_buf *b, ui32 repeat); + + ////////////////////////////////////////////////////////////////////////// + void vsx_ict_forward(const float *r, const float *g, const float *b, + float *y, float *cb, float *cr, ui32 repeat); + + ////////////////////////////////////////////////////////////////////////// + void vsx_ict_backward(const float *y, const float *cb, const float *cr, + float *r, float *g, float *b, ui32 repeat); + } } diff --git a/external/OpenJPH/src/core/transform/ojph_colour_vsx.cpp b/external/OpenJPH/src/core/transform/ojph_colour_vsx.cpp new file mode 100644 index 000000000..ef3582454 --- /dev/null +++ b/external/OpenJPH/src/core/transform/ojph_colour_vsx.cpp @@ -0,0 +1,668 @@ +//***************************************************************************/ +// This software is released under the 2-Clause BSD license, included +// below. +// +// Copyright (c) 2021, Aous Naman +// Copyright (c) 2021, Kakadu Software Pty Ltd, Australia +// Copyright (c) 2021, The University of New South Wales, Australia +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************/ +// This file is part of the OpenJPH software implementation. +// File: ojph_colour_vsx.cpp +// Author: Aous Naman +// Date: 9 February 2021 +//***************************************************************************/ + +#include +#include +#include "ojph_simd_vsx.h" + +#include "ojph_defs.h" +#include "ojph_mem.h" +#include "ojph_colour.h" +#include "ojph_colour_local.h" + +namespace ojph { + namespace local { + + ////////////////////////////////////////////////////////////////////////// + static inline + v128_t ojph_convert_float_to_i32(v128_t a) + { // We implement ojph_round, which is + // val + (val >= 0.0f ? 0.5f : -0.5f), where val is float; this is + // round to nearest with ties away from zero, which is exactly what + // xvrspi does. The instruction is used via inline asm because + // GCC's vec_round rounds ties to even. + vsx_v_f32 w; + __asm__("xvrspi %x0,%x1" : "=wa"(w) : "wa"((vsx_v_f32)a)); + return (v128_t)vec_cts(w, 0); // saturating convert to int32 + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_convert(const line_buf *src_line, + const ui32 src_line_offset, + line_buf *dst_line, + const ui32 dst_line_offset, + si64 shift, ui32 width) + { + if (src_line->flags & line_buf::LFT_32BIT) + { + if (dst_line->flags & line_buf::LFT_32BIT) + { + const si32 *sp = src_line->i32 + src_line_offset; + si32 *dp = dst_line->i32 + dst_line_offset; + v128_t sh = vsx_i32x4_splat((si32)shift); + for (int i = (width + 3) >> 2; i > 0; --i, sp+=4, dp+=4) + { + v128_t s = vsx_v128_load(sp); + s = vsx_i32x4_add(s, sh); + vsx_v128_store(dp, s); + } + } + else + { + const si32 *sp = src_line->i32 + src_line_offset; + si64 *dp = dst_line->i64 + dst_line_offset; + v128_t sh = vsx_i64x2_splat(shift); + for (int i = (width + 3) >> 2; i > 0; --i, sp+=4, dp+=4) + { + v128_t s, t; + s = vsx_v128_load(sp); + + t = vsx_i64x2_extend_low_i32x4(s); + t = vsx_i64x2_add(t, sh); + vsx_v128_store(dp, t); + + t = vsx_i64x2_extend_high_i32x4(s); + t = vsx_i64x2_add(t, sh); + vsx_v128_store(dp + 2, t); + } + } + } + else + { + assert(src_line->flags | line_buf::LFT_64BIT); + assert(dst_line->flags | line_buf::LFT_32BIT); + const si64 *sp = src_line->i64 + src_line_offset; + si32 *dp = dst_line->i32 + dst_line_offset; + v128_t sh = vsx_i64x2_splat(shift); + for (int i = (width + 3) >> 2; i > 0; --i, sp+=4, dp+=4) + { + v128_t s0, s1; + s0 = vsx_v128_load(sp); + s0 = vsx_i64x2_add(s0, sh); + s1 = vsx_v128_load(sp + 2); + s1 = vsx_i64x2_add(s1, sh); + s0 = vsx_i32x4_shuffle(s0, s1, 0, 2, 4 + 0, 4 + 2); + vsx_v128_store(dp, s0); + } + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_convert_nlt_type3(const line_buf *src_line, + const ui32 src_line_offset, + line_buf *dst_line, + const ui32 dst_line_offset, + si64 shift, ui32 width) + { + if (src_line->flags & line_buf::LFT_32BIT) + { + if (dst_line->flags & line_buf::LFT_32BIT) + { + const si32 *sp = src_line->i32 + src_line_offset; + si32 *dp = dst_line->i32 + dst_line_offset; + v128_t sh = vsx_i32x4_splat((si32)(-shift)); + v128_t zero = vsx_i32x4_splat(0); + for (int i = (width + 3) >> 2; i > 0; --i, sp += 4, dp += 4) + { + v128_t s = vsx_v128_load(sp); + v128_t c = vsx_i32x4_lt(s, zero); // 0xFFFFFFFF for -ve value + v128_t v_m_sh = vsx_i32x4_sub(sh, s); // - shift - value + v_m_sh = vsx_v128_and(c, v_m_sh); // keep only - shift - value + s = vsx_v128_andnot(s, c); // keep only +ve or 0 + s = vsx_v128_or(s, v_m_sh); // combine + vsx_v128_store(dp, s); + } + } + else + { + const si32 *sp = src_line->i32 + src_line_offset; + si64 *dp = dst_line->i64 + dst_line_offset; + v128_t sh = vsx_i64x2_splat(-shift); + v128_t zero = vsx_i32x4_splat(0); + for (int i = (width + 3) >> 2; i > 0; --i, sp += 4, dp += 4) + { + v128_t s, u, c, v_m_sh; + s = vsx_v128_load(sp); + + u = vsx_i64x2_extend_low_i32x4(s); + c = vsx_i64x2_lt(u, zero); // 64b -1 for -ve value + v_m_sh = vsx_i64x2_sub(sh, u); // - shift - value + v_m_sh = vsx_v128_and(c, v_m_sh); // keep only - shift - value + u = vsx_v128_andnot(u, c); // keep only +ve or 0 + u = vsx_v128_or(u, v_m_sh); // combine + + vsx_v128_store(dp, u); + + u = vsx_i64x2_extend_high_i32x4(s); + c = vsx_i64x2_lt(u, zero); // 64b -1 for -ve value + v_m_sh = vsx_i64x2_sub(sh, u); // - shift - value + v_m_sh = vsx_v128_and(c, v_m_sh); // keep only - shift - value + u = vsx_v128_andnot(u, c); // keep only +ve or 0 + u = vsx_v128_or(u, v_m_sh); // combine + + vsx_v128_store(dp + 2, u); + } + } + } + else + { + assert(src_line->flags | line_buf::LFT_64BIT); + assert(dst_line->flags | line_buf::LFT_32BIT); + const si64 *sp = src_line->i64 + src_line_offset; + si32 *dp = dst_line->i32 + dst_line_offset; + v128_t sh = vsx_i64x2_splat(-shift); + v128_t zero = vsx_i32x4_splat(0); + for (int i = (width + 3) >> 2; i > 0; --i, sp += 4, dp += 4) + { + // s for source, t for target, p for positive, n for negative, + // m for mask, and tm for temp + v128_t s, t0, t1, p, n, m, tm; + s = vsx_v128_load(sp); + m = vsx_i64x2_lt(s, zero); // 64b -1 for -ve value + tm = vsx_i64x2_sub(sh, s); // - shift - value + n = vsx_v128_and(m, tm); // -ve + p = vsx_v128_andnot(s, m); // +ve + t0 = vsx_v128_or(n, p); + + s = vsx_v128_load(sp + 2); + m = vsx_i64x2_lt(s, zero); // 64b -1 for -ve value + tm = vsx_i64x2_sub(sh, s); // - shift - value + n = vsx_v128_and(m, tm); // -ve + p = vsx_v128_andnot(s, m); // +ve + t1 = vsx_v128_or(n, p); + + t0 = vsx_i32x4_shuffle(t0, t1, 0, 2, 4 + 0, 4 + 2); + vsx_v128_store(dp, t0); + } + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_cnvrt_si32_to_float_shftd(const si32 *sp, float *dp, float mul, + ui32 width) + { + v128_t shift = vsx_f32x4_splat(0.5f); + v128_t m = vsx_f32x4_splat(mul); + for (ui32 i = (width + 3) >> 2; i > 0; --i, sp+=4, dp+=4) + { + v128_t t = vsx_v128_load(sp); + v128_t s = vsx_f32x4_convert_i32x4(t); + s = vsx_f32x4_mul(s, m); + s = vsx_f32x4_sub(s, shift); + vsx_v128_store(dp, s); + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_cnvrt_si32_to_float(const si32 *sp, float *dp, float mul, + ui32 width) + { + v128_t m = vsx_f32x4_splat(mul); + for (ui32 i = (width + 3) >> 2; i > 0; --i, sp+=4, dp+=4) + { + v128_t t = vsx_v128_load(sp); + v128_t s = vsx_f32x4_convert_i32x4(t); + s = vsx_f32x4_mul(s, m); + vsx_v128_store(dp, s); + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_cnvrt_float_to_si32_shftd(const float *sp, si32 *dp, float mul, + ui32 width) + { + const v128_t half = vsx_f32x4_splat(0.5f); + v128_t m = vsx_f32x4_splat(mul); + for (int i = (width + 3) >> 2; i > 0; --i, sp+=4, dp+=4) + { + v128_t t = vsx_v128_load(sp); + v128_t s = vsx_f32x4_add(t, half); + s = vsx_f32x4_mul(s, m); + s = vsx_f32x4_add(s, half); // + 0.5 and followed by floor next + vsx_v128_store(dp, ojph_convert_float_to_i32(s)); + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_cnvrt_float_to_si32(const float *sp, si32 *dp, float mul, + ui32 width) + { + const v128_t half = vsx_f32x4_splat(0.5f); + v128_t m = vsx_f32x4_splat(mul); + for (int i = (width + 3) >> 2; i > 0; --i, sp+=4, dp+=4) + { + v128_t t = vsx_v128_load(sp); + v128_t s = vsx_f32x4_mul(t, m); + s = vsx_f32x4_add(s, half); // + 0.5 and followed by floor next + vsx_v128_store(dp, ojph_convert_float_to_i32(s)); + } + } + + ////////////////////////////////////////////////////////////////////////// + static inline + v128_t ojph_vsx_i32x4_max_ge(v128_t a, v128_t b, v128_t x, v128_t y) + { + v128_t c = vsx_f32x4_ge(x, y); // 0xFFFFFFFF for x >= y + return (v128_t)vec_sel((vsx_v_u32)b, (vsx_v_u32)a, (vsx_v_u32)c); + } + + ////////////////////////////////////////////////////////////////////////// + static inline + v128_t ojph_vsx_i32x4_min_lt(v128_t a, v128_t b, v128_t x, v128_t y) + { + v128_t c = vsx_f32x4_lt(x, y); // 0xFFFFFFFF for x < y + return (v128_t)vec_sel((vsx_v_u32)b, (vsx_v_u32)a, (vsx_v_u32)c); + } + + ////////////////////////////////////////////////////////////////////////// + template + static inline + void local_vsx_irv_convert_to_integer(const line_buf *src_line, + line_buf *dst_line, ui32 dst_line_offset, + ui32 bit_depth, bool is_signed, ui32 width) + { + assert((src_line->flags & line_buf::LFT_32BIT) && + (src_line->flags & line_buf::LFT_INTEGER) == 0 && + (dst_line->flags & line_buf::LFT_32BIT) && + (dst_line->flags & line_buf::LFT_INTEGER)); + + assert(bit_depth <= 32); + const float* sp = src_line->f32; + si32* dp = dst_line->i32 + dst_line_offset; + // There is the possibility that converting to integer will + // exceed the dynamic range of 32bit integer; therefore, care must be + // exercised. + // We look if the floating point number is outside the half-closed + // interval [-0.5f, 0.5f). If so, we limit the resulting integer + // to the maximum/minimum that number supports. + si32 neg_limit = (si32)INT_MIN >> (32 - bit_depth); + v128_t mul = vsx_f32x4_splat((float)(1ull << bit_depth)); + v128_t fl_up_lim = vsx_f32x4_splat(-(float)neg_limit); // val < upper + v128_t fl_low_lim = vsx_f32x4_splat((float)neg_limit); // val >= lower + v128_t s32_up_lim = vsx_i32x4_splat(INT_MAX >> (32 - bit_depth)); + v128_t s32_low_lim = vsx_i32x4_splat(INT_MIN >> (32 - bit_depth)); + + if (is_signed) + { + const v128_t zero = vsx_f32x4_splat(0.0f); + v128_t bias = vsx_i32x4_splat(-(si32)((1ULL << (bit_depth - 1)) + 1)); + for (int i = (int)width; i > 0; i -= 4, sp += 4, dp += 4) { + v128_t t = vsx_v128_load(sp); + t = vsx_f32x4_mul(t, mul); + v128_t u = ojph_convert_float_to_i32(t); + u = ojph_vsx_i32x4_max_ge(u, s32_low_lim, t, fl_low_lim); + u = ojph_vsx_i32x4_min_lt(u, s32_up_lim, t, fl_up_lim); + if (NLT_TYPE3) + { + v128_t c = vsx_i32x4_gt(zero, u); // 0xFFFFFFFF for -ve value + v128_t neg = vsx_i32x4_sub(bias, u); // -bias -value + neg = vsx_v128_and(c, neg); // keep only - bias - value + u = vsx_v128_andnot(u, c); // keep only +ve or 0 + u = vsx_v128_or(neg, u); // combine + } + vsx_v128_store(dp, u); + } + } + else + { + v128_t ihalf = vsx_i32x4_splat((si32)(1ULL << (bit_depth - 1))); + for (int i = (int)width; i > 0; i -= 4, sp += 4, dp += 4) { + v128_t t = vsx_v128_load(sp); + t = vsx_f32x4_mul(t, mul); + v128_t u = ojph_convert_float_to_i32(t); + u = ojph_vsx_i32x4_max_ge(u, s32_low_lim, t, fl_low_lim); + u = ojph_vsx_i32x4_min_lt(u, s32_up_lim, t, fl_up_lim); + u = vsx_i32x4_add(u, ihalf); + vsx_v128_store(dp, u); + } + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_convert_to_integer(const line_buf *src_line, + line_buf *dst_line, ui32 dst_line_offset, + ui32 bit_depth, bool is_signed, ui32 width) + { + local_vsx_irv_convert_to_integer(src_line, dst_line, + dst_line_offset, bit_depth, is_signed, width); + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_convert_to_integer_nlt_type3(const line_buf *src_line, + line_buf *dst_line, ui32 dst_line_offset, + ui32 bit_depth, bool is_signed, ui32 width) + { + local_vsx_irv_convert_to_integer(src_line, dst_line, + dst_line_offset, bit_depth, is_signed, width); + } + + ////////////////////////////////////////////////////////////////////////// + template + static inline + void local_vsx_irv_convert_to_float(const line_buf *src_line, + ui32 src_line_offset, line_buf *dst_line, + ui32 bit_depth, bool is_signed, ui32 width) + { + assert((src_line->flags & line_buf::LFT_32BIT) && + (src_line->flags & line_buf::LFT_INTEGER) && + (dst_line->flags & line_buf::LFT_32BIT) && + (dst_line->flags & line_buf::LFT_INTEGER) == 0); + + assert(bit_depth <= 32); + v128_t mul = vsx_f32x4_splat((float)(1.0 / (double)(1ULL << bit_depth))); + + const si32* sp = src_line->i32 + src_line_offset; + float* dp = dst_line->f32; + if (is_signed) + { + v128_t zero = vsx_i32x4_splat(0); + v128_t bias = vsx_i32x4_splat(-(si32)((1ULL << (bit_depth - 1)) + 1)); + for (int i = (int)width; i > 0; i -= 4, sp += 4, dp += 4) { + v128_t t = vsx_v128_load(sp); + if (NLT_TYPE3) + { + v128_t c = vsx_i32x4_lt(t, zero); // 0xFFFFFFFF for -ve value + v128_t neg = vsx_i32x4_sub(bias, t); // - bias - value + neg = vsx_v128_and(c, neg); // keep only - bias - value + c = vsx_v128_andnot(t, c); // keep only +ve or 0 + t = vsx_v128_or(neg, c); // combine + } + v128_t v = vsx_f32x4_convert_i32x4(t); + v = vsx_f32x4_mul(v, mul); + vsx_v128_store(dp, v); + } + } + else + { + v128_t half = vsx_i32x4_splat((si32)(1ULL << (bit_depth - 1))); + for (int i = (int)width; i > 0; i -= 4, sp += 4, dp += 4) { + v128_t t = vsx_v128_load(sp); + t = vsx_i32x4_sub(t, half); + v128_t v = vsx_f32x4_convert_i32x4(t); + v = vsx_f32x4_mul(v, mul); + vsx_v128_store(dp, v); + } + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_convert_to_float(const line_buf *src_line, + ui32 src_line_offset, line_buf *dst_line, + ui32 bit_depth, bool is_signed, ui32 width) + { + local_vsx_irv_convert_to_float(src_line, src_line_offset, + dst_line, bit_depth, is_signed, width); + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_convert_to_float_nlt_type3(const line_buf *src_line, + ui32 src_line_offset, line_buf *dst_line, + ui32 bit_depth, bool is_signed, ui32 width) + { + local_vsx_irv_convert_to_float(src_line, src_line_offset, + dst_line, bit_depth, is_signed, width); + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rct_forward(const line_buf *r, + const line_buf *g, + const line_buf *b, + line_buf *y, line_buf *cb, line_buf *cr, + ui32 repeat) + { + assert((y->flags & line_buf::LFT_INTEGER) && + (cb->flags & line_buf::LFT_INTEGER) && + (cr->flags & line_buf::LFT_INTEGER) && + (r->flags & line_buf::LFT_INTEGER) && + (g->flags & line_buf::LFT_INTEGER) && + (b->flags & line_buf::LFT_INTEGER)); + + if (y->flags & line_buf::LFT_32BIT) + { + assert((y->flags & line_buf::LFT_32BIT) && + (cb->flags & line_buf::LFT_32BIT) && + (cr->flags & line_buf::LFT_32BIT) && + (r->flags & line_buf::LFT_32BIT) && + (g->flags & line_buf::LFT_32BIT) && + (b->flags & line_buf::LFT_32BIT)); + const si32 *rp = r->i32, * gp = g->i32, * bp = b->i32; + si32 *yp = y->i32, * cbp = cb->i32, * crp = cr->i32; + + for (int i = (repeat + 3) >> 2; i > 0; --i) + { + v128_t mr = vsx_v128_load(rp); + v128_t mg = vsx_v128_load(gp); + v128_t mb = vsx_v128_load(bp); + v128_t t = vsx_i32x4_add(mr, mb); + t = vsx_i32x4_add(t, vsx_i32x4_shl(mg, 1)); + vsx_v128_store(yp, vsx_i32x4_shr(t, 2)); + t = vsx_i32x4_sub(mb, mg); + vsx_v128_store(cbp, t); + t = vsx_i32x4_sub(mr, mg); + vsx_v128_store(crp, t); + + rp += 4; gp += 4; bp += 4; + yp += 4; cbp += 4; crp += 4; + } + } + else + { + assert((y->flags & line_buf::LFT_64BIT) && + (cb->flags & line_buf::LFT_64BIT) && + (cr->flags & line_buf::LFT_64BIT) && + (r->flags & line_buf::LFT_32BIT) && + (g->flags & line_buf::LFT_32BIT) && + (b->flags & line_buf::LFT_32BIT)); + const si32 *rp = r->i32, *gp = g->i32, *bp = b->i32; + si64 *yp = y->i64, *cbp = cb->i64, *crp = cr->i64; + for (int i = (repeat + 3) >> 2; i > 0; --i) + { + v128_t mr32 = vsx_v128_load(rp); + v128_t mg32 = vsx_v128_load(gp); + v128_t mb32 = vsx_v128_load(bp); + v128_t mr, mg, mb, t; + mr = vsx_i64x2_extend_low_i32x4(mr32); + mg = vsx_i64x2_extend_low_i32x4(mg32); + mb = vsx_i64x2_extend_low_i32x4(mb32); + + t = vsx_i64x2_add(mr, mb); + t = vsx_i64x2_add(t, vsx_i64x2_shl(mg, 1)); + vsx_v128_store(yp, vsx_i64x2_shr(t, 2)); + t = vsx_i64x2_sub(mb, mg); + vsx_v128_store(cbp, t); + t = vsx_i64x2_sub(mr, mg); + vsx_v128_store(crp, t); + + yp += 2; cbp += 2; crp += 2; + + mr = vsx_i64x2_extend_high_i32x4(mr32); + mg = vsx_i64x2_extend_high_i32x4(mg32); + mb = vsx_i64x2_extend_high_i32x4(mb32); + + t = vsx_i64x2_add(mr, mb); + t = vsx_i64x2_add(t, vsx_i64x2_shl(mg, 1)); + vsx_v128_store(yp, vsx_i64x2_shr(t, 2)); + t = vsx_i64x2_sub(mb, mg); + vsx_v128_store(cbp, t); + t = vsx_i64x2_sub(mr, mg); + vsx_v128_store(crp, t); + + rp += 4; gp += 4; bp += 4; + yp += 2; cbp += 2; crp += 2; + } + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rct_backward(const line_buf *y, + const line_buf *cb, + const line_buf *cr, + line_buf *r, line_buf *g, line_buf *b, + ui32 repeat) + { + assert((y->flags & line_buf::LFT_INTEGER) && + (cb->flags & line_buf::LFT_INTEGER) && + (cr->flags & line_buf::LFT_INTEGER) && + (r->flags & line_buf::LFT_INTEGER) && + (g->flags & line_buf::LFT_INTEGER) && + (b->flags & line_buf::LFT_INTEGER)); + + if (y->flags & line_buf::LFT_32BIT) + { + assert((y->flags & line_buf::LFT_32BIT) && + (cb->flags & line_buf::LFT_32BIT) && + (cr->flags & line_buf::LFT_32BIT) && + (r->flags & line_buf::LFT_32BIT) && + (g->flags & line_buf::LFT_32BIT) && + (b->flags & line_buf::LFT_32BIT)); + const si32 *yp = y->i32, *cbp = cb->i32, *crp = cr->i32; + si32 *rp = r->i32, *gp = g->i32, *bp = b->i32; + for (int i = (repeat + 3) >> 2; i > 0; --i) + { + v128_t my = vsx_v128_load(yp); + v128_t mcb = vsx_v128_load(cbp); + v128_t mcr = vsx_v128_load(crp); + + v128_t t = vsx_i32x4_add(mcb, mcr); + t = vsx_i32x4_sub(my, vsx_i32x4_shr(t, 2)); + vsx_v128_store(gp, t); + v128_t u = vsx_i32x4_add(mcb, t); + vsx_v128_store(bp, u); + u = vsx_i32x4_add(mcr, t); + vsx_v128_store(rp, u); + + yp += 4; cbp += 4; crp += 4; + rp += 4; gp += 4; bp += 4; + } + } + else + { + assert((y->flags & line_buf::LFT_64BIT) && + (cb->flags & line_buf::LFT_64BIT) && + (cr->flags & line_buf::LFT_64BIT) && + (r->flags & line_buf::LFT_32BIT) && + (g->flags & line_buf::LFT_32BIT) && + (b->flags & line_buf::LFT_32BIT)); + const si64 *yp = y->i64, *cbp = cb->i64, *crp = cr->i64; + si32 *rp = r->i32, *gp = g->i32, *bp = b->i32; + for (int i = (repeat + 3) >> 2; i > 0; --i) + { + v128_t my, mcb, mcr, tr0, tg0, tb0, tr1, tg1, tb1; + my = vsx_v128_load(yp); + mcb = vsx_v128_load(cbp); + mcr = vsx_v128_load(crp); + + tg0 = vsx_i64x2_add(mcb, mcr); + tg0 = vsx_i64x2_sub(my, vsx_i64x2_shr(tg0, 2)); + tb0 = vsx_i64x2_add(mcb, tg0); + tr0 = vsx_i64x2_add(mcr, tg0); + + yp += 2; cbp += 2; crp += 2; + + my = vsx_v128_load(yp); + mcb = vsx_v128_load(cbp); + mcr = vsx_v128_load(crp); + + tg1 = vsx_i64x2_add(mcb, mcr); + tg1 = vsx_i64x2_sub(my, vsx_i64x2_shr(tg1, 2)); + tb1 = vsx_i64x2_add(mcb, tg1); + tr1 = vsx_i64x2_add(mcr, tg1); + + tr0 = vsx_i32x4_shuffle(tr0, tr1, 0, 2, 4 + 0, 4 + 2); + tg0 = vsx_i32x4_shuffle(tg0, tg1, 0, 2, 4 + 0, 4 + 2); + tb0 = vsx_i32x4_shuffle(tb0, tb1, 0, 2, 4 + 0, 4 + 2); + + vsx_v128_store(rp, tr0); + vsx_v128_store(gp, tg0); + vsx_v128_store(bp, tb0); + + yp += 2; cbp += 2; crp += 2; + rp += 4; gp += 4; bp += 4; + } + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_ict_forward(const float *r, const float *g, const float *b, + float *y, float *cb, float *cr, ui32 repeat) + { + v128_t alpha_rf = vsx_f32x4_splat(CT_CNST::ALPHA_RF); + v128_t alpha_gf = vsx_f32x4_splat(CT_CNST::ALPHA_GF); + v128_t alpha_bf = vsx_f32x4_splat(CT_CNST::ALPHA_BF); + v128_t beta_cbf = vsx_f32x4_splat(CT_CNST::BETA_CbF); + v128_t beta_crf = vsx_f32x4_splat(CT_CNST::BETA_CrF); + for (ui32 i = (repeat + 3) >> 2; i > 0; --i) + { + v128_t mr = vsx_v128_load(r); + v128_t mb = vsx_v128_load(b); + v128_t my = vsx_f32x4_mul(alpha_rf, mr); + my = vsx_f32x4_add(my, vsx_f32x4_mul(alpha_gf, vsx_v128_load(g))); + my = vsx_f32x4_add(my, vsx_f32x4_mul(alpha_bf, mb)); + vsx_v128_store(y, my); + vsx_v128_store(cb, vsx_f32x4_mul(beta_cbf, vsx_f32x4_sub(mb, my))); + vsx_v128_store(cr, vsx_f32x4_mul(beta_crf, vsx_f32x4_sub(mr, my))); + + r += 4; g += 4; b += 4; + y += 4; cb += 4; cr += 4; + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_ict_backward(const float *y, const float *cb, const float *cr, + float *r, float *g, float *b, ui32 repeat) + { + v128_t gamma_cr2g = vsx_f32x4_splat(CT_CNST::GAMMA_CR2G); + v128_t gamma_cb2g = vsx_f32x4_splat(CT_CNST::GAMMA_CB2G); + v128_t gamma_cr2r = vsx_f32x4_splat(CT_CNST::GAMMA_CR2R); + v128_t gamma_cb2b = vsx_f32x4_splat(CT_CNST::GAMMA_CB2B); + for (ui32 i = (repeat + 3) >> 2; i > 0; --i) + { + v128_t my = vsx_v128_load(y); + v128_t mcr = vsx_v128_load(cr); + v128_t mcb = vsx_v128_load(cb); + v128_t mg = vsx_f32x4_sub(my, vsx_f32x4_mul(gamma_cr2g, mcr)); + vsx_v128_store(g, vsx_f32x4_sub(mg, vsx_f32x4_mul(gamma_cb2g, mcb))); + vsx_v128_store(r, vsx_f32x4_add(my, vsx_f32x4_mul(gamma_cr2r, mcr))); + vsx_v128_store(b, vsx_f32x4_add(my, vsx_f32x4_mul(gamma_cb2b, mcb))); + + y += 4; cb += 4; cr += 4; + r += 4; g += 4; b += 4; + } + } + + } +} diff --git a/external/OpenJPH/src/core/transform/ojph_transform.cpp b/external/OpenJPH/src/core/transform/ojph_transform.cpp index f67ea1b6a..e587b13eb 100644 --- a/external/OpenJPH/src/core/transform/ojph_transform.cpp +++ b/external/OpenJPH/src/core/transform/ojph_transform.cpp @@ -168,6 +168,21 @@ namespace ojph { #elif defined(OJPH_ARCH_ARM) + #elif defined(OJPH_ARCH_PPC64LE) + + if (get_cpu_ext_level() >= PPC_CPU_EXT_LEVEL_ARCH_3_00) + { + // 128-bit VSX kernels; see ojph_simd_vsx.h + rev_vert_step = vsx_rev_vert_step; + rev_horz_ana = vsx_rev_horz_ana; + rev_horz_syn = vsx_rev_horz_syn; + + irv_vert_step = vsx_irv_vert_step; + irv_vert_times_K = vsx_irv_vert_times_K; + irv_horz_ana = vsx_irv_horz_ana; + irv_horz_syn = vsx_irv_horz_syn; + } + #endif // !(defined(OJPH_ARCH_X86_64) || defined(OJPH_ARCH_I386)) #endif // !OJPH_DISABLE_SIMD diff --git a/external/OpenJPH/src/core/transform/ojph_transform_avx2.cpp b/external/OpenJPH/src/core/transform/ojph_transform_avx2.cpp index 51e59877b..16664c2cb 100644 --- a/external/OpenJPH/src/core/transform/ojph_transform_avx2.cpp +++ b/external/OpenJPH/src/core/transform/ojph_transform_avx2.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2019, Aous Naman +// Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -56,8 +56,8 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// // https://github.com/seung-lab/dijkstra3d/blob/master/libdivide.h - static inline - __m256i avx2_mm256_srai_epi64(__m256i a, int amt, __m256i m) + static inline + __m256i avx2_mm256_srai_epi64(__m256i a, int amt, __m256i m) { // note than m must be obtained using // __m256i m = _mm256_set1_epi64x(1ULL << (63 - amt)); @@ -85,7 +85,7 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - static inline + static inline void avx2_interleave32(float* dp, float* spl, float* sph, int width) { for (; width > 0; width -= 16, dp += 16, spl += 8, sph += 8) @@ -102,43 +102,50 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - static inline - void avx2_deinterleave64(double* dpl, double* dph, double* sp, int width) + static inline + void avx2_deinterleave64(void* dpl, void* dph, const void* sp, int width) { - for (; width > 0; width -= 8, sp += 8, dpl += 4, dph += 4) + for (; width > 0; width -= 8, + sp = (const char*)sp + 64, + dpl = (char*)dpl + 32, + dph = (char*)dph + 32) { - __m256d a = _mm256_load_pd(sp); - __m256d b = _mm256_load_pd(sp + 4); - __m256d c = _mm256_permute2f128_pd(a, b, (2 << 4) | (0)); - __m256d d = _mm256_permute2f128_pd(a, b, (3 << 4) | (1)); - __m256d e = _mm256_shuffle_pd(c, d, 0x0); - __m256d f = _mm256_shuffle_pd(c, d, 0xF); - _mm256_store_pd(dpl, e); - _mm256_store_pd(dph, f); + __m256i a = _mm256_load_si256((const __m256i*)sp); + __m256i b = _mm256_load_si256((const __m256i*)((const char*)sp + 32)); + __m256i c = _mm256_permute2f128_si256(a, b, (2 << 4) | (0)); + __m256i d = _mm256_permute2f128_si256(a, b, (3 << 4) | (1)); + __m256i e = _mm256_unpacklo_epi64(c, d); + __m256i f = _mm256_unpackhi_epi64(c, d); + _mm256_store_si256((__m256i*)dpl, e); + _mm256_store_si256((__m256i*)dph, f); } } ////////////////////////////////////////////////////////////////////////// - static inline - void avx2_interleave64(double* dp, double* spl, double* sph, int width) + static inline + void avx2_interleave64(void* dp, const void* spl, const void* sph, + int width) { - for (; width > 0; width -= 8, dp += 8, spl += 4, sph += 4) + for (; width > 0; width -= 8, + dp = (char*)dp + 64, + spl = (const char*)spl + 32, + sph = (const char*)sph + 32) { - __m256d a = _mm256_load_pd(spl); - __m256d b = _mm256_load_pd(sph); - __m256d c = _mm256_unpacklo_pd(a, b); - __m256d d = _mm256_unpackhi_pd(a, b); - __m256d e = _mm256_permute2f128_pd(c, d, (2 << 4) | (0)); - __m256d f = _mm256_permute2f128_pd(c, d, (3 << 4) | (1)); - _mm256_store_pd(dp, e); - _mm256_store_pd(dp + 4, f); + __m256i a = _mm256_load_si256((const __m256i*)spl); + __m256i b = _mm256_load_si256((const __m256i*)sph); + __m256i c = _mm256_unpacklo_epi64(a, b); + __m256i d = _mm256_unpackhi_epi64(a, b); + __m256i e = _mm256_permute2f128_si256(c, d, (2 << 4) | (0)); + __m256i f = _mm256_permute2f128_si256(c, d, (3 << 4) | (1)); + _mm256_store_si256((__m256i*)dp, e); + _mm256_store_si256((__m256i*)((char*)dp + 32), f); } } ///////////////////////////////////////////////////////////////////////// static - void avx2_rev_vert_step32(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx2_rev_vert_step32(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { const si32 a = s->rev.Aatk; @@ -149,7 +156,7 @@ namespace ojph { si32* dst = aug->i32; const si32* src1 = sig->i32, * src2 = other->i32; - // The general definition of the wavelet in Part 2 is slightly + // The general definition of the wavelet in Part 2 is slightly // different to part 2, although they are mathematically equivalent // here, we identify the simpler form from Part 1 and employ them if (a == 1) @@ -267,19 +274,19 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// static - void avx2_rev_vert_step64(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx2_rev_vert_step64(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { const si32 a = s->rev.Aatk; const si32 b = s->rev.Batk; const ui8 e = s->rev.Eatk; __m256i vb = _mm256_set1_epi64x(b); - __m256i ve = _mm256_set1_epi64x(1LL << (63 - e)); + __m256i ve = _mm256_set1_epi64x(1LL << (63 - e)); si64* dst = aug->i64; const si64* src1 = sig->i64, * src2 = other->i64; - // The general definition of the wavelet in Part 2 is slightly + // The general definition of the wavelet in Part 2 is slightly // different to part 2, although they are mathematically equivalent // here, we identify the simpler form from Part 1 and employ them if (a == 1) @@ -377,23 +384,23 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void avx2_rev_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx2_rev_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { - if (((sig != NULL) && (sig->flags & line_buf::LFT_32BIT)) || + if (((sig != NULL) && (sig->flags & line_buf::LFT_32BIT)) || ((aug != NULL) && (aug->flags & line_buf::LFT_32BIT)) || - ((other != NULL) && (other->flags & line_buf::LFT_32BIT))) + ((other != NULL) && (other->flags & line_buf::LFT_32BIT))) { assert((sig == NULL || sig->flags & line_buf::LFT_32BIT) && - (other == NULL || other->flags & line_buf::LFT_32BIT) && + (other == NULL || other->flags & line_buf::LFT_32BIT) && (aug == NULL || aug->flags & line_buf::LFT_32BIT)); avx2_rev_vert_step32(s, sig, other, aug, repeat, synthesis); } - else + else { assert((sig == NULL || sig->flags & line_buf::LFT_64BIT) && - (other == NULL || other->flags & line_buf::LFT_64BIT) && + (other == NULL || other->flags & line_buf::LFT_64BIT) && (aug == NULL || aug->flags & line_buf::LFT_64BIT)); avx2_rev_vert_step64(s, sig, other, aug, repeat, synthesis); } @@ -401,8 +408,8 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// static - void avx2_rev_horz_ana32(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void avx2_rev_horz_ana32(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) @@ -569,17 +576,17 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// static - void avx2_rev_horz_ana64(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void avx2_rev_horz_ana64(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) { // split src into ldst and hdst { - double* dpl = (double*)(even ? ldst->p : hdst->p); - double* dph = (double*)(even ? hdst->p : ldst->p); - double* sp = (double*)src->p; + void* dpl = even ? ldst->p : hdst->p; + void* dph = even ? hdst->p : ldst->p; + const void* sp = src->p; int w = (int)width; avx2_deinterleave64(dpl, dph, sp, w); } @@ -717,29 +724,29 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void avx2_rev_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void avx2_rev_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { - if (src->flags & line_buf::LFT_32BIT) + if (src->flags & line_buf::LFT_32BIT) { assert((ldst == NULL || ldst->flags & line_buf::LFT_32BIT) && (hdst == NULL || hdst->flags & line_buf::LFT_32BIT)); avx2_rev_horz_ana32(atk, ldst, hdst, src, width, even); } - else + else { assert((ldst == NULL || ldst->flags & line_buf::LFT_64BIT) && - (hdst == NULL || hdst->flags & line_buf::LFT_64BIT) && + (hdst == NULL || hdst->flags & line_buf::LFT_64BIT) && (src == NULL || src->flags & line_buf::LFT_64BIT)); avx2_rev_horz_ana64(atk, ldst, hdst, src, width, even); } - } - + } + ////////////////////////////////////////////////////////////////////////// static - void avx2_rev_horz_syn32(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void avx2_rev_horz_syn32(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -906,8 +913,8 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// static - void avx2_rev_horz_syn64(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void avx2_rev_horz_syn64(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -924,7 +931,7 @@ namespace ojph { const si32 b = s->rev.Batk; const ui8 e = s->rev.Eatk; __m256i vb = _mm256_set1_epi64x(b); - __m256i ve = _mm256_set1_epi64x(1LL << (63 - e)); + __m256i ve = _mm256_set1_epi64x(1LL << (63 - e)); // extension oth[-1] = oth[0]; @@ -1038,9 +1045,9 @@ namespace ojph { // combine both lsrc and hsrc into dst { - double* dp = (double*)dst->p; - double* spl = (double*)(even ? lsrc->p : hsrc->p); - double* sph = (double*)(even ? hsrc->p : lsrc->p); + void* dp = dst->p; + const void* spl = even ? lsrc->p : hsrc->p; + const void* sph = even ? hsrc->p : lsrc->p; int w = (int)width; avx2_interleave64(dp, spl, sph, w); } @@ -1051,23 +1058,23 @@ namespace ojph { else dst->i64[0] = hsrc->i64[0] >> 1; } - } + } ///////////////////////////////////////////////////////////////////////// - void avx2_rev_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void avx2_rev_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { - if (dst->flags & line_buf::LFT_32BIT) + if (dst->flags & line_buf::LFT_32BIT) { - assert((lsrc == NULL || lsrc->flags & line_buf::LFT_32BIT) && + assert((lsrc == NULL || lsrc->flags & line_buf::LFT_32BIT) && (hsrc == NULL || hsrc->flags & line_buf::LFT_32BIT)); avx2_rev_horz_syn32(atk, dst, lsrc, hsrc, width, even); } - else + else { assert((dst == NULL || dst->flags & line_buf::LFT_64BIT) && - (lsrc == NULL || lsrc->flags & line_buf::LFT_64BIT) && + (lsrc == NULL || lsrc->flags & line_buf::LFT_64BIT) && (hsrc == NULL || hsrc->flags & line_buf::LFT_64BIT)); avx2_rev_horz_syn64(atk, dst, lsrc, hsrc, width, even); } diff --git a/external/OpenJPH/src/core/transform/ojph_transform_avx512.cpp b/external/OpenJPH/src/core/transform/ojph_transform_avx512.cpp index 36276e485..f5853c904 100644 --- a/external/OpenJPH/src/core/transform/ojph_transform_avx512.cpp +++ b/external/OpenJPH/src/core/transform/ojph_transform_avx512.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2019-2024, Aous Naman +// Copyright (c) 2019-2024, Aous Naman // Copyright (c) 2019-2024, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019-2024, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -56,7 +56,7 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// // We split multiples of 32 followed by multiples of 16, because // we assume byte_alignment == 64 - static + static void avx512_deinterleave32(float* dpl, float* dph, float* sp, int width) { __m512i idx1 = _mm512_set_epi32( @@ -92,7 +92,7 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// // We split multiples of 32 followed by multiples of 16, because // we assume byte_alignment == 64 - static + static void avx512_interleave32(float* dp, float* spl, float* sph, int width) { __m512i idx1 = _mm512_set_epi32( @@ -128,7 +128,7 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// // We split multiples of 32 followed by multiples of 16, because // we assume byte_alignment == 64 - static void avx512_deinterleave64(double* dpl, double* dph, double* sp, + static void avx512_deinterleave64(void* dpl, void* dph, const void* sp, int width) { __m512i idx1 = _mm512_set_epi64( @@ -137,33 +137,39 @@ namespace ojph { __m512i idx2 = _mm512_set_epi64( 0x0F, 0x0D, 0x0B, 0x09, 0x07, 0x05, 0x03, 0x01 ); - for (; width > 8; width -= 16, sp += 16, dpl += 8, dph += 8) + for (; width > 8; width -= 16, + sp = (const char*)sp + 128, + dpl = (char*)dpl + 64, + dph = (char*)dph + 64) { - __m512d a = _mm512_load_pd(sp); - __m512d b = _mm512_load_pd(sp + 16); - __m512d c = _mm512_permutex2var_pd(a, idx1, b); - __m512d d = _mm512_permutex2var_pd(a, idx2, b); - _mm512_store_pd(dpl, c); - _mm512_store_pd(dph, d); + __m512i a = _mm512_load_si512(sp); + __m512i b = _mm512_load_si512((const char*)sp + 64); + __m512i c = _mm512_permutex2var_epi64(a, idx1, b); + __m512i d = _mm512_permutex2var_epi64(a, idx2, b); + _mm512_store_si512(dpl, c); + _mm512_store_si512(dph, d); } - for (; width > 0; width -= 8, sp += 8, dpl += 4, dph += 4) + for (; width > 0; width -= 8, + sp = (const char*)sp + 64, + dpl = (char*)dpl + 32, + dph = (char*)dph + 32) { - __m256d a = _mm256_load_pd(sp); - __m256d b = _mm256_load_pd(sp + 4); - __m256d c = _mm256_permute2f128_pd(a, b, (2 << 4) | (0)); - __m256d d = _mm256_permute2f128_pd(a, b, (3 << 4) | (1)); - __m256d e = _mm256_shuffle_pd(c, d, 0x0); - __m256d f = _mm256_shuffle_pd(c, d, 0xF); - _mm256_store_pd(dpl, e); - _mm256_store_pd(dph, f); + __m256i a = _mm256_load_si256((const __m256i*)sp); + __m256i b = _mm256_load_si256((const __m256i*)((const char*)sp + 32)); + __m256i c = _mm256_permute2f128_si256(a, b, (2 << 4) | (0)); + __m256i d = _mm256_permute2f128_si256(a, b, (3 << 4) | (1)); + __m256i e = _mm256_unpacklo_epi64(c, d); + __m256i f = _mm256_unpackhi_epi64(c, d); + _mm256_store_si256((__m256i*)dpl, e); + _mm256_store_si256((__m256i*)dph, f); } } ////////////////////////////////////////////////////////////////////////// // We split multiples of 32 followed by multiples of 16, because // we assume byte_alignment == 64 - static void avx512_interleave64(double* dp, double* spl, double* sph, - int width) + static void avx512_interleave64(void* dp, const void* spl, + const void* sph, int width) { __m512i idx1 = _mm512_set_epi64( 0xB, 0x3, 0xA, 0x2, 0x9, 0x1, 0x8, 0x0 @@ -171,25 +177,31 @@ namespace ojph { __m512i idx2 = _mm512_set_epi64( 0xF, 0x7, 0xE, 0x6, 0xD, 0x5, 0xC, 0x4 ); - for (; width > 8; width -= 16, dp += 16, spl += 8, sph += 8) + for (; width > 8; width -= 16, + dp = (char*)dp + 128, + spl = (const char*)spl + 64, + sph = (const char*)sph + 64) { - __m512d a = _mm512_load_pd(spl); - __m512d b = _mm512_load_pd(sph); - __m512d c = _mm512_permutex2var_pd(a, idx1, b); - __m512d d = _mm512_permutex2var_pd(a, idx2, b); - _mm512_store_pd(dp, c); - _mm512_store_pd(dp + 16, d); + __m512i a = _mm512_load_si512(spl); + __m512i b = _mm512_load_si512(sph); + __m512i c = _mm512_permutex2var_epi64(a, idx1, b); + __m512i d = _mm512_permutex2var_epi64(a, idx2, b); + _mm512_store_si512(dp, c); + _mm512_store_si512((char*)dp + 64, d); } - for (; width > 0; width -= 8, dp += 8, spl += 4, sph += 4) + for (; width > 0; width -= 8, + dp = (char*)dp + 64, + spl = (const char*)spl + 32, + sph = (const char*)sph + 32) { - __m256d a = _mm256_load_pd(spl); - __m256d b = _mm256_load_pd(sph); - __m256d c = _mm256_unpacklo_pd(a, b); - __m256d d = _mm256_unpackhi_pd(a, b); - __m256d e = _mm256_permute2f128_pd(c, d, (2 << 4) | (0)); - __m256d f = _mm256_permute2f128_pd(c, d, (3 << 4) | (1)); - _mm256_store_pd(dp, e); - _mm256_store_pd(dp + 4, f); + __m256i a = _mm256_load_si256((const __m256i*)spl); + __m256i b = _mm256_load_si256((const __m256i*)sph); + __m256i c = _mm256_unpacklo_epi64(a, b); + __m256i d = _mm256_unpackhi_epi64(a, b); + __m256i e = _mm256_permute2f128_si256(c, d, (2 << 4) | (0)); + __m256i f = _mm256_permute2f128_si256(c, d, (3 << 4) | (1)); + _mm256_store_si256((__m256i*)dp, e); + _mm256_store_si256((__m256i*)((char*)dp + 32), f); } } @@ -205,8 +217,8 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - void avx512_irv_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx512_irv_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { float a = s->irv.Aatk; @@ -235,8 +247,8 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void avx512_irv_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void avx512_irv_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) @@ -311,10 +323,10 @@ namespace ojph { hdst->f32[0] = src->f32[0] * 2.0f; } } - + ////////////////////////////////////////////////////////////////////////// - void avx512_irv_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void avx512_irv_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -382,7 +394,7 @@ namespace ojph { float* sph = even ? hsrc->f32 : lsrc->f32; int w = (int)width; avx512_interleave32(dp, spl, sph, w); - } + } } else { if (even) @@ -394,8 +406,8 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// - void avx512_rev_vert_step32(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx512_rev_vert_step32(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { const si32 a = s->rev.Aatk; @@ -406,7 +418,7 @@ namespace ojph { si32* dst = aug->i32; const si32* src1 = sig->i32, * src2 = other->i32; - // The general definition of the wavelet in Part 2 is slightly + // The general definition of the wavelet in Part 2 is slightly // different to part 2, although they are mathematically equivalent // here, we identify the simpler form from Part 1 and employ them if (a == 1) @@ -523,8 +535,8 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void avx512_rev_vert_step64(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx512_rev_vert_step64(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { const si32 a = s->rev.Aatk; @@ -534,7 +546,7 @@ namespace ojph { si64* dst = aug->i64; const si64* src1 = sig->i64, * src2 = other->i64; - // The general definition of the wavelet in Part 2 is slightly + // The general definition of the wavelet in Part 2 is slightly // different to part 2, although they are mathematically equivalent // here, we identify the simpler form from Part 1 and employ them if (a == 1) @@ -619,7 +631,7 @@ namespace ojph { _mm512_store_si512((__m512i*)dst, d); } } - else { + else { // general case // 64bit multiplication is not supported in AVX512F + AVX512CD; // in particular, _mm256_mullo_epi64. @@ -665,31 +677,31 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void avx512_rev_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx512_rev_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { - if (((sig != NULL) && (sig->flags & line_buf::LFT_32BIT)) || + if (((sig != NULL) && (sig->flags & line_buf::LFT_32BIT)) || ((aug != NULL) && (aug->flags & line_buf::LFT_32BIT)) || - ((other != NULL) && (other->flags & line_buf::LFT_32BIT))) + ((other != NULL) && (other->flags & line_buf::LFT_32BIT))) { assert((sig == NULL || sig->flags & line_buf::LFT_32BIT) && - (other == NULL || other->flags & line_buf::LFT_32BIT) && + (other == NULL || other->flags & line_buf::LFT_32BIT) && (aug == NULL || aug->flags & line_buf::LFT_32BIT)); avx512_rev_vert_step32(s, sig, other, aug, repeat, synthesis); } - else + else { assert((sig == NULL || sig->flags & line_buf::LFT_64BIT) && - (other == NULL || other->flags & line_buf::LFT_64BIT) && + (other == NULL || other->flags & line_buf::LFT_64BIT) && (aug == NULL || aug->flags & line_buf::LFT_64BIT)); avx512_rev_vert_step64(s, sig, other, aug, repeat, synthesis); } } ///////////////////////////////////////////////////////////////////////// - void avx512_rev_horz_ana32(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void avx512_rev_horz_ana32(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) @@ -701,7 +713,7 @@ namespace ojph { float* sp = src->f32; int w = (int)width; avx512_deinterleave32(dpl, dph, sp, w); - } + } si32* hp = hdst->i32, * lp = ldst->i32; ui32 l_width = (width + (even ? 1 : 0)) >> 1; // low pass @@ -853,22 +865,22 @@ namespace ojph { hdst->i32[0] = src->i32[0] << 1; } } - + ///////////////////////////////////////////////////////////////////////// - void avx512_rev_horz_ana64(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void avx512_rev_horz_ana64(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) { // split src into ldst and hdst { - double* dpl = (double*)(even ? ldst->p : hdst->p); - double* dph = (double*)(even ? hdst->p : ldst->p); - double* sp = (double*)(src->p); + void* dpl = even ? ldst->p : hdst->p; + void* dph = even ? hdst->p : ldst->p; + const void* sp = src->p; int w = (int)width; avx512_deinterleave64(dpl, dph, sp, w); - } + } si64* hp = hdst->i64, * lp = ldst->i64; ui32 l_width = (width + (even ? 1 : 0)) >> 1; // low pass @@ -975,7 +987,7 @@ namespace ojph { _mm512_store_si512((__m512i*)dp, d); } } - else + else { // general case // 64bit multiplication is not supported in AVX512F + AVX512CD; @@ -1036,28 +1048,28 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void avx512_rev_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void avx512_rev_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { - if (src->flags & line_buf::LFT_32BIT) + if (src->flags & line_buf::LFT_32BIT) { assert((ldst == NULL || ldst->flags & line_buf::LFT_32BIT) && (hdst == NULL || hdst->flags & line_buf::LFT_32BIT)); avx512_rev_horz_ana32(atk, ldst, hdst, src, width, even); } - else + else { assert((ldst == NULL || ldst->flags & line_buf::LFT_64BIT) && - (hdst == NULL || hdst->flags & line_buf::LFT_64BIT) && + (hdst == NULL || hdst->flags & line_buf::LFT_64BIT) && (src == NULL || src->flags & line_buf::LFT_64BIT)); avx512_rev_horz_ana64(atk, ldst, hdst, src, width, even); } - } + } ////////////////////////////////////////////////////////////////////////// - void avx512_rev_horz_syn32(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void avx512_rev_horz_syn32(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -1212,7 +1224,7 @@ namespace ojph { float* sph = even ? hsrc->f32 : lsrc->f32; int w = (int)width; avx512_interleave32(dp, spl, sph, w); - } + } } else { if (even) @@ -1223,8 +1235,8 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - void avx512_rev_horz_syn64(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void avx512_rev_horz_syn64(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -1334,11 +1346,11 @@ namespace ojph { _mm512_store_si512((__m512i*)dp, d); } } - else + else { // general case // 64bit multiplication is not supported in AVX512F + AVX512CD; - // in particular, _mm256_mullo_epi64. + // in particular, _mm256_mullo_epi64. if (ev) for (ui32 i = aug_width; i > 0; --i, sp++, dp++) *dp -= (b + a * (sp[-1] + sp[0])) >> e; @@ -1388,12 +1400,12 @@ namespace ojph { // combine both lsrc and hsrc into dst { - double* dp = (double*)(dst->p); - double* spl = (double*)(even ? lsrc->p : hsrc->p); - double* sph = (double*)(even ? hsrc->p : lsrc->p); + void* dp = dst->p; + const void* spl = even ? lsrc->p : hsrc->p; + const void* sph = even ? hsrc->p : lsrc->p; int w = (int)width; avx512_interleave64(dp, spl, sph, w); - } + } } else { if (even) @@ -1404,20 +1416,20 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void avx512_rev_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void avx512_rev_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { - if (dst->flags & line_buf::LFT_32BIT) + if (dst->flags & line_buf::LFT_32BIT) { - assert((lsrc == NULL || lsrc->flags & line_buf::LFT_32BIT) && + assert((lsrc == NULL || lsrc->flags & line_buf::LFT_32BIT) && (hsrc == NULL || hsrc->flags & line_buf::LFT_32BIT)); avx512_rev_horz_syn32(atk, dst, lsrc, hsrc, width, even); } - else + else { assert((dst == NULL || dst->flags & line_buf::LFT_64BIT) && - (lsrc == NULL || lsrc->flags & line_buf::LFT_64BIT) && + (lsrc == NULL || lsrc->flags & line_buf::LFT_64BIT) && (hsrc == NULL || hsrc->flags & line_buf::LFT_64BIT)); avx512_rev_horz_syn64(atk, dst, lsrc, hsrc, width, even); } diff --git a/external/OpenJPH/src/core/transform/ojph_transform_local.h b/external/OpenJPH/src/core/transform/ojph_transform_local.h index acf9ee6da..0eb6a120c 100644 --- a/external/OpenJPH/src/core/transform/ojph_transform_local.h +++ b/external/OpenJPH/src/core/transform/ojph_transform_local.h @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2019, Aous Naman +// Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -63,21 +63,21 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////// - void gen_irv_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void gen_irv_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// void gen_irv_vert_times_K(float K, const line_buf* aug, ui32 repeat); ///////////////////////////////////////////////////////////////////////// - void gen_irv_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void gen_irv_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// - void gen_irv_horz_syn(const param_atk *atk, const line_buf* dst, - const line_buf *lsrc, const line_buf *hsrc, + void gen_irv_horz_syn(const param_atk *atk, const line_buf* dst, + const line_buf *lsrc, const line_buf *hsrc, ui32 width, bool even); ////////////////////////////////////////////////////////////////////////// @@ -85,18 +85,18 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////// - void gen_rev_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void gen_rev_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// - void gen_rev_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void gen_rev_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// - void gen_rev_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void gen_rev_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even); ////////////////////////////////////////////////////////////////////////// @@ -112,8 +112,8 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////// - void sse_irv_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void sse_irv_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// @@ -121,12 +121,12 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// void sse_irv_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// void sse_irv_horz_syn(const param_atk *atk, const line_buf* dst, - const line_buf *lsrc, const line_buf *hsrc, + const line_buf *lsrc, const line_buf *hsrc, ui32 width, bool even); ////////////////////////////////////////////////////////////////////////// @@ -142,18 +142,18 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////// - void sse2_rev_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void sse2_rev_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// void sse2_rev_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// void sse2_rev_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even); @@ -170,8 +170,8 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////// - void avx_irv_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx_irv_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// @@ -179,12 +179,12 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// void avx_irv_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// void avx_irv_horz_syn(const param_atk *atk, const line_buf* dst, - const line_buf *lsrc, const line_buf *hsrc, + const line_buf *lsrc, const line_buf *hsrc, ui32 width, bool even); ////////////////////////////////////////////////////////////////////////// @@ -200,18 +200,18 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////// - void avx2_rev_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx2_rev_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// void avx2_rev_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// void avx2_rev_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even); ////////////////////////////////////////////////////////////////////////// @@ -227,8 +227,8 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////// - void avx512_irv_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void avx512_irv_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// @@ -236,12 +236,12 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// void avx512_irv_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// void avx512_irv_horz_syn(const param_atk *atk, const line_buf* dst, - const line_buf *lsrc, const line_buf *hsrc, + const line_buf *lsrc, const line_buf *hsrc, ui32 width, bool even); @@ -251,17 +251,17 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// void avx512_rev_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// void avx512_rev_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// void avx512_rev_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even); ////////////////////////////////////////////////////////////////////////// @@ -277,8 +277,8 @@ namespace ojph { ////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////// - void wasm_irv_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void wasm_irv_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// @@ -286,12 +286,12 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// void wasm_irv_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// void wasm_irv_horz_syn(const param_atk *atk, const line_buf* dst, - const line_buf *lsrc, const line_buf *hsrc, + const line_buf *lsrc, const line_buf *hsrc, ui32 width, bool even); ////////////////////////////////////////////////////////////////////////// @@ -300,17 +300,66 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// void wasm_rev_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis); ///////////////////////////////////////////////////////////////////////// void wasm_rev_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + const line_buf* hdst, const line_buf* src, ui32 width, bool even); ///////////////////////////////////////////////////////////////////////// void wasm_rev_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + const line_buf* lsrc, const line_buf* hsrc, + ui32 width, bool even); + + ////////////////////////////////////////////////////////////////////////// + // + // + // VSX Functions + // + // + ////////////////////////////////////////////////////////////////////////// + + ////////////////////////////////////////////////////////////////////////// + // Irreversible functions + ////////////////////////////////////////////////////////////////////////// + + ///////////////////////////////////////////////////////////////////////// + void vsx_irv_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, + ui32 repeat, bool synthesis); + + ///////////////////////////////////////////////////////////////////////// + void vsx_irv_vert_times_K(float K, const line_buf* aug, ui32 repeat); + + ///////////////////////////////////////////////////////////////////////// + void vsx_irv_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, + ui32 width, bool even); + + ///////////////////////////////////////////////////////////////////////// + void vsx_irv_horz_syn(const param_atk *atk, const line_buf* dst, + const line_buf *lsrc, const line_buf *hsrc, + ui32 width, bool even); + + ////////////////////////////////////////////////////////////////////////// + // Reversible functions + ////////////////////////////////////////////////////////////////////////// + + ///////////////////////////////////////////////////////////////////////// + void vsx_rev_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, + ui32 repeat, bool synthesis); + + ///////////////////////////////////////////////////////////////////////// + void vsx_rev_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, + ui32 width, bool even); + + ///////////////////////////////////////////////////////////////////////// + void vsx_rev_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even); } } diff --git a/external/OpenJPH/src/core/transform/ojph_transform_sse2.cpp b/external/OpenJPH/src/core/transform/ojph_transform_sse2.cpp index 54770ff60..d2dadfe6a 100644 --- a/external/OpenJPH/src/core/transform/ojph_transform_sse2.cpp +++ b/external/OpenJPH/src/core/transform/ojph_transform_sse2.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2019, Aous Naman +// Copyright (c) 2019, Aous Naman // Copyright (c) 2019, Kakadu Software Pty Ltd, Australia // Copyright (c) 2019, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -56,7 +56,7 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// // https://github.com/seung-lab/dijkstra3d/blob/master/libdivide.h - static inline __m128i sse2_mm_srai_epi64(__m128i a, int amt, __m128i m) + static inline __m128i sse2_mm_srai_epi64(__m128i a, int amt, __m128i m) { // note than m must be obtained using // __m128i m = _mm_set1_epi64x(1ULL << (63 - amt)); @@ -97,39 +97,46 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - static inline - void sse2_deinterleave64(double* dpl, double* dph, double* sp, int width) + static inline + void sse2_deinterleave64(void* dpl, void* dph, const void* sp, int width) { - for (; width > 0; width -= 4, sp += 4, dpl += 2, dph += 2) + for (; width > 0; width -= 4, + sp = (const char*)sp + 32, + dpl = (char*)dpl + 16, + dph = (char*)dph + 16) { - __m128d a = _mm_load_pd(sp); - __m128d b = _mm_load_pd(sp + 2); - __m128d c = _mm_shuffle_pd(a, b, 0); - __m128d d = _mm_shuffle_pd(a, b, 3); - _mm_store_pd(dpl, c); - _mm_store_pd(dph, d); + __m128i a = _mm_load_si128((const __m128i*)sp); + __m128i b = _mm_load_si128((const __m128i*)((const char*)sp + 16)); + __m128i c = _mm_unpacklo_epi64(a, b); + __m128i d = _mm_unpackhi_epi64(a, b); + _mm_store_si128((__m128i*)dpl, c); + _mm_store_si128((__m128i*)dph, d); } - } + } ////////////////////////////////////////////////////////////////////////// - static inline - void sse2_interleave64(double* dp, double* spl, double* sph, int width) + static inline + void sse2_interleave64(void* dp, const void* spl, const void* sph, + int width) { - for (; width > 0; width -= 4, dp += 4, spl += 2, sph += 2) + for (; width > 0; width -= 4, + dp = (char*)dp + 32, + spl = (const char*)spl + 16, + sph = (const char*)sph + 16) { - __m128d a = _mm_load_pd(spl); - __m128d b = _mm_load_pd(sph); - __m128d c = _mm_unpacklo_pd(a, b); - __m128d d = _mm_unpackhi_pd(a, b); - _mm_store_pd(dp, c); - _mm_store_pd(dp + 2, d); + __m128i a = _mm_load_si128((const __m128i*)spl); + __m128i b = _mm_load_si128((const __m128i*)sph); + __m128i c = _mm_unpacklo_epi64(a, b); + __m128i d = _mm_unpackhi_epi64(a, b); + _mm_store_si128((__m128i*)dp, c); + _mm_store_si128((__m128i*)((char*)dp + 16), d); } } ///////////////////////////////////////////////////////////////////////// static - void sse2_rev_vert_step32(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void sse2_rev_vert_step32(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { const si32 a = s->rev.Aatk; @@ -139,7 +146,7 @@ namespace ojph { si32* dst = aug->i32; const si32* src1 = sig->i32, * src2 = other->i32; - // The general definition of the wavelet in Part 2 is slightly + // The general definition of the wavelet in Part 2 is slightly // different to part 2, although they are mathematically equivalent // here, we identify the simpler form from Part 1 and employ them if (a == 1) @@ -239,8 +246,8 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// static - void sse2_rev_vert_step64(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void sse2_rev_vert_step64(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { const si64 a = s->rev.Aatk; @@ -251,7 +258,7 @@ namespace ojph { si64* dst = aug->i64; const si64* src1 = sig->i64, * src2 = other->i64; - // The general definition of the wavelet in Part 2 is slightly + // The general definition of the wavelet in Part 2 is slightly // different to part 2, although they are mathematically equivalent // here, we identify the simpler form from Part 1 and employ them if (a == 1) @@ -348,23 +355,23 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void sse2_rev_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void sse2_rev_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { - if (((sig != NULL) && (sig->flags & line_buf::LFT_32BIT)) || + if (((sig != NULL) && (sig->flags & line_buf::LFT_32BIT)) || ((aug != NULL) && (aug->flags & line_buf::LFT_32BIT)) || - ((other != NULL) && (other->flags & line_buf::LFT_32BIT))) + ((other != NULL) && (other->flags & line_buf::LFT_32BIT))) { assert((sig == NULL || sig->flags & line_buf::LFT_32BIT) && - (other == NULL || other->flags & line_buf::LFT_32BIT) && + (other == NULL || other->flags & line_buf::LFT_32BIT) && (aug == NULL || aug->flags & line_buf::LFT_32BIT)); sse2_rev_vert_step32(s, sig, other, aug, repeat, synthesis); } - else + else { assert((sig == NULL || sig->flags & line_buf::LFT_64BIT) && - (other == NULL || other->flags & line_buf::LFT_64BIT) && + (other == NULL || other->flags & line_buf::LFT_64BIT) && (aug == NULL || aug->flags & line_buf::LFT_64BIT)); sse2_rev_vert_step64(s, sig, other, aug, repeat, synthesis); } @@ -372,8 +379,8 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// static - void sse2_rev_horz_ana32(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void sse2_rev_horz_ana32(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) @@ -519,17 +526,17 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// static - void sse2_rev_horz_ana64(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void sse2_rev_horz_ana64(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) { // split src into ldst and hdst { - double* dpl = (double*)(even ? ldst->p : hdst->p); - double* dph = (double*)(even ? hdst->p : ldst->p); - double* sp = (double*)src->p; + void* dpl = even ? ldst->p : hdst->p; + void* dph = even ? hdst->p : ldst->p; + const void* sp = src->p; int w = (int)width; sse2_deinterleave64(dpl, dph, sp, w); } @@ -666,28 +673,28 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void sse2_rev_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void sse2_rev_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { - if (src->flags & line_buf::LFT_32BIT) + if (src->flags & line_buf::LFT_32BIT) { assert((ldst == NULL || ldst->flags & line_buf::LFT_32BIT) && (hdst == NULL || hdst->flags & line_buf::LFT_32BIT)); sse2_rev_horz_ana32(atk, ldst, hdst, src, width, even); } - else + else { assert((ldst == NULL || ldst->flags & line_buf::LFT_64BIT) && - (hdst == NULL || hdst->flags & line_buf::LFT_64BIT) && + (hdst == NULL || hdst->flags & line_buf::LFT_64BIT) && (src == NULL || src->flags & line_buf::LFT_64BIT)); sse2_rev_horz_ana64(atk, ldst, hdst, src, width, even); } - } - + } + ////////////////////////////////////////////////////////////////////////// - void sse2_rev_horz_syn32(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void sse2_rev_horz_syn32(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -834,8 +841,8 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - void sse2_rev_horz_syn64(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void sse2_rev_horz_syn64(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -965,9 +972,9 @@ namespace ojph { // combine both lsrc and hsrc into dst { - double* dp = (double*)dst->p; - double* spl = (double*)(even ? lsrc->p : hsrc->p); - double* sph = (double*)(even ? hsrc->p : lsrc->p); + void* dp = dst->p; + const void* spl = even ? lsrc->p : hsrc->p; + const void* sph = even ? hsrc->p : lsrc->p; int w = (int)width; sse2_interleave64(dp, spl, sph, w); } @@ -981,24 +988,24 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void sse2_rev_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void sse2_rev_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { - if (dst->flags & line_buf::LFT_32BIT) + if (dst->flags & line_buf::LFT_32BIT) { - assert((lsrc == NULL || lsrc->flags & line_buf::LFT_32BIT) && + assert((lsrc == NULL || lsrc->flags & line_buf::LFT_32BIT) && (hsrc == NULL || hsrc->flags & line_buf::LFT_32BIT)); sse2_rev_horz_syn32(atk, dst, lsrc, hsrc, width, even); } - else + else { assert((dst == NULL || dst->flags & line_buf::LFT_64BIT) && - (lsrc == NULL || lsrc->flags & line_buf::LFT_64BIT) && + (lsrc == NULL || lsrc->flags & line_buf::LFT_64BIT) && (hsrc == NULL || hsrc->flags & line_buf::LFT_64BIT)); sse2_rev_horz_syn64(atk, dst, lsrc, hsrc, width, even); } - } + } } // !local } // !ojph diff --git a/external/OpenJPH/src/core/transform/ojph_transform_vsx.cpp b/external/OpenJPH/src/core/transform/ojph_transform_vsx.cpp new file mode 100644 index 000000000..d45458210 --- /dev/null +++ b/external/OpenJPH/src/core/transform/ojph_transform_vsx.cpp @@ -0,0 +1,1317 @@ +//***************************************************************************/ +// This software is released under the 2-Clause BSD license, included +// below. +// +// Copyright (c) 2021, Aous Naman +// Copyright (c) 2021, Kakadu Software Pty Ltd, Australia +// Copyright (c) 2021, The University of New South Wales, Australia +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +// TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +//***************************************************************************/ +// This file is part of the OpenJPH software implementation. +// File: ojph_transform_vsx.cpp +// Author: Aous Naman +// Date: 09 February 2021 +//***************************************************************************/ + +#include +#include "ojph_simd_vsx.h" + +#include "ojph_defs.h" +#include "ojph_arch.h" +#include "ojph_mem.h" +#include "ojph_params.h" +#include "../codestream/ojph_params_local.h" + +#include "ojph_transform.h" +#include "ojph_transform_local.h" + +namespace ojph { + namespace local { + + ////////////////////////////////////////////////////////////////////////// + static inline + void vsx_deinterleave32(float* dpl, float* dph, float* sp, int width) + { + for (; width > 0; width -= 8, sp += 8, dpl += 4, dph += 4) + { + v128_t a = vsx_v128_load(sp); + v128_t b = vsx_v128_load(sp + 4); + v128_t c = vsx_i32x4_shuffle(a, b, 0, 2, 4 + 0, 4 + 2); + v128_t d = vsx_i32x4_shuffle(a, b, 1, 3, 4 + 1, 4 + 3); + // v128_t c = _mm_shuffle_ps(a, b, _MM_SHUFFLE(2, 0, 2, 0)); + // v128_t d = _mm_shuffle_ps(a, b, _MM_SHUFFLE(3, 1, 3, 1)); + vsx_v128_store(dpl, c); + vsx_v128_store(dph, d); + } + } + + ////////////////////////////////////////////////////////////////////////// + static inline + void vsx_interleave32(float* dp, float* spl, float* sph, int width) + { + for (; width > 0; width -= 8, dp += 8, spl += 4, sph += 4) + { + v128_t a = vsx_v128_load(spl); + v128_t b = vsx_v128_load(sph); + v128_t c = vsx_i32x4_shuffle(a, b, 0, 4 + 0, 1, 4 + 1); + v128_t d = vsx_i32x4_shuffle(a, b, 2, 4 + 2, 3, 4 + 3); + // v128_t c = _mm_unpacklo_ps(a, b); + // v128_t d = _mm_unpackhi_ps(a, b); + vsx_v128_store(dp, c); + vsx_v128_store(dp + 4, d); + } + } + + ////////////////////////////////////////////////////////////////////////// + static inline + void vsx_deinterleave64(void* dpl, void* dph, const void* sp, int width) + { + for (; width > 0; width -= 4, + sp = (const char*)sp + 32, + dpl = (char*)dpl + 16, + dph = (char*)dph + 16) + { + v128_t a = vsx_v128_load(sp); + v128_t b = vsx_v128_load((const char*)sp + 16); + v128_t c = vsx_i64x2_shuffle(a, b, 0, 2 + 0); + v128_t d = vsx_i64x2_shuffle(a, b, 1, 2 + 1); + vsx_v128_store(dpl, c); + vsx_v128_store(dph, d); + } + } + + ////////////////////////////////////////////////////////////////////////// + static inline + void vsx_interleave64(void* dp, const void* spl, const void* sph, + int width) + { + for (; width > 0; width -= 4, + dp = (char*)dp + 32, + spl = (const char*)spl + 16, + sph = (const char*)sph + 16) + { + v128_t a = vsx_v128_load(spl); + v128_t b = vsx_v128_load(sph); + v128_t c = vsx_i64x2_shuffle(a, b, 0, 2 + 0); + v128_t d = vsx_i64x2_shuffle(a, b, 1, 2 + 1); + vsx_v128_store(dp, c); + vsx_v128_store((char*)dp + 16, d); + } + } + + ////////////////////////////////////////////////////////////////////////// + static inline void vsx_multiply_const(float* p, float f, int width) + { + v128_t factor = vsx_f32x4_splat(f); + for (; width > 0; width -= 4, p += 4) + { + v128_t s = vsx_v128_load(p); + vsx_v128_store(p, vsx_f32x4_mul(factor, s)); + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, + ui32 repeat, bool synthesis) + { + float a = s->irv.Aatk; + if (synthesis) + a = -a; + + v128_t factor = vsx_f32x4_splat(a); + + float* dst = aug->f32; + const float* src1 = sig->f32, * src2 = other->f32; + int i = (int)repeat; + for ( ; i > 0; i -= 4, dst += 4, src1 += 4, src2 += 4) + { + v128_t s1 = vsx_v128_load(src1); + v128_t s2 = vsx_v128_load(src2); + v128_t d = vsx_v128_load(dst); + d = vsx_f32x4_add(d, vsx_f32x4_mul(factor, vsx_f32x4_add(s1, s2))); + vsx_v128_store(dst, d); + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_vert_times_K(float K, const line_buf* aug, ui32 repeat) + { + vsx_multiply_const(aug->f32, K, (int)repeat); + } + + ///////////////////////////////////////////////////////////////////////// + void vsx_irv_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, + ui32 width, bool even) + { + if (width > 1) + { + // split src into ldst and hdst + { + float* dpl = even ? ldst->f32 : hdst->f32; + float* dph = even ? hdst->f32 : ldst->f32; + float* sp = src->f32; + int w = (int)width; + vsx_deinterleave32(dpl, dph, sp, w); + } + + // the actual horizontal transform + float* hp = hdst->f32, * lp = ldst->f32; + ui32 l_width = (width + (even ? 1 : 0)) >> 1; // low pass + ui32 h_width = (width + (even ? 0 : 1)) >> 1; // high pass + ui32 num_steps = atk->get_num_steps(); + for (ui32 j = num_steps; j > 0; --j) + { + const lifting_step* s = atk->get_step(j - 1); + const float a = s->irv.Aatk; + + // extension + lp[-1] = lp[0]; + lp[l_width] = lp[l_width - 1]; + // lifting step + const float* sp = lp; + float* dp = hp; + int i = (int)h_width; + v128_t f = vsx_f32x4_splat(a); + if (even) + { + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t m = vsx_v128_load(sp); + v128_t n = vsx_v128_load(sp + 1); + v128_t p = vsx_v128_load(dp); + p = vsx_f32x4_add(p, vsx_f32x4_mul(f, vsx_f32x4_add(m, n))); + vsx_v128_store(dp, p); + } + } + else + { + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t m = vsx_v128_load(sp); + v128_t n = vsx_v128_load(sp - 1); + v128_t p = vsx_v128_load(dp); + p = vsx_f32x4_add(p, vsx_f32x4_mul(f, vsx_f32x4_add(m, n))); + vsx_v128_store(dp, p); + } + } + + // swap buffers + float* t = lp; lp = hp; hp = t; + even = !even; + ui32 w = l_width; l_width = h_width; h_width = w; + } + + { // multiply by K or 1/K + float K = atk->get_K(); + float K_inv = 1.0f / K; + vsx_multiply_const(lp, K_inv, (int)l_width); + vsx_multiply_const(hp, K, (int)h_width); + } + } + else { + if (even) + ldst->f32[0] = src->f32[0]; + else + hdst->f32[0] = src->f32[0] * 2.0f; + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_irv_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, + ui32 width, bool even) + { + if (width > 1) + { + bool ev = even; + float* oth = hsrc->f32, * aug = lsrc->f32; + ui32 aug_width = (width + (even ? 1 : 0)) >> 1; // low pass + ui32 oth_width = (width + (even ? 0 : 1)) >> 1; // high pass + + { // multiply by K or 1/K + float K = atk->get_K(); + float K_inv = 1.0f / K; + vsx_multiply_const(aug, K, (int)aug_width); + vsx_multiply_const(oth, K_inv, (int)oth_width); + } + + // the actual horizontal transform + ui32 num_steps = atk->get_num_steps(); + for (ui32 j = 0; j < num_steps; ++j) + { + const lifting_step* s = atk->get_step(j); + const float a = s->irv.Aatk; + + // extension + oth[-1] = oth[0]; + oth[oth_width] = oth[oth_width - 1]; + // lifting step + const float* sp = oth; + float* dp = aug; + int i = (int)aug_width; + v128_t f = vsx_f32x4_splat(a); + if (ev) + { + for ( ; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t m = vsx_v128_load(sp); + v128_t n = vsx_v128_load(sp - 1); + v128_t p = vsx_v128_load(dp); + p = vsx_f32x4_sub(p, vsx_f32x4_mul(f, vsx_f32x4_add(m, n))); + vsx_v128_store(dp, p); + } + } + else + { + for ( ; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t m = vsx_v128_load(sp); + v128_t n = vsx_v128_load(sp + 1); + v128_t p = vsx_v128_load(dp); + p = vsx_f32x4_sub(p, vsx_f32x4_mul(f, vsx_f32x4_add(m, n))); + vsx_v128_store(dp, p); + } + } + + // swap buffers + float* t = aug; aug = oth; oth = t; + ev = !ev; + ui32 w = aug_width; aug_width = oth_width; oth_width = w; + } + + // combine both lsrc and hsrc into dst + { + float* dp = dst->f32; + float* spl = even ? lsrc->f32 : hsrc->f32; + float* sph = even ? hsrc->f32 : lsrc->f32; + int w = (int)width; + vsx_interleave32(dp, spl, sph, w); + } + } + else { + if (even) + dst->f32[0] = lsrc->f32[0]; + else + dst->f32[0] = hsrc->f32[0] * 0.5f; + } + } + + ///////////////////////////////////////////////////////////////////////// + void vsx_rev_vert_step32(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, + ui32 repeat, bool synthesis) + { + const si32 a = s->rev.Aatk; + const si32 b = s->rev.Batk; + const ui8 e = s->rev.Eatk; + v128_t va = vsx_i32x4_splat(a); + v128_t vb = vsx_i32x4_splat(b); + + si32* dst = aug->i32; + const si32* src1 = sig->i32, * src2 = other->i32; + // The general definition of the wavelet in Part 2 is slightly + // different to part 2, although they are mathematically equivalent + // here, we identify the simpler form from Part 1 and employ them + if (a == 1) + { // 5/3 update and any case with a == 1 + int i = (int)repeat; + if (synthesis) + for (; i > 0; i -= 4, dst += 4, src1 += 4, src2 += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_add(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dst, d); + } + else + for (; i > 0; i -= 4, dst += 4, src1 += 4, src2 += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_add(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dst, d); + } + } + else if (a == -1 && b == 1 && e == 1) + { // 5/3 predict + int i = (int)repeat; + if (synthesis) + for (; i > 0; i -= 4, dst += 4, src1 += 4, src2 += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t w = vsx_i32x4_shr(t, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dst, d); + } + else + for (; i > 0; i -= 4, dst += 4, src1 += 4, src2 += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t w = vsx_i32x4_shr(t, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dst, d); + } + } + else if (a == -1) + { // any case with a == -1, which is not 5/3 predict + int i = (int)repeat; + if (synthesis) + for (; i > 0; i -= 4, dst += 4, src1 += 4, src2 += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_sub(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dst, d); + } + else + for (; i > 0; i -= 4, dst += 4, src1 += 4, src2 += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_sub(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dst, d); + } + } + else + { // general case + int i = (int)repeat; + if (synthesis) + for (; i > 0; i -= 4, dst += 4, src1 += 4, src2 += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t u = vsx_i32x4_mul(va, t); + v128_t v = vsx_i32x4_add(vb, u); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dst, d); + } + else + for (; i > 0; i -= 4, dst += 4, src1 += 4, src2 += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t u = vsx_i32x4_mul(va, t); + v128_t v = vsx_i32x4_add(vb, u); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dst, d); + } + } + } + + ///////////////////////////////////////////////////////////////////////// + void vsx_rev_vert_step64(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, + ui32 repeat, bool synthesis) + { + const si32 a = s->rev.Aatk; + const si32 b = s->rev.Batk; + const ui8 e = s->rev.Eatk; + v128_t va = vsx_i64x2_splat(a); + v128_t vb = vsx_i64x2_splat(b); + + si64* dst = aug->i64; + const si64* src1 = sig->i64, * src2 = other->i64; + // The general definition of the wavelet in Part 2 is slightly + // different to part 2, although they are mathematically equivalent + // here, we identify the simpler form from Part 1 and employ them + if (a == 1) + { // 5/3 update and any case with a == 1 + int i = (int)repeat; + if (synthesis) + for (; i > 0; i -= 2, dst += 2, src1 += 2, src2 += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_add(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dst, d); + } + else + for (; i > 0; i -= 2, dst += 2, src1 += 2, src2 += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_add(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dst, d); + } + } + else if (a == -1 && b == 1 && e == 1) + { // 5/3 predict + int i = (int)repeat; + if (synthesis) + for (; i > 0; i -= 2, dst += 2, src1 += 2, src2 += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t w = vsx_i64x2_shr(t, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dst, d); + } + else + for (; i > 0; i -= 2, dst += 2, src1 += 2, src2 += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t w = vsx_i64x2_shr(t, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dst, d); + } + } + else if (a == -1) + { // any case with a == -1, which is not 5/3 predict + int i = (int)repeat; + if (synthesis) + for (; i > 0; i -= 2, dst += 2, src1 += 2, src2 += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_sub(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dst, d); + } + else + for (; i > 0; i -= 2, dst += 2, src1 += 2, src2 += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_sub(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dst, d); + } + } + else + { // general case + int i = (int)repeat; + if (synthesis) + for (; i > 0; i -= 2, dst += 2, src1 += 2, src2 += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t u = vsx_i64x2_mul(va, t); + v128_t v = vsx_i64x2_add(vb, u); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dst, d); + } + else + for (; i > 0; i -= 2, dst += 2, src1 += 2, src2 += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)src1); + v128_t s2 = vsx_v128_load((v128_t*)src2); + v128_t d = vsx_v128_load((v128_t*)dst); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t u = vsx_i64x2_mul(va, t); + v128_t v = vsx_i64x2_add(vb, u); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dst, d); + } + } + } + + ///////////////////////////////////////////////////////////////////////// + void vsx_rev_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, + ui32 repeat, bool synthesis) + { + if (((sig != NULL) && (sig->flags & line_buf::LFT_32BIT)) || + ((aug != NULL) && (aug->flags & line_buf::LFT_32BIT)) || + ((other != NULL) && (other->flags & line_buf::LFT_32BIT))) + { + assert((sig == NULL || sig->flags & line_buf::LFT_32BIT) && + (other == NULL || other->flags & line_buf::LFT_32BIT) && + (aug == NULL || aug->flags & line_buf::LFT_32BIT)); + vsx_rev_vert_step32(s, sig, other, aug, repeat, synthesis); + } + else + { + assert((sig == NULL || sig->flags & line_buf::LFT_64BIT) && + (other == NULL || other->flags & line_buf::LFT_64BIT) && + (aug == NULL || aug->flags & line_buf::LFT_64BIT)); + vsx_rev_vert_step64(s, sig, other, aug, repeat, synthesis); + } + } + + ///////////////////////////////////////////////////////////////////////// + static + void vsx_rev_horz_ana32(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, + ui32 width, bool even) + { + if (width > 1) + { + // combine both lsrc and hsrc into dst + { + float* dpl = even ? ldst->f32 : hdst->f32; + float* dph = even ? hdst->f32 : ldst->f32; + float* sp = src->f32; + int w = (int)width; + vsx_deinterleave32(dpl, dph, sp, w); + } + + si32* hp = hdst->i32, * lp = ldst->i32; + ui32 l_width = (width + (even ? 1 : 0)) >> 1; // low pass + ui32 h_width = (width + (even ? 0 : 1)) >> 1; // high pass + ui32 num_steps = atk->get_num_steps(); + for (ui32 j = num_steps; j > 0; --j) + { + // first lifting step + const lifting_step* s = atk->get_step(j - 1); + const si32 a = s->rev.Aatk; + const si32 b = s->rev.Batk; + const ui8 e = s->rev.Eatk; + v128_t va = vsx_i32x4_splat(a); + v128_t vb = vsx_i32x4_splat(b); + + // extension + lp[-1] = lp[0]; + lp[l_width] = lp[l_width - 1]; + // lifting step + const si32* sp = lp; + si32* dp = hp; + if (a == 1) + { // 5/3 update and any case with a == 1 + int i = (int)h_width; + if (even) + { + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_add(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else + { + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_add(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + } + else if (a == -1 && b == 1 && e == 1) + { // 5/3 predict + int i = (int)h_width; + if (even) + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t w = vsx_i32x4_shr(t, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t w = vsx_i32x4_shr(t, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else if (a == -1) + { // any case with a == -1, which is not 5/3 predict + int i = (int)h_width; + if (even) + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_sub(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_sub(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else + { // general case + int i = (int)h_width; + if (even) + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t u = vsx_i32x4_mul(va, t); + v128_t v = vsx_i32x4_add(vb, u); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t u = vsx_i32x4_mul(va, t); + v128_t v = vsx_i32x4_add(vb, u); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + + // swap buffers + si32* t = lp; lp = hp; hp = t; + even = !even; + ui32 w = l_width; l_width = h_width; h_width = w; + } + } + else { + if (even) + ldst->i32[0] = src->i32[0]; + else + hdst->i32[0] = src->i32[0] << 1; + } + } + + ///////////////////////////////////////////////////////////////////////// + static + void vsx_rev_horz_ana64(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, + ui32 width, bool even) + { + if (width > 1) + { + // combine both lsrc and hsrc into dst + { + void* dpl = even ? ldst->p : hdst->p; + void* dph = even ? hdst->p : ldst->p; + const void* sp = src->p; + int w = (int)width; + vsx_deinterleave64(dpl, dph, sp, w); + } + + si64* hp = hdst->i64, * lp = ldst->i64; + ui32 l_width = (width + (even ? 1 : 0)) >> 1; // low pass + ui32 h_width = (width + (even ? 0 : 1)) >> 1; // high pass + ui32 num_steps = atk->get_num_steps(); + for (ui32 j = num_steps; j > 0; --j) + { + // first lifting step + const lifting_step* s = atk->get_step(j - 1); + const si32 a = s->rev.Aatk; + const si32 b = s->rev.Batk; + const ui8 e = s->rev.Eatk; + v128_t va = vsx_i64x2_splat(a); + v128_t vb = vsx_i64x2_splat(b); + + // extension + lp[-1] = lp[0]; + lp[l_width] = lp[l_width - 1]; + // lifting step + const si64* sp = lp; + si64* dp = hp; + if (a == 1) + { // 5/3 update and any case with a == 1 + int i = (int)h_width; + if (even) + { + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_add(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else + { + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_add(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + } + else if (a == -1 && b == 1 && e == 1) + { // 5/3 predict + int i = (int)h_width; + if (even) + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t w = vsx_i64x2_shr(t, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t w = vsx_i64x2_shr(t, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else if (a == -1) + { // any case with a == -1, which is not 5/3 predict + int i = (int)h_width; + if (even) + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_sub(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_sub(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else + { // general case + int i = (int)h_width; + if (even) + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t u = vsx_i64x2_mul(va, t); + v128_t v = vsx_i64x2_add(vb, u); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t u = vsx_i64x2_mul(va, t); + v128_t v = vsx_i64x2_add(vb, u); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + + // swap buffers + si64* t = lp; lp = hp; hp = t; + even = !even; + ui32 w = l_width; l_width = h_width; h_width = w; + } + } + else { + if (even) + ldst->i64[0] = src->i64[0]; + else + hdst->i64[0] = src->i64[0] << 1; + } + } + + ///////////////////////////////////////////////////////////////////////// + void vsx_rev_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, + ui32 width, bool even) + { + if (src->flags & line_buf::LFT_32BIT) + { + assert((ldst == NULL || ldst->flags & line_buf::LFT_32BIT) && + (hdst == NULL || hdst->flags & line_buf::LFT_32BIT)); + vsx_rev_horz_ana32(atk, ldst, hdst, src, width, even); + } + else + { + assert((ldst == NULL || ldst->flags & line_buf::LFT_64BIT) && + (hdst == NULL || hdst->flags & line_buf::LFT_64BIT) && + (src == NULL || src->flags & line_buf::LFT_64BIT)); + vsx_rev_horz_ana64(atk, ldst, hdst, src, width, even); + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_horz_syn32(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, + ui32 width, bool even) + { + if (width > 1) + { + bool ev = even; + si32* oth = hsrc->i32, * aug = lsrc->i32; + ui32 aug_width = (width + (even ? 1 : 0)) >> 1; // low pass + ui32 oth_width = (width + (even ? 0 : 1)) >> 1; // high pass + ui32 num_steps = atk->get_num_steps(); + for (ui32 j = 0; j < num_steps; ++j) + { + const lifting_step* s = atk->get_step(j); + const si32 a = s->rev.Aatk; + const si32 b = s->rev.Batk; + const ui8 e = s->rev.Eatk; + v128_t va = vsx_i32x4_splat(a); + v128_t vb = vsx_i32x4_splat(b); + + // extension + oth[-1] = oth[0]; + oth[oth_width] = oth[oth_width - 1]; + // lifting step + const si32* sp = oth; + si32* dp = aug; + if (a == 1) + { // 5/3 update and any case with a == 1 + int i = (int)aug_width; + if (ev) + { + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_add(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else + { + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_add(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + } + else if (a == -1 && b == 1 && e == 1) + { // 5/3 predict + int i = (int)aug_width; + if (ev) + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t w = vsx_i32x4_shr(t, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t w = vsx_i32x4_shr(t, e); + d = vsx_i32x4_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else if (a == -1) + { // any case with a == -1, which is not 5/3 predict + int i = (int)aug_width; + if (ev) + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_sub(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t v = vsx_i32x4_sub(vb, t); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else + { // general case + int i = (int)aug_width; + if (ev) + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t u = vsx_i32x4_mul(va, t); + v128_t v = vsx_i32x4_add(vb, u); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 4, sp += 4, dp += 4) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i32x4_add(s1, s2); + v128_t u = vsx_i32x4_mul(va, t); + v128_t v = vsx_i32x4_add(vb, u); + v128_t w = vsx_i32x4_shr(v, e); + d = vsx_i32x4_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + + // swap buffers + si32* t = aug; aug = oth; oth = t; + ev = !ev; + ui32 w = aug_width; aug_width = oth_width; oth_width = w; + } + + // combine both lsrc and hsrc into dst + { + float* dp = dst->f32; + float* spl = even ? lsrc->f32 : hsrc->f32; + float* sph = even ? hsrc->f32 : lsrc->f32; + int w = (int)width; + vsx_interleave32(dp, spl, sph, w); + } + } + else { + if (even) + dst->i32[0] = lsrc->i32[0]; + else + dst->i32[0] = hsrc->i32[0] >> 1; + } + } + + ////////////////////////////////////////////////////////////////////////// + void vsx_rev_horz_syn64(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, + ui32 width, bool even) + { + if (width > 1) + { + bool ev = even; + si64* oth = hsrc->i64, * aug = lsrc->i64; + ui32 aug_width = (width + (even ? 1 : 0)) >> 1; // low pass + ui32 oth_width = (width + (even ? 0 : 1)) >> 1; // high pass + ui32 num_steps = atk->get_num_steps(); + for (ui32 j = 0; j < num_steps; ++j) + { + const lifting_step* s = atk->get_step(j); + const si32 a = s->rev.Aatk; + const si32 b = s->rev.Batk; + const ui8 e = s->rev.Eatk; + v128_t va = vsx_i64x2_splat(a); + v128_t vb = vsx_i64x2_splat(b); + + // extension + oth[-1] = oth[0]; + oth[oth_width] = oth[oth_width - 1]; + // lifting step + const si64* sp = oth; + si64* dp = aug; + if (a == 1) + { // 5/3 update and any case with a == 1 + int i = (int)aug_width; + if (ev) + { + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_add(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else + { + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_add(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + } + else if (a == -1 && b == 1 && e == 1) + { // 5/3 predict + int i = (int)aug_width; + if (ev) + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t w = vsx_i64x2_shr(t, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t w = vsx_i64x2_shr(t, e); + d = vsx_i64x2_add(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else if (a == -1) + { // any case with a == -1, which is not 5/3 predict + int i = (int)aug_width; + if (ev) + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_sub(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t v = vsx_i64x2_sub(vb, t); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + else + { // general case + int i = (int)aug_width; + if (ev) + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp - 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t u = vsx_i64x2_mul(va, t); + v128_t v = vsx_i64x2_add(vb, u); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + else + for (; i > 0; i -= 2, sp += 2, dp += 2) + { + v128_t s1 = vsx_v128_load((v128_t*)sp); + v128_t s2 = vsx_v128_load((v128_t*)(sp + 1)); + v128_t d = vsx_v128_load((v128_t*)dp); + v128_t t = vsx_i64x2_add(s1, s2); + v128_t u = vsx_i64x2_mul(va, t); + v128_t v = vsx_i64x2_add(vb, u); + v128_t w = vsx_i64x2_shr(v, e); + d = vsx_i64x2_sub(d, w); + vsx_v128_store((v128_t*)dp, d); + } + } + + // swap buffers + si64* t = aug; aug = oth; oth = t; + ev = !ev; + ui32 w = aug_width; aug_width = oth_width; oth_width = w; + } + + // combine both lsrc and hsrc into dst + { + void* dp = dst->p; + const void* spl = even ? lsrc->p : hsrc->p; + const void* sph = even ? hsrc->p : lsrc->p; + int w = (int)width; + vsx_interleave64(dp, spl, sph, w); + } + } + else { + if (even) + dst->i64[0] = lsrc->i64[0]; + else + dst->i64[0] = hsrc->i64[0] >> 1; + } + } + + ///////////////////////////////////////////////////////////////////////// + void vsx_rev_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, + ui32 width, bool even) + { + if (dst->flags & line_buf::LFT_32BIT) + { + assert((lsrc == NULL || lsrc->flags & line_buf::LFT_32BIT) && + (hsrc == NULL || hsrc->flags & line_buf::LFT_32BIT)); + vsx_rev_horz_syn32(atk, dst, lsrc, hsrc, width, even); + } + else + { + assert((dst == NULL || dst->flags & line_buf::LFT_64BIT) && + (lsrc == NULL || lsrc->flags & line_buf::LFT_64BIT) && + (hsrc == NULL || hsrc->flags & line_buf::LFT_64BIT)); + vsx_rev_horz_syn64(atk, dst, lsrc, hsrc, width, even); + } + } + + } // !local +} // !ojph diff --git a/external/OpenJPH/src/core/transform/ojph_transform_wasm.cpp b/external/OpenJPH/src/core/transform/ojph_transform_wasm.cpp index 341cfc321..ab371852f 100644 --- a/external/OpenJPH/src/core/transform/ojph_transform_wasm.cpp +++ b/external/OpenJPH/src/core/transform/ojph_transform_wasm.cpp @@ -2,21 +2,21 @@ // This software is released under the 2-Clause BSD license, included // below. // -// Copyright (c) 2021, Aous Naman +// Copyright (c) 2021, Aous Naman // Copyright (c) 2021, Kakadu Software Pty Ltd, Australia // Copyright (c) 2021, The University of New South Wales, Australia -// +// // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: -// +// // 1. Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. -// +// // 2. Redistributions in binary form must reproduce the above copyright // notice, this list of conditions and the following disclaimer in the // documentation and/or other materials provided with the distribution. -// +// // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED // TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A @@ -85,34 +85,41 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - static inline - void wasm_deinterleave64(double* dpl, double* dph, double* sp, int width) + static inline + void wasm_deinterleave64(void* dpl, void* dph, const void* sp, int width) { - for (; width > 0; width -= 4, sp += 4, dpl += 2, dph += 2) + for (; width > 0; width -= 4, + sp = (const char*)sp + 32, + dpl = (char*)dpl + 16, + dph = (char*)dph + 16) { v128_t a = wasm_v128_load(sp); - v128_t b = wasm_v128_load(sp + 2); + v128_t b = wasm_v128_load((const char*)sp + 16); v128_t c = wasm_i64x2_shuffle(a, b, 0, 2 + 0); v128_t d = wasm_i64x2_shuffle(a, b, 1, 2 + 1); wasm_v128_store(dpl, c); wasm_v128_store(dph, d); } - } + } ////////////////////////////////////////////////////////////////////////// - static inline - void wasm_interleave64(double* dp, double* spl, double* sph, int width) + static inline + void wasm_interleave64(void* dp, const void* spl, const void* sph, + int width) { - for (; width > 0; width -= 4, dp += 4, spl += 2, sph += 2) + for (; width > 0; width -= 4, + dp = (char*)dp + 32, + spl = (const char*)spl + 16, + sph = (const char*)sph + 16) { v128_t a = wasm_v128_load(spl); v128_t b = wasm_v128_load(sph); v128_t c = wasm_i64x2_shuffle(a, b, 0, 2 + 0); v128_t d = wasm_i64x2_shuffle(a, b, 1, 2 + 1); wasm_v128_store(dp, c); - wasm_v128_store(dp + 2, d); + wasm_v128_store((char*)dp + 16, d); } - } + } ////////////////////////////////////////////////////////////////////////// static inline void wasm_multiply_const(float* p, float f, int width) @@ -126,8 +133,8 @@ namespace ojph { } ////////////////////////////////////////////////////////////////////////// - void wasm_irv_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void wasm_irv_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { float a = s->irv.Aatk; @@ -156,8 +163,8 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void wasm_irv_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void wasm_irv_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) @@ -169,7 +176,7 @@ namespace ojph { float* sp = src->f32; int w = (int)width; wasm_deinterleave32(dpl, dph, sp, w); - } + } // the actual horizontal transform float* hp = hdst->f32, * lp = ldst->f32; @@ -232,10 +239,10 @@ namespace ojph { hdst->f32[0] = src->f32[0] * 2.0f; } } - + ////////////////////////////////////////////////////////////////////////// - void wasm_irv_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void wasm_irv_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -303,7 +310,7 @@ namespace ojph { float* sph = even ? hsrc->f32 : lsrc->f32; int w = (int)width; wasm_interleave32(dp, spl, sph, w); - } + } } else { if (even) @@ -314,8 +321,8 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void wasm_rev_vert_step32(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void wasm_rev_vert_step32(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { const si32 a = s->rev.Aatk; @@ -326,7 +333,7 @@ namespace ojph { si32* dst = aug->i32; const si32* src1 = sig->i32, * src2 = other->i32; - // The general definition of the wavelet in Part 2 is slightly + // The general definition of the wavelet in Part 2 is slightly // different to part 2, although they are mathematically equivalent // here, we identify the simpler form from Part 1 and employ them if (a == 1) @@ -411,7 +418,7 @@ namespace ojph { wasm_v128_store((v128_t*)dst, d); } } - else + else { // general case int i = (int)repeat; if (synthesis) @@ -444,8 +451,8 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void wasm_rev_vert_step64(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void wasm_rev_vert_step64(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { const si32 a = s->rev.Aatk; @@ -456,7 +463,7 @@ namespace ojph { si64* dst = aug->i64; const si64* src1 = sig->i64, * src2 = other->i64; - // The general definition of the wavelet in Part 2 is slightly + // The general definition of the wavelet in Part 2 is slightly // different to part 2, although they are mathematically equivalent // here, we identify the simpler form from Part 1 and employ them if (a == 1) @@ -541,7 +548,7 @@ namespace ojph { wasm_v128_store((v128_t*)dst, d); } } - else + else { // general case int i = (int)repeat; if (synthesis) @@ -574,23 +581,23 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void wasm_rev_vert_step(const lifting_step* s, const line_buf* sig, - const line_buf* other, const line_buf* aug, + void wasm_rev_vert_step(const lifting_step* s, const line_buf* sig, + const line_buf* other, const line_buf* aug, ui32 repeat, bool synthesis) { - if (((sig != NULL) && (sig->flags & line_buf::LFT_32BIT)) || + if (((sig != NULL) && (sig->flags & line_buf::LFT_32BIT)) || ((aug != NULL) && (aug->flags & line_buf::LFT_32BIT)) || - ((other != NULL) && (other->flags & line_buf::LFT_32BIT))) + ((other != NULL) && (other->flags & line_buf::LFT_32BIT))) { assert((sig == NULL || sig->flags & line_buf::LFT_32BIT) && - (other == NULL || other->flags & line_buf::LFT_32BIT) && + (other == NULL || other->flags & line_buf::LFT_32BIT) && (aug == NULL || aug->flags & line_buf::LFT_32BIT)); wasm_rev_vert_step32(s, sig, other, aug, repeat, synthesis); } - else + else { assert((sig == NULL || sig->flags & line_buf::LFT_64BIT) && - (other == NULL || other->flags & line_buf::LFT_64BIT) && + (other == NULL || other->flags & line_buf::LFT_64BIT) && (aug == NULL || aug->flags & line_buf::LFT_64BIT)); wasm_rev_vert_step64(s, sig, other, aug, repeat, synthesis); } @@ -598,8 +605,8 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// static - void wasm_rev_horz_ana32(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void wasm_rev_horz_ana32(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) @@ -611,7 +618,7 @@ namespace ojph { float* sp = src->f32; int w = (int)width; wasm_deinterleave32(dpl, dph, sp, w); - } + } si32* hp = hdst->i32, * lp = ldst->i32; ui32 l_width = (width + (even ? 1 : 0)) >> 1; // low pass @@ -719,7 +726,7 @@ namespace ojph { wasm_v128_store((v128_t*)dp, d); } } - else + else { // general case int i = (int)h_width; if (even) @@ -742,7 +749,7 @@ namespace ojph { v128_t s2 = wasm_v128_load((v128_t*)(sp - 1)); v128_t d = wasm_v128_load((v128_t*)dp); v128_t t = wasm_i32x4_add(s1, s2); - v128_t u = wasm_i32x4_mul(va, t); + v128_t u = wasm_i32x4_mul(va, t); v128_t v = wasm_i32x4_add(vb, u); v128_t w = wasm_i32x4_shr(v, e); d = wasm_i32x4_add(d, w); @@ -766,20 +773,20 @@ namespace ojph { ///////////////////////////////////////////////////////////////////////// static - void wasm_rev_horz_ana64(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void wasm_rev_horz_ana64(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { if (width > 1) { // combine both lsrc and hsrc into dst { - double* dpl = (double*)(even ? ldst->p : hdst->p); - double* dph = (double*)(even ? hdst->p : ldst->p); - double* sp = (double*)src->p; + void* dpl = even ? ldst->p : hdst->p; + void* dph = even ? hdst->p : ldst->p; + const void* sp = src->p; int w = (int)width; wasm_deinterleave64(dpl, dph, sp, w); - } + } si64* hp = hdst->i64, * lp = ldst->i64; ui32 l_width = (width + (even ? 1 : 0)) >> 1; // low pass @@ -887,7 +894,7 @@ namespace ojph { wasm_v128_store((v128_t*)dp, d); } } - else + else { // general case int i = (int)h_width; if (even) @@ -910,7 +917,7 @@ namespace ojph { v128_t s2 = wasm_v128_load((v128_t*)(sp - 1)); v128_t d = wasm_v128_load((v128_t*)dp); v128_t t = wasm_i64x2_add(s1, s2); - v128_t u = wasm_i64x2_mul(va, t); + v128_t u = wasm_i64x2_mul(va, t); v128_t v = wasm_i64x2_add(vb, u); v128_t w = wasm_i64x2_shr(v, e); d = wasm_i64x2_add(d, w); @@ -933,28 +940,28 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void wasm_rev_horz_ana(const param_atk* atk, const line_buf* ldst, - const line_buf* hdst, const line_buf* src, + void wasm_rev_horz_ana(const param_atk* atk, const line_buf* ldst, + const line_buf* hdst, const line_buf* src, ui32 width, bool even) { - if (src->flags & line_buf::LFT_32BIT) + if (src->flags & line_buf::LFT_32BIT) { assert((ldst == NULL || ldst->flags & line_buf::LFT_32BIT) && (hdst == NULL || hdst->flags & line_buf::LFT_32BIT)); wasm_rev_horz_ana32(atk, ldst, hdst, src, width, even); } - else + else { assert((ldst == NULL || ldst->flags & line_buf::LFT_64BIT) && - (hdst == NULL || hdst->flags & line_buf::LFT_64BIT) && + (hdst == NULL || hdst->flags & line_buf::LFT_64BIT) && (src == NULL || src->flags & line_buf::LFT_64BIT)); wasm_rev_horz_ana64(atk, ldst, hdst, src, width, even); } - } + } ////////////////////////////////////////////////////////////////////////// - void wasm_rev_horz_syn32(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void wasm_rev_horz_syn32(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -1065,7 +1072,7 @@ namespace ojph { wasm_v128_store((v128_t*)dp, d); } } - else + else { // general case int i = (int)aug_width; if (ev) @@ -1088,7 +1095,7 @@ namespace ojph { v128_t s2 = wasm_v128_load((v128_t*)(sp + 1)); v128_t d = wasm_v128_load((v128_t*)dp); v128_t t = wasm_i32x4_add(s1, s2); - v128_t u = wasm_i32x4_mul(va, t); + v128_t u = wasm_i32x4_mul(va, t); v128_t v = wasm_i32x4_add(vb, u); v128_t w = wasm_i32x4_shr(v, e); d = wasm_i32x4_sub(d, w); @@ -1118,10 +1125,10 @@ namespace ojph { dst->i32[0] = hsrc->i32[0] >> 1; } } - + ////////////////////////////////////////////////////////////////////////// - void wasm_rev_horz_syn64(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void wasm_rev_horz_syn64(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { if (width > 1) @@ -1232,7 +1239,7 @@ namespace ojph { wasm_v128_store((v128_t*)dp, d); } } - else + else { // general case int i = (int)aug_width; if (ev) @@ -1255,7 +1262,7 @@ namespace ojph { v128_t s2 = wasm_v128_load((v128_t*)(sp + 1)); v128_t d = wasm_v128_load((v128_t*)dp); v128_t t = wasm_i64x2_add(s1, s2); - v128_t u = wasm_i64x2_mul(va, t); + v128_t u = wasm_i64x2_mul(va, t); v128_t v = wasm_i64x2_add(vb, u); v128_t w = wasm_i64x2_shr(v, e); d = wasm_i64x2_sub(d, w); @@ -1271,9 +1278,9 @@ namespace ojph { // combine both lsrc and hsrc into dst { - double* dp = (double*)dst->p; - double* spl = (double*)(even ? lsrc->p : hsrc->p); - double* sph = (double*)(even ? hsrc->p : lsrc->p); + void* dp = dst->p; + const void* spl = even ? lsrc->p : hsrc->p; + const void* sph = even ? hsrc->p : lsrc->p; int w = (int)width; wasm_interleave64(dp, spl, sph, w); } @@ -1287,24 +1294,24 @@ namespace ojph { } ///////////////////////////////////////////////////////////////////////// - void wasm_rev_horz_syn(const param_atk* atk, const line_buf* dst, - const line_buf* lsrc, const line_buf* hsrc, + void wasm_rev_horz_syn(const param_atk* atk, const line_buf* dst, + const line_buf* lsrc, const line_buf* hsrc, ui32 width, bool even) { - if (dst->flags & line_buf::LFT_32BIT) + if (dst->flags & line_buf::LFT_32BIT) { - assert((lsrc == NULL || lsrc->flags & line_buf::LFT_32BIT) && + assert((lsrc == NULL || lsrc->flags & line_buf::LFT_32BIT) && (hsrc == NULL || hsrc->flags & line_buf::LFT_32BIT)); wasm_rev_horz_syn32(atk, dst, lsrc, hsrc, width, even); } - else + else { assert((dst == NULL || dst->flags & line_buf::LFT_64BIT) && - (lsrc == NULL || lsrc->flags & line_buf::LFT_64BIT) && + (lsrc == NULL || lsrc->flags & line_buf::LFT_64BIT) && (hsrc == NULL || hsrc->flags & line_buf::LFT_64BIT)); wasm_rev_horz_syn64(atk, dst, lsrc, hsrc, width, even); } - } + } } // !local } // !ojph diff --git a/src/lib/OpenEXRCore/internal_ht.cpp b/src/lib/OpenEXRCore/internal_ht.cpp index 1a1fee368..ee50b8a4e 100644 --- a/src/lib/OpenEXRCore/internal_ht.cpp +++ b/src/lib/OpenEXRCore/internal_ht.cpp @@ -457,7 +457,7 @@ ht_apply_impl (exr_encode_pipeline_t* encode) for (int16_t c = 0; c < encode->channel_count; c++) { if (cs_channel_info[c].kind != visual) - cod.get_coc(c).set_reversible(true); + cod.set_reversible(c, true); } }