diff --git a/.github/workflows/haskell-ci.yml b/.github/workflows/haskell-ci.yml index 96eccee..d3523dd 100644 --- a/.github/workflows/haskell-ci.yml +++ b/.github/workflows/haskell-ci.yml @@ -16,13 +16,13 @@ jobs: fail-fast: true matrix: os: [ubuntu-latest] - ghc: ['9.4', '9.6', '9.8', '9.10'] + ghc: ['9.4', '9.6', '9.8', '9.10', '9.12'] include: - os: macOS-latest - ghc: 'latest' + ghc: '9.12' steps: - uses: actions/checkout@v3 - - uses: haskell/actions/setup@v2 + - uses: haskell-actions/setup@v2 id: setup-haskell-cabal with: ghc-version: ${{ matrix.ghc }} @@ -40,12 +40,14 @@ jobs: cabal sdist -z -o . cabal get base16-*.tar.gz cd base16-*/ - cabal build base16:tests --enable-tests --enable-benchmarks - cabal test --enable-tests --enable-benchmarks --test-show-details=direct all + cabal configure --enable-tests --enable-benchmarks -f simd + cabal build base16:tests + cabal test --test-show-details=direct all - name: Bench run: | cd base16-*/ - cabal bench --enable-tests --enable-benchmarks --benchmark-option=-l all + cabal configure --enable-tests --enable-benchmarks -f simd + cabal bench --benchmark-option=-l all - name: Haddock run: | cd base16-*/ @@ -60,7 +62,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: haskell/actions/setup@v2 + - uses: haskell-actions/setup@v2 id: setup-haskell-cabal with: ghc-version: 'latest' @@ -75,21 +77,3 @@ jobs: key: ${{ runner.os }}-latest - name: Test run: cabal test --ghc-options='-fcheck-prim-bounds -fno-ignore-asserts' - - i386: - needs: build - runs-on: ubuntu-latest - container: - image: i386/ubuntu:bionic - steps: - - name: Install - run: | - apt-get update -y - apt-get install -y autoconf build-essential zlib1g-dev libgmp-dev curl libncurses5 libtinfo5 libncurses5-dev libtinfo-dev - curl --proto '=https' --tlsv1.2 -sSf https://get-ghcup.haskell.org | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 BOOTSTRAP_HASKELL_INSTALL_NO_STACK=1 sh - - uses: actions/checkout@v1 - - name: Test - run: | - source ~/.ghcup/env - cabal update - cabal test diff --git a/base16.cabal b/base16.cabal index eee8e0e..42cec2a 100644 --- a/base16.cabal +++ b/base16.cabal @@ -32,6 +32,16 @@ source-repository head type: git location: https://github.com/emilypi/base16.git +flag simd + description: Enable SIMD detection and code paths. It may incur some unnecessary overhead for most workloads. + default: False + manual: True + +flag pure-haskell + description: Do not use any C implementation, which is useful when using the JavaScript or WASM compiler backends. This flag disables simd. + default: False + manual: True + library exposed-modules: Data.Base16.Types @@ -62,6 +72,22 @@ library default-language: Haskell2010 ghc-options: -Wall + if arch(javascript) || flag(pure-haskell) + cpp-options: -DPURE_HASKELL + + if flag(simd) && !(arch(javascript) || flag(pure-haskell)) + cpp-options: -DSIMD + + exposed-modules: + Data.ByteString.Base16.Internal.Simd + + include-dirs: cbits + c-sources: + cbits/base16.c + cbits/avx_base16.c + cbits/sse_base16.c + cc-options: -O3 -std=c2x + test-suite base16-tests other-modules: Internal default-language: Haskell2010 diff --git a/benchmarks/Base16Bench.hs b/benchmarks/Base16Bench.hs index c20bf0b..d8067d0 100644 --- a/benchmarks/Base16Bench.hs +++ b/benchmarks/Base16Bench.hs @@ -14,6 +14,8 @@ import "base16" Data.ByteString.Base16 as B16 import "base16" Data.ByteString.Short.Base16 as BS16 import "base16-bytestring" Data.ByteString.Base16 as Bos import Data.ByteString.Random (random) +import Data.ByteString.Base16.Internal.Simd (c_isSIMDAvailable) +import qualified Data.ByteString.Base16.Internal.Simd as Simd main :: IO () @@ -21,86 +23,98 @@ main = defaultMain [ env bs $ \ ~((bs25,bs100,bs1k,bs10k,bs100k,bs1mm),(bs25L,bs100L,bs1kL,bs10kL,bs100kL,bs1mmL)) -> bgroup "encode" - [ bgroup "25" - [ bench "base16-short" $ whnf BS16.encodeBase16' bs25L + [ bgroup "25" [ bench "base16-short" $ whnf BS16.encodeBase16' bs25L , bench "base16-bytestring" $ whnf Bos.encode bs25 - , bench "base16" $ whnf B16.encodeBase16' bs25 + , bench "base16 (scalar)" $ whnf (assertBase16 . B16.encodeBase16_) bs25 + , bench "base16 (simd)" $ whnf Simd.encodeBase16SIMD bs25 ] - , bgroup "100" - [ bench "base16-short" $ whnf BS16.encodeBase16' bs100L + , bgroup "100" [ bench "base16-short" $ whnf BS16.encodeBase16' bs100L , bench "base16-bytestring" $ whnf Bos.encode bs100 - , bench "base16" $ whnf B16.encodeBase16' bs100 + , bench "base16 (scalar)" $ whnf (assertBase16 . B16.encodeBase16_) bs100 + , bench "base16 (simd)" $ whnf Simd.encodeBase16SIMD bs100 ] - , bgroup "1k" - [ bench "base16-short" $ whnf BS16.encodeBase16' bs1kL + , bgroup "1k" [ bench "base16-short" $ whnf BS16.encodeBase16' bs1kL , bench "base16-bytestring" $ whnf Bos.encode bs1k - , bench "base16" $ whnf B16.encodeBase16' bs1k + , bench "base16 (scalar)" $ whnf (assertBase16 . B16.encodeBase16_) bs1k + , bench "base16 (simd)" $ whnf Simd.encodeBase16SIMD bs1k ] - , bgroup "10k" - [ bench "base16-short" $ whnf BS16.encodeBase16' bs10kL + , bgroup "10k" [ bench "base16-short" $ whnf BS16.encodeBase16' bs10kL , bench "base16-bytestring" $ whnf Bos.encode bs10k - , bench "base16" $ whnf B16.encodeBase16' bs10k + , bench "base16 (scalar)" $ whnf (assertBase16 . B16.encodeBase16_) bs10k + , bench "base16 (simd)" $ whnf Simd.encodeBase16SIMD bs10k ] - , bgroup "100k" - [ bench "base16-short" $ whnf BS16.encodeBase16' bs100kL + , bgroup "100k" [ bench "base16-short" $ whnf BS16.encodeBase16' bs100kL , bench "base16-bytestring" $ whnf Bos.encode bs100k - , bench "base16" $ whnf B16.encodeBase16' bs100k + , bench "base16 (scalar)" $ whnf (assertBase16 . B16.encodeBase16_) bs100k + , bench "base16 (simd)" $ whnf Simd.encodeBase16SIMD bs100k ] - , bgroup "1mm" - [ bench "base16-short" $ whnf BS16.encodeBase16' bs1mmL + , bgroup "1mm" [ bench "base16-short" $ whnf BS16.encodeBase16' bs1mmL , bench "base16-bytestring" $ whnf Bos.encode bs1mm - , bench "base16" $ whnf B16.encodeBase16' bs1mm + , bench "base16 (scalar)" $ whnf (assertBase16 . B16.encodeBase16_) bs1mm + , bench "base16 (simd)" $ whnf Simd.encodeBase16SIMD bs1mm ] ] , env bs' $ \ ~((bs25,bs100,bs1k,bs10k,bs100k,bs1mm),(bs25L,bs100L,bs1kL,bs10kL,bs100kL,bs1mmL)) -> bgroup "decode" - [ bgroup "25" - [ bench "base16-short" $ whnf BS16.decodeBase16 bs25L + [ bgroup "25" [ bench "base16-short" $ whnf BS16.decodeBase16 bs25L , bench "base16-bytestring" $ whnf Bos.decode $ extractBase16 bs25 - , bench "base16" $ whnf B16.decodeBase16 bs25 - , bench "base16-untyped" $ whnf B16.decodeBase16Untyped $ extractBase16 bs25 + , bench "base16 (scalar)" $ whnf B16.decodeBase16Typed_ bs25 + , bench "base16 (simd)" $ whnf Simd.decodeBase16SIMD bs25 + , bench "base16-untyped (scalar)" $ whnf B16.decodeBase16_ $ extractBase16 bs25 + , bench "base16-untyped (simd)" $ whnf Simd.decodeBase16UntypedSIMD $ extractBase16 bs25 , bench "isBase16" $ whnf B16.isBase16 $ extractBase16 bs25 - , bench "isValidBase16" $ whnf B16.isValidBase16 $ extractBase16 bs25 + , bench "isValidBase16 (scalar)" $ whnf B16.isValidBase16 $ extractBase16 bs25 + , bench "isValidBase16 (simd)" $ whnf Simd.isValidBase16SIMD $ extractBase16 bs25 ] - , bgroup "100" - [ bench "base16-short" $ whnf BS16.decodeBase16 bs100L + , bgroup "100" [ bench "base16-short" $ whnf BS16.decodeBase16 bs100L , bench "base16-bytestring" $ whnf Bos.decode $ extractBase16 bs100 - , bench "base16" $ whnf B16.decodeBase16 bs100 - , bench "base16-untyped" $ whnf B16.decodeBase16Untyped $ extractBase16 bs100 + , bench "base16 (scalar)" $ whnf B16.decodeBase16Typed_ bs100 + , bench "base16 (simd)" $ whnf Simd.decodeBase16SIMD bs100 + , bench "base16-untyped (scalar)" $ whnf B16.decodeBase16_ $ extractBase16 bs100 + , bench "base16-untyped (simd)" $ whnf Simd.decodeBase16UntypedSIMD $ extractBase16 bs100 , bench "isBase16" $ whnf B16.isBase16 $ extractBase16 bs100 - , bench "isValidBase16" $ whnf B16.isValidBase16 $ extractBase16 bs100 + , bench "isValidBase16 (scalar)" $ whnf B16.isValidBase16 $ extractBase16 bs100 + , bench "isValidBase16 (simd)" $ whnf Simd.isValidBase16SIMD $ extractBase16 bs100 ] - , bgroup "1k" - [ bench "base16-short" $ whnf BS16.decodeBase16 bs1kL + , bgroup "1k" [ bench "base16-short" $ whnf BS16.decodeBase16 bs1kL , bench "base16-bytestring" $ whnf Bos.decode $ extractBase16 bs1k - , bench "base16" $ whnf B16.decodeBase16 bs1k - , bench "base16-untyped" $ whnf B16.decodeBase16Untyped $ extractBase16 bs1k + , bench "base16 (scalar)" $ whnf B16.decodeBase16Typed_ bs1k + , bench "base16 (simd)" $ whnf Simd.decodeBase16SIMD bs1k + , bench "base16-untyped (scalar)" $ whnf B16.decodeBase16_ $ extractBase16 bs1k + , bench "base16-untyped (simd)" $ whnf Simd.decodeBase16UntypedSIMD $ extractBase16 bs1k , bench "isBase16" $ whnf B16.isBase16 $ extractBase16 bs1k - , bench "isValidBase16" $ whnf B16.isValidBase16 $ extractBase16 bs1k + , bench "isValidBase16 (scalar)" $ whnf B16.isValidBase16 $ extractBase16 bs1k + , bench "isValidBase16 (simd)" $ whnf Simd.isValidBase16SIMD $ extractBase16 bs1k ] - , bgroup "10k" - [ bench "base16-short" $ whnf BS16.decodeBase16 bs10kL + , bgroup "10k" [ bench "base16-short" $ whnf BS16.decodeBase16 bs10kL , bench "base16-bytestring" $ whnf Bos.decode $ extractBase16 bs10k - , bench "base16" $ whnf B16.decodeBase16 bs10k - , bench "base16-untyped" $ whnf B16.decodeBase16Untyped $ extractBase16 bs10k + , bench "base16 (scalar)" $ whnf B16.decodeBase16Typed_ bs10k + , bench "base16 (simd)" $ whnf Simd.decodeBase16SIMD bs10k + , bench "base16-untyped (scalar)" $ whnf B16.decodeBase16_ $ extractBase16 bs10k + , bench "base16-untyped (simd)" $ whnf Simd.decodeBase16UntypedSIMD $ extractBase16 bs10k , bench "isBase16" $ whnf B16.isBase16 $ extractBase16 bs10k - , bench "isValidBase16" $ whnf B16.isValidBase16 $ extractBase16 bs10k + , bench "isValidBase16 (scalar)" $ whnf B16.isValidBase16 $ extractBase16 bs10k + , bench "isValidBase16 (simd)" $ whnf Simd.isValidBase16SIMD $ extractBase16 bs10k ] - , bgroup "100k" - [ bench "base16-short" $ whnf BS16.decodeBase16 bs100kL + , bgroup "100k" [ bench "base16-short" $ whnf BS16.decodeBase16 bs100kL , bench "base16-bytestring" $ whnf Bos.decode $ extractBase16 bs100k - , bench "base16" $ whnf B16.decodeBase16 bs100k - , bench "base16-untyped" $ whnf B16.decodeBase16Untyped $ extractBase16 bs100k + , bench "base16 (scalar)" $ whnf B16.decodeBase16Typed_ bs100k + , bench "base16 (simd)" $ whnf Simd.decodeBase16SIMD bs100k + , bench "base16-untyped (scalar)" $ whnf B16.decodeBase16_ $ extractBase16 bs100k + , bench "base16-untyped (simd)" $ whnf Simd.decodeBase16UntypedSIMD $ extractBase16 bs100k , bench "isBase16" $ whnf B16.isBase16 $ extractBase16 bs100k - , bench "isValidBase16" $ whnf B16.isValidBase16 $ extractBase16 bs100k + , bench "isValidBase16 (scalar)" $ whnf B16.isValidBase16 $ extractBase16 bs100k + , bench "isValidBase16 (simd)" $ whnf Simd.isValidBase16SIMD $ extractBase16 bs100k ] - , bgroup "1mm" - [ bench "base16-short" $ whnf BS16.decodeBase16 bs1mmL + , bgroup "1mm" [ bench "base16-short" $ whnf BS16.decodeBase16 bs1mmL , bench "base16-bytestring" $ whnf Bos.decode $ extractBase16 bs1mm - , bench "base16" $ whnf B16.decodeBase16 bs1mm - , bench "base16-untyped" $ whnf B16.decodeBase16Untyped $ extractBase16 bs1mm + , bench "base16 (scalar)" $ whnf B16.decodeBase16Typed_ bs1mm + , bench "base16 (simd)" $ whnf Simd.decodeBase16SIMD bs1mm + , bench "base16-untyped (scalar)" $ whnf B16.decodeBase16_ $ extractBase16 bs1mm + , bench "base16-untyped (simd)" $ whnf Simd.decodeBase16UntypedSIMD $ extractBase16 bs1mm , bench "isBase16" $ whnf B16.isBase16 $ extractBase16 bs1mm - , bench "isValidBase16" $ whnf B16.isValidBase16 $ extractBase16 bs1mm + , bench "isValidBase16 (scalar)" $ whnf B16.isValidBase16 $ extractBase16 bs1mm + , bench "isValidBase16 (simd)" $ whnf Simd.isValidBase16SIMD $ extractBase16 bs1mm ] ] ] diff --git a/cbits/avx_base16.c b/cbits/avx_base16.c new file mode 100644 index 0000000..0df3c6e --- /dev/null +++ b/cbits/avx_base16.c @@ -0,0 +1,225 @@ +#ifdef __x86_64__ +#include +#include "base16.h" + + +__attribute__((target("avx2"))) +void avx_encodeBase16(const uint8_t* restrict input, size_t len, + uint8_t* restrict output) { + + static const _Alignas(32) uint8_t lookup[32] = { + '0','1','2','3','4','5','6','7', + '8','9','a','b','c','d','e','f', + '0','1','2','3','4','5','6','7', + '8','9','a','b','c','d','e','f', + }; + + const auto masklow = _mm256_set1_epi8(0xf); + const auto lut = _mm256_load_si256((__m256i*)lookup); + + const size_t iter_len = sizeof(lut); + + // Fix alignment in input buffer + if(__builtin_expect(len >= iter_len, 1)) { + auto extra_iters = iter_len - ((intptr_t)(input) & (iter_len-1)); + + auto v = _mm256_lddqu_si256((__m256i*)(input)); + v = _mm256_permute4x64_epi64(v, 0b11011000); + + auto lows = _mm256_and_si256(v, masklow); + auto highs = _mm256_srli_epi16(v, 4); + highs = _mm256_and_si256(highs, masklow); + + lows = _mm256_shuffle_epi8(lut, lows); + highs = _mm256_shuffle_epi8(lut, highs); + + auto ret1 = _mm256_unpacklo_epi8(highs, lows); + auto ret2 = _mm256_unpackhi_epi8(highs, lows); + + _mm256_storeu_si256((__m256i*)output, ret1); + _mm256_storeu_si256(((__m256i*)output)+1, ret2); + + len -= extra_iters; + output += 2*extra_iters; + input += extra_iters; + } + + auto i_ptr = (__m256i*)input; + auto o_ptr = (__m256i*)output; + + for(size_t i = 0; i < len/iter_len; ++i, ++i_ptr, o_ptr+=2) { + auto v = _mm256_load_si256(i_ptr); + + // Because unpack is kind of awkward in AVX, we need to + // shuffle first. That way the first elements end + // up in the low part of each 128 bit lane + v = _mm256_permute4x64_epi64(v, 0b11011000); + + auto lows = _mm256_and_si256(v, masklow); + auto highs = _mm256_srli_epi16(v, 4); + highs = _mm256_and_si256(highs, masklow); + + // Do substitutions + lows = _mm256_shuffle_epi8(lut, lows); + highs = _mm256_shuffle_epi8(lut, highs); + + auto ret1 = _mm256_unpacklo_epi8(highs, lows); + auto ret2 = _mm256_unpackhi_epi8(highs, lows); + + _mm256_storeu_si256(o_ptr, ret1); + _mm256_storeu_si256(o_ptr+1, ret2); + } + + const uint8_t* rem_i_ptr = (const uint8_t*)i_ptr; + uint8_t* rem_o_ptr = (uint8_t*)o_ptr; + for(size_t i = len%iter_len; i > 0; --i, ++rem_i_ptr, rem_o_ptr+=2) { + uint8_t x = *rem_i_ptr; + *rem_o_ptr = lookup[x>>4]; + *(rem_o_ptr+1) = lookup[x&0xf]; + } +} + +__attribute__((target("avx2"))) +void avx_decodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output) { + auto i_ptr = (const __m256i*)input; + auto o_ptr = (__m256i*)output; + + const auto c9 = _mm256_set1_epi8('9'); + const auto lnibbles = _mm256_set1_epi8(0x0f); + + // When vectors are fetched, the weights of each char is like so + // H1 L1 H2 L2... + // The following permutation allows us to go from that to + // L1 L2... H1 H2... (inside each 128 bit lane) + static const _Alignas(32) uint8_t perm_lohi[32] = { + 1, 3, 5, 7, 9, 11, 13, 15, + 0, 2, 4, 6, 8, 10, 12, 14, + 1, 3, 5, 7, 9, 11, 13, 15, + 0, 2, 4, 6, 8, 10, 12, 14, + }; + + const auto p_lohi = _mm256_load_si256((const __m256i*)perm_lohi); + const auto p_hilo = _mm256_shuffle_epi32(p_lohi, 0b01001110); + + const size_t iter_len = 2*sizeof(c9); + + for(size_t i = 0; i < len/iter_len; ++i,i_ptr+=2,++o_ptr) { + auto rl = _mm256_lddqu_si256(i_ptr); + auto rh = _mm256_lddqu_si256(i_ptr+1); + + const auto lettermask_l = _mm256_cmpgt_epi8(rl, c9); + const auto lettermask_h = _mm256_cmpgt_epi8(rh, c9); + + // Adds 0x9 to low nibbles of each char, where there are letters + rl = _mm256_add_epi8(_mm256_and_si256(lettermask_l, c9), rl); + rh = _mm256_add_epi8(_mm256_and_si256(lettermask_h, c9), rh); + + rl = _mm256_and_si256(rl, lnibbles); + rh = _mm256_and_si256(rh, lnibbles); + // Here, all characters have been converted to their numeric + // value. It remains to do the horizontal sums... + + // The variable names should be understood as what happens + // *inside each 128 bit lane* + const auto hil_lol = _mm256_shuffle_epi8(rl, p_hilo); + const auto loh_hih = _mm256_shuffle_epi8(rh, p_lohi); + + auto hil_hih = _mm256_blend_epi16(hil_lol, loh_hih, 0b11110000); + auto loh_lol = _mm256_blend_epi16(hil_lol, loh_hih, 0b00001111); + + hil_hih = _mm256_slli_epi16(hil_hih, 4); + auto lol_loh = _mm256_shuffle_epi32(loh_lol, 0b01001110); + + auto result = _mm256_add_epi8(hil_hih, lol_loh); + // Compared to SSE, we need an additional permute. Indeed, ordering + // the lanes of rl and rh as ( 0, 1), ( 2, 3) we have decoded and + // stored (0' || 2', 1' || 3') + + _mm256_storeu_si256(o_ptr, _mm256_permute4x64_epi64(result, 0b11011000)); + } + + const uint8_t* rem_i_ptr = (const uint8_t*)i_ptr; + uint8_t* rem_o_ptr = (uint8_t*)o_ptr; + for(size_t i = len%iter_len; i > 0; i-=2, rem_i_ptr+=2, ++rem_o_ptr) { + int h = *rem_i_ptr; + int l = *(rem_i_ptr+1); + + h = (h > '9' ? h + 0x9 : h) & 0x0f; + l = (l > '9' ? l + 0x9 : l) & 0x0f; + + *rem_o_ptr = (h << 4) | l; + } +} + +__attribute__((target("avx2"))) +bool avx_isValidBase16(const uint8_t* restrict input, size_t len) { + + // The valid alphabet has three ranges, '0' - '9', 'a' - 'f', 'A'-'F' + // Testing that a vector is 0 is easy, so we use _active low_ logic + // !( ('0' <= x && x <= '9') || ('a' <= x && x <= 'f') || ... ) rewrites as + // (x < '0' || x > '9') && (x < 'a' || x > 'f') && ... + + const auto c0 = _mm256_set1_epi8('0'); + const auto c9 = _mm256_set1_epi8('9'); + const auto ca = _mm256_set1_epi8('a'); + const auto cf = _mm256_set1_epi8('f'); + const auto cA = _mm256_set1_epi8('A'); + const auto cF = _mm256_set1_epi8('F'); + + const size_t iter_len = sizeof(c0); + + if(__builtin_expect(len >= iter_len, 1)) { + auto extra_iters = iter_len - ((intptr_t)(input) & (iter_len-1)); + + auto x = _mm256_lddqu_si256((__m256i*)input); + + auto num = _mm256_or_si256(_mm256_cmpgt_epi8(c0, x), + _mm256_cmpgt_epi8(x, c9)); + + auto lower = _mm256_or_si256(_mm256_cmpgt_epi8(ca, x), + _mm256_cmpgt_epi8(x, cf)); + + auto upper = _mm256_or_si256(_mm256_cmpgt_epi8(cA, x), + _mm256_cmpgt_epi8(x, cF)); + + // If num && lower && upper == 0, then all characters are good and testz returns 1 + if(__builtin_expect(!_mm256_testz_si256(_mm256_and_si256(num, lower), upper) , 0)) { + return false; + } + + input += extra_iters; + len -= extra_iters; + } + + auto i_ptr = (const __m256i*)input; + + for(size_t i = 0; i < len/iter_len; ++i, ++i_ptr) { + auto x = _mm256_load_si256(i_ptr); + + auto num = _mm256_or_si256(_mm256_cmpgt_epi8(c0, x), + _mm256_cmpgt_epi8(x, c9)); + + auto lower = _mm256_or_si256(_mm256_cmpgt_epi8(ca, x), + _mm256_cmpgt_epi8(x, cf)); + + auto upper = _mm256_or_si256(_mm256_cmpgt_epi8(cA, x), + _mm256_cmpgt_epi8(x, cF)); + + // If num && lower && upper == 0, then all characters are good and testz returns 1 + if(__builtin_expect(!_mm256_testz_si256(_mm256_and_si256(num, lower), upper) , 0)) { + return false; + } + } + + const uint8_t* rem_i_ptr = (const uint8_t*)i_ptr; + for(size_t i = len%iter_len; i > 0; --i, ++rem_i_ptr) { + uint8_t x = *rem_i_ptr; + if(__builtin_expect((x < '0' || x > '9') && + (x < 'a' || x > 'f') && + (x < 'A' || x > 'F'), 0)) { + return false; + } + } + return true; +} +#endif diff --git a/cbits/base16.c b/cbits/base16.c new file mode 100644 index 0000000..d6e036e --- /dev/null +++ b/cbits/base16.c @@ -0,0 +1,60 @@ +#include "base16.h" + +static void (*resolve_encode (void)) (const uint8_t* restrict,size_t,uint8_t* restrict) { +#ifdef __x86_64__ + __builtin_cpu_init(); + if(__builtin_cpu_supports("avx2")) { + return avx_encodeBase16; + } else if(__builtin_cpu_supports("sse4.1")) { + return sse_encodeBase16; + } +#endif + + return nullptr; +} + +static void (*resolve_decode (void)) (const uint8_t* restrict,size_t,uint8_t* restrict) { +#ifdef __x86_64__ + __builtin_cpu_init(); + if(__builtin_cpu_supports("avx2")) { + return avx_decodeBase16; + } else if(__builtin_cpu_supports("sse4.1")) { + return sse_decodeBase16; + } +#endif + + return nullptr; +} + +static bool (*resolve_isvalid (void)) (const uint8_t* restrict,size_t){ +#ifdef __x86_64__ + __builtin_cpu_init(); + if(__builtin_cpu_supports("avx2")) { + return avx_isValidBase16; + } else if(__builtin_cpu_supports("sse4.1")) { + return sse_isValidBase16; + } +#endif + + return nullptr; +} + +bool isSIMDAvailable() { + +#ifdef __x86_64__ + if(__builtin_cpu_supports("sse4.1")) { + return true; + } +#endif + + return false; +} + +void encodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output) +__attribute__ ((ifunc ("resolve_encode"))); + +void decodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output) +__attribute__ ((ifunc ("resolve_decode"))); + +bool isValidBase16(const uint8_t* restrict input, size_t len) +__attribute__ ((ifunc ("resolve_isvalid"))); diff --git a/cbits/base16.h b/cbits/base16.h new file mode 100644 index 0000000..1c2856d --- /dev/null +++ b/cbits/base16.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +// For optimal performance, input and output should be aligned to 32 byte boundaries, +// both at input and output. +void encodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output); + +// decodeBase16 accepts mixed-case input. Warning: No check is done by the function itself. +void decodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output); + +// Checks that input obeys the valid alphabet +bool isValidBase16(const uint8_t* restrict input, size_t len); + +// Checks that the above functions are legal to call, i.e. there is some +// available implementation that will run on the current machine. +bool isSIMDAvailable(); + +// The above functions are indirect: at object load time, CPU features are probed +// and the appropriate vectorized version is chosen among the set below. +// Note that if SSE 4.1 is not supported, then they will crash! + +#ifdef __x86_64__ + +// No feature check is performed by the code paths below. Use at your own risk. +// SSE specific code path; requires SSE 4.1 +void sse_encodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output); +void sse_decodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output); +bool sse_isValidBase16(const uint8_t* restrict input, size_t len); + +// AVX specific code path; requires AVX 2 +void avx_encodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output); +void avx_decodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output); +bool avx_isValidBase16(const uint8_t* restrict input, size_t len); + +#endif diff --git a/cbits/sse_base16.c b/cbits/sse_base16.c new file mode 100644 index 0000000..2c7969e --- /dev/null +++ b/cbits/sse_base16.c @@ -0,0 +1,206 @@ +#ifdef __x86_64__ + +#include +#include "base16.h" + +__attribute__((target("sse4.1,no-avx"))) +void sse_encodeBase16(const uint8_t* restrict input, size_t len, + uint8_t* restrict output) { + + static const _Alignas(16) uint8_t lookup[16] = { + '0','1','2','3','4','5','6','7', + '8','9','a','b','c','d','e','f', + }; + + const auto masklow = _mm_set1_epi8(0xf); + const auto lut = _mm_load_si128((__m128i*)lookup); + + const size_t iter_len = sizeof(lut); + + // Fix alignment in input buffer + if(__builtin_expect(len >= iter_len, 1)) { + auto extra_iters = iter_len - ((intptr_t)(input) & (iter_len-1)); + + auto v = _mm_lddqu_si128((__m128i*)(input)); + auto lows = _mm_and_si128(v, masklow); + auto highs = _mm_srli_epi16(v, 4); + highs = _mm_and_si128(highs, masklow); + + lows = _mm_shuffle_epi8(lut, lows); + highs = _mm_shuffle_epi8(lut, highs); + + auto ret1 = _mm_unpacklo_epi8(highs, lows); + auto ret2 = _mm_unpackhi_epi8(highs, lows); + + _mm_storeu_si128((__m128i*)output, ret1); + _mm_storeu_si128(((__m128i*)output)+1, ret2); + + len -= extra_iters; + output += 2*extra_iters; + input += extra_iters; + } + + auto i_ptr = (__m128i*)input; + auto o_ptr = (__m128i*)output; + + for(size_t i = 0; i < len/iter_len; ++i, ++i_ptr, o_ptr+=2) { + auto v = _mm_load_si128(i_ptr); + + auto lows = _mm_and_si128(v, masklow); + auto highs = _mm_srli_epi16(v, 4); + highs = _mm_and_si128(highs, masklow); + + // Do substitutions + lows = _mm_shuffle_epi8(lut, lows); + highs = _mm_shuffle_epi8(lut, highs); + + auto ret1 = _mm_unpacklo_epi8(highs, lows); + auto ret2 = _mm_unpackhi_epi8(highs, lows); + + _mm_storeu_si128(o_ptr, ret1); + _mm_storeu_si128(o_ptr+1, ret2); + } + + const uint8_t* rem_i_ptr = (const uint8_t*)i_ptr; + uint8_t* rem_o_ptr = (uint8_t*)o_ptr; + for(size_t i = len%iter_len; i > 0; --i, ++rem_i_ptr, rem_o_ptr+=2) { + uint8_t x = *rem_i_ptr; + *rem_o_ptr = lookup[x>>4]; + *(rem_o_ptr+1) = lookup[x&0xf]; + } +} + +__attribute__((target("sse4.1,no-avx"))) +void sse_decodeBase16(const uint8_t* restrict input, size_t len, uint8_t* restrict output) { + auto i_ptr = (const __m128i*)input; + auto o_ptr = (__m128i*)output; + + const auto c9 = _mm_set1_epi8('9'); + const auto lnibbles = _mm_set1_epi8(0x0f); + + // When vectors are fetched, the weights of each char is like so + // H1 L1 H2 L2... + // The following permutation allows us to go from that to + // L1 L2... H1 H2... + static const _Alignas(16) uint8_t perm_lohi[16] = { + 1, 3, 5, 7, 9, 11, 13, 15, + 0, 2, 4, 6, 8, 10, 12, 14, + }; + + const auto p_lohi = _mm_load_si128((const __m128i*)perm_lohi); + const auto p_hilo = _mm_shuffle_epi32(p_lohi, 0b01001110); + + const size_t iter_len = 2*sizeof(c9); + + for(size_t i = 0; i < len/iter_len; ++i,i_ptr+=2,++o_ptr) { + auto rl = _mm_lddqu_si128(i_ptr); + auto rh = _mm_lddqu_si128(i_ptr+1); + + const auto lettermask_l = _mm_cmpgt_epi8(rl, c9); + const auto lettermask_h = _mm_cmpgt_epi8(rh, c9); + + // Adds 0x9 to low nibbles of each char, where there are letters + rl = _mm_add_epi8(_mm_and_si128(lettermask_l, c9), rl); + rh = _mm_add_epi8(_mm_and_si128(lettermask_h, c9), rh); + + rl = _mm_and_si128(rl, lnibbles); + rh = _mm_and_si128(rh, lnibbles); + // Here, all characters have been converted to their numeric + // value. It remains to do the horizontal sums... + + const auto hil_lol = _mm_shuffle_epi8(rl, p_hilo); + const auto loh_hih = _mm_shuffle_epi8(rh, p_lohi); + + auto hil_hih = _mm_blend_epi16(hil_lol, loh_hih, 0b11110000); + auto loh_lol = _mm_blend_epi16(hil_lol, loh_hih, 0b00001111); + + hil_hih = _mm_slli_epi16(hil_hih, 4); + auto lol_loh = _mm_shuffle_epi32(loh_lol, 0b01001110); + + _mm_storeu_si128(o_ptr, _mm_add_epi8(hil_hih, lol_loh)); + } + + const uint8_t* rem_i_ptr = (const uint8_t*)i_ptr; + uint8_t* rem_o_ptr = (uint8_t*)o_ptr; + for(size_t i = len%iter_len; i > 0; i-=2, rem_i_ptr+=2, ++rem_o_ptr) { + int h = *rem_i_ptr; + int l = *(rem_i_ptr+1); + + h = (h > '9' ? h + 0x9 : h) & 0x0f; + l = (l > '9' ? l + 0x9 : l) & 0x0f; + + *rem_o_ptr = (h << 4) | l; + } +} + +__attribute__((target("sse4.1,no-avx"))) +bool sse_isValidBase16(const uint8_t* restrict input, size_t len) { + + // The valid alphabet has three ranges, '0' - '9', 'a' - 'f', 'A'-'F' + // Testing that a vector is 0 is easy, so we use _active low_ logic + // !( ('0' <= x && x <= '9') || ('a' <= x && x <= 'f') || ... ) rewrites as + // (x < '0' || x > '9') && (x < 'a' || x > 'f') && ... + + const auto c0 = _mm_set1_epi8('0'); + const auto c9 = _mm_set1_epi8('9'); + const auto ca = _mm_set1_epi8('a'); + const auto cf = _mm_set1_epi8('f'); + const auto cA = _mm_set1_epi8('A'); + const auto cF = _mm_set1_epi8('F'); + + const size_t iter_len = sizeof(c0); + + if(__builtin_expect(len >= iter_len, 1)) { + auto extra_iters = iter_len - ((intptr_t)(input) & (iter_len-1)); + + auto x = _mm_lddqu_si128((__m128i*)(input)); + + auto num = _mm_or_si128(_mm_cmpgt_epi8(c0, x), + _mm_cmpgt_epi8(x, c9)); + + auto lower = _mm_or_si128(_mm_cmpgt_epi8(ca, x), + _mm_cmpgt_epi8(x, cf)); + + auto upper = _mm_or_si128(_mm_cmpgt_epi8(cA, x), + _mm_cmpgt_epi8(x, cF)); + + // If num && lower && upper == 0, then all characters are good and testz returns 1 + if(__builtin_expect(!_mm_testz_si128(_mm_and_si128(num, lower), upper) , 0)) { + return false; + } + + input += extra_iters; + len -= extra_iters; + } + + auto i_ptr = (const __m128i*)input; + for(size_t i = 0; i < len/iter_len; ++i, ++i_ptr) { + auto x = _mm_load_si128(i_ptr); + + auto num = _mm_or_si128(_mm_cmpgt_epi8(c0, x), + _mm_cmpgt_epi8(x, c9)); + + auto lower = _mm_or_si128(_mm_cmpgt_epi8(ca, x), + _mm_cmpgt_epi8(x, cf)); + + auto upper = _mm_or_si128(_mm_cmpgt_epi8(cA, x), + _mm_cmpgt_epi8(x, cF)); + + // If num && lower && upper == 0, then all characters are good and testz returns 1 + if(__builtin_expect(!_mm_testz_si128(_mm_and_si128(num, lower), upper) , 0)) { + return false; + } + } + + const uint8_t* rem_i_ptr = (const uint8_t*)i_ptr; + for(size_t i = len%iter_len; i > 0; --i, ++rem_i_ptr) { + uint8_t x = *rem_i_ptr; + if(__builtin_expect((x < '0' || x > '9') && + (x < 'a' || x > 'f') && + (x < 'A' || x > 'F'), 0)) { + return false; + } + } + return true; +} +#endif diff --git a/src/Data/ByteString/Base16.hs b/src/Data/ByteString/Base16.hs index 81c0ff8..4371a60 100644 --- a/src/Data/ByteString/Base16.hs +++ b/src/Data/ByteString/Base16.hs @@ -1,5 +1,7 @@ {-# LANGUAGE BangPatterns #-} {-# LANGUAGE MagicHash #-} +{-# LANGUAGE CPP #-} +{-# OPTIONS_GHC -Wno-unused-matches #-} -- | -- Module : Data.ByteString.Base16 -- Copyright : (c) 2020-2023 Emily Pillmore @@ -24,6 +26,9 @@ module Data.ByteString.Base16 , isBase16 , isValidBase16 , parseBase16 +, encodeBase16_ +, decodeBase16Typed_ +, decodeBase16_ ) where @@ -36,10 +41,12 @@ import Data.ByteString.Base16.Internal.Head import Data.ByteString.Base16.Internal.Utils (aix) import Data.Text (Text) import qualified Data.Text.Encoding as T - import Foreign.ForeignPtr import Foreign.Ptr import Foreign.Storable +#if !defined(PURE_HASKELL) && defined(SIMD) +import Data.ByteString.Base16.Internal.Simd +#endif -- $setup -- @@ -70,7 +77,15 @@ encodeBase16 = fmap T.decodeUtf8 . encodeBase16' -- "53756e" -- encodeBase16' :: ByteString -> Base16 ByteString -encodeBase16' = assertBase16 . encodeBase16_ +encodeBase16' = +#if !defined(PURE_HASKELL) && defined(SIMD) + if c_isSIMDAvailable + then encodeBase16SIMD + else assertBase16 . encodeBase16_ +#else + assertBase16 . encodeBase16_ +#endif + {-# INLINE encodeBase16' #-} -- | Decode a Base16-encoded 'ByteString' value. @@ -83,7 +98,14 @@ encodeBase16' = assertBase16 . encodeBase16_ -- "Sun" -- decodeBase16 :: Base16 ByteString -> ByteString -decodeBase16 = decodeBase16Typed_ +decodeBase16 = +#if !defined(PURE_HASKELL) && defined(SIMD) + if c_isSIMDAvailable + then decodeBase16SIMD + else decodeBase16Typed_ +#else + decodeBase16Typed_ +#endif {-# INLINE decodeBase16 #-} -- | Decode Base16 'Text'. @@ -112,7 +134,14 @@ decodeBase16' = decodeBase16Typed_ . fmap T.encodeUtf8 -- Left "invalid character at offset: 1" -- decodeBase16Untyped :: ByteString -> Either Text ByteString -decodeBase16Untyped = decodeBase16_ +decodeBase16Untyped = +#if !defined(PURE_HASKELL) && defined(SIMD) + if c_isSIMDAvailable + then decodeBase16UntypedSIMD + else decodeBase16_ +#else + decodeBase16_ +#endif {-# INLINE decodeBase16Untyped #-} -- | Decode a Base16-encoded 'ByteString' value leniently, using a @@ -174,10 +203,18 @@ parseBase16 bs = assertBase16 bs <$ decodeBase16Untyped bs -- True -- isValidBase16 :: ByteString -> Bool -isValidBase16 (BS ptr len) = - accursedUnutterablePerformIO $ do +isValidBase16 bs@(BS ptr len) = +#if !defined(PURE_HASKELL) && defined(SIMD) + if c_isSIMDAvailable + then isValidBase16SIMD bs + else accursedUnutterablePerformIO $ do withForeignPtr ptr $ \bptr -> go bptr (plusPtr bptr len) +#else + accursedUnutterablePerformIO $ do + withForeignPtr ptr $ \bptr -> + go bptr (plusPtr bptr len) +#endif where !valid = "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\xff\xff\xff\xff\xff\xff\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x0a\x0b\x0c\x0d\x0e\x0f\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"# go !bptr !end diff --git a/src/Data/ByteString/Base16/Internal/Simd.hs b/src/Data/ByteString/Base16/Internal/Simd.hs new file mode 100644 index 0000000..21078cc --- /dev/null +++ b/src/Data/ByteString/Base16/Internal/Simd.hs @@ -0,0 +1,107 @@ +{-# LANGUAGE CApiFFI #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE ScopedTypeVariables #-} +{-# LANGUAGE OverloadedStrings #-} + +module Data.ByteString.Base16.Internal.Simd + ( c_isSIMDAvailable + , c_encodeBase16SIMD + , c_isValidBase16SIMD + , c_decodeBase16SIMD + , isValidBase16SIMD + , decodeBase16SIMD + , decodeBase16UntypedSIMD + , encodeBase16SIMD + ) where + +import Foreign.C.Types +import Foreign.Ptr +import qualified Foreign.Marshal.Alloc as Foreign +import Data.ByteString (StrictByteString) +import System.IO.Unsafe +import Data.ByteString.Unsafe (unsafeUseAsCStringLen, unsafePackMallocCStringLen) +import qualified Data.ByteString as BS + +import Data.Base16.Types.Internal +import qualified Foreign.Marshal.Utils as Foreign +import Data.Text (Text) +import Data.Base16.Types (assertBase16) + +foreign import ccall "isSIMDAvailable" + c_isSIMDAvailable :: Bool + +foreign import ccall "encodeBase16" + c_encodeBase16SIMD + :: Ptr CUChar + -- ^ Buffer that holds the input value + -> CSize + -- ^ Length of the input + -> Ptr CUChar + -- ^ Buffer that will hold the encoded base16 + -> IO () + +foreign import ccall "isValidBase16" + c_isValidBase16SIMD + :: Ptr CUChar + -- ^ Buffer that holds the value to be validated + -> CSize + -- ^ Length of the input + -> IO Bool + +foreign import ccall "decodeBase16" + c_decodeBase16SIMD + :: Ptr CUChar + -- ^ Buffer that holds the input base16 + -> CSize + -- ^ Length of the input + -> Ptr CUChar + -- ^ Buffer that will hold the decoded value + -> IO () + +isValidBase16SIMD + :: StrictByteString + -> Bool +isValidBase16SIMD bytestring = unsafeDupablePerformIO $ + unsafeUseAsCStringLen bytestring $ \(cString, cStringLen) -> do + c_isValidBase16SIMD (castPtr cString :: Ptr CUChar) (fromIntegral @Int @CSize cStringLen) + +decodeBase16SIMD + :: Base16 StrictByteString + -> StrictByteString +decodeBase16SIMD (Base16 bytestring) = unsafeDupablePerformIO $ + unsafeUseAsCStringLen bytestring $ \(cString, cStringLen) -> do + let outputLength :: Int = cStringLen `div` 2 + Foreign.allocaBytesAligned outputLength 32 $ \outputPtr -> do + c_decodeBase16SIMD + (castPtr @CChar @CUChar cString) + (fromIntegral @Int @CSize cStringLen) + outputPtr + bsPtr <- Foreign.mallocBytes outputLength + Foreign.copyBytes bsPtr outputPtr outputLength + unsafePackMallocCStringLen (castPtr bsPtr :: Ptr CChar, fromIntegral outputLength) + +decodeBase16UntypedSIMD + :: StrictByteString + -> Either Text StrictByteString +decodeBase16UntypedSIMD bytestring = do + case BS.length bytestring `rem` 2 of + 0 -> + if isValidBase16SIMD bytestring + then Right $ decodeBase16SIMD $ assertBase16 bytestring + else Left "invalid characters" + _ -> Left "Size of input bytestring is not even" + +encodeBase16SIMD + :: StrictByteString + -> Base16 StrictByteString +encodeBase16SIMD bytestring = unsafeDupablePerformIO $ + unsafeUseAsCStringLen bytestring $ \(cString, cStringLen) -> do + let outputLength :: Int = cStringLen * 2 + Foreign.allocaBytesAligned outputLength 32 $ \outputPtr -> do + c_encodeBase16SIMD + (castPtr @CChar @CUChar cString) + (fromIntegral @Int @CSize cStringLen) + outputPtr + bsPtr <- Foreign.mallocBytes outputLength + Foreign.copyBytes bsPtr outputPtr outputLength + assertBase16 <$> unsafePackMallocCStringLen (castPtr bsPtr :: Ptr CChar, fromIntegral outputLength)