diff --git a/CMakeLists.txt b/CMakeLists.txt index 8879b400..1f0937f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,7 +191,7 @@ endif() # Release flags set(RELEASE_CXX_FLAGS - -O3 -DNDEBUG + -O3 -DNDEBUG -flto=full ) foreach(flag ${RELEASE_CXX_FLAGS}) check_cxx_compiler_flag(${flag} has_flag_${flag}) diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 9658c255..a95c8150 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -44,24 +44,22 @@ fi chunk_size=51200 # for rs-fnt with different packet sizes -word_size=2 for word_size in 1 2; do - for type_size in 2 4; do - max_len=$((256**word_size)) - if ((type_size>word_size)); then - for ec_type in rs-fnt rs-fnt-sys; do - for k in 16 64; do - for n in 32 256 1024; do - if ((nk)); then - m=$((n-k)) - for pkt_size in 512; do - ${bin} -e ${ec_type} -w ${word_size} -t ${type_size} -k ${k} -m ${m} -c ${chunk_size} -s ${sce_type} -g ${threads_nb} -f ${show_type} -p ${pkt_size} -n ${samples_nb} - show_type=0 - done - fi + type_size=$((word_size*2)) + max_len=$((256**word_size)) + if ((type_size>word_size)); then + for ec_type in rs-fnt rs-fnt-sys; do + for k in 16 64; do + for n in 32 256 1024; do + if ((nk)); then + m=$((n-k)) + for pkt_size in 32 64 128 256 512 1024; do + ${bin} -e ${ec_type} -w ${word_size} -t ${type_size} -k ${k} -m ${m} -c ${chunk_size} -s ${sce_type} -g ${threads_nb} -f ${show_type} -p ${pkt_size} -n ${samples_nb} + show_type=0 done - done + fi done - fi - done + done + done + fi done diff --git a/scripts/test_ec.sh b/scripts/test_ec.sh index 4b133150..4b7e2b47 100755 --- a/scripts/test_ec.sh +++ b/scripts/test_ec.sh @@ -174,7 +174,6 @@ do fec_type=$(echo $i|cut -d_ -f1) word_size=$(echo $i|cut -d_ -f2) - do_test enconly ${fec_type} ${word_size} 50 50 "" "" do_test all ${fec_type} ${word_size} 3 3 "" "" do_test all ${fec_type} ${word_size} 3 3 "0 1" "0" do_test all ${fec_type} ${word_size} 3 5 "0 1" "0" diff --git a/src/quadiron_c.cpp b/src/quadiron_c.cpp index 82542445..3400eebd 100644 --- a/src/quadiron_c.cpp +++ b/src/quadiron_c.cpp @@ -215,6 +215,16 @@ int quadiron_fnt32_decode( if (!res) return -1; + // reset metadata of data + for (unsigned i = 0; i < fec->n_data; i++) { + parities_props[i].clear(); + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = parities_props[i].fnt_serialize(metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + return 0; } @@ -262,7 +272,6 @@ int quadiron_fnt32_reconstruct( return -1; } parities_vec[i] = data[i] + metadata_size; - data_vec[i] = data[i] + metadata_size; } for (unsigned i = 0; i < fec->n_parities; i++) { if (!missing_idxs[fec->n_data + i]) { diff --git a/src/simd_128.h b/src/simd_128.h index 856588ec..7f034c2c 100644 --- a/src/simd_128.h +++ b/src/simd_128.h @@ -40,29 +40,23 @@ typedef __m128i VecType; /* ============= Constant variable ============ */ -// @note: using const leads to an lint error of initialization of 'variable' -// with static storage duration may throw an exception that cannot be caught - -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F4_U32 = _mm_set1_epi32(65537); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F4_MINUS_ONE_U32 = _mm_set1_epi32(65536); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_U32 = _mm_set1_epi32(257); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_MINUS_ONE_U32 = _mm_set1_epi32(256); - -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_U16 = _mm_set1_epi16(257); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_MINUS_ONE_U16 = _mm_set1_epi16(256); +template +inline VecType one(); +template <> +inline VecType one() +{ + return _mm_set1_epi16(1); +} +template <> +inline VecType one() +{ + return _mm_set1_epi32(1); +} -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ZERO = _mm_setzero_si128(); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ONE_U16 = _mm_set1_epi16(1); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ONE_U32 = _mm_set1_epi32(1); +inline VecType zero() +{ + return _mm_setzero_si128(); +} // NOLINTNEXTLINE(cert-err58-cpp) const VecType MASK8_LO = _mm_set1_epi16(0x80); @@ -78,25 +72,25 @@ inline void store_to_mem(VecType* address, VecType reg) _mm_store_si128(address, reg); } -inline VecType bit_and(VecType x, VecType y) +inline VecType bit_and(const VecType& x, const VecType& y) { return _mm_and_si128(x, y); } -inline VecType bit_xor(VecType x, VecType y) +inline VecType bit_xor(const VecType& x, const VecType& y) { return _mm_xor_si128(x, y); } -inline uint16_t msb8_mask(VecType x) +inline uint16_t msb8_mask(const VecType& x) { return _mm_movemask_epi8(x); } -inline bool and_is_zero(VecType x, VecType y) +inline bool and_is_zero(const VecType& x, const VecType& y) { return _mm_testz_si128(x, y); } -inline bool is_zero(VecType x) +inline bool is_zero(const VecType& x) { - return _mm_testc_si128(ZERO, x); + return _mm_testc_si128(zero(), x); } #define SHIFTR(x, imm8) (_mm_srli_si128(x, imm8)) @@ -119,66 +113,66 @@ inline VecType set_one(uint16_t val) } template -inline VecType add(VecType x, VecType y); +inline VecType add(const VecType& x, const VecType& y); template <> -inline VecType add(VecType x, VecType y) +inline VecType add(const VecType& x, const VecType& y) { return _mm_add_epi32(x, y); } template <> -inline VecType add(VecType x, VecType y) +inline VecType add(const VecType& x, const VecType& y) { return _mm_add_epi16(x, y); } template -inline VecType sub(VecType x, VecType y); +inline VecType sub(const VecType& x, const VecType& y); template <> -inline VecType sub(VecType x, VecType y) +inline VecType sub(const VecType& x, const VecType& y) { return _mm_sub_epi32(x, y); } template <> -inline VecType sub(VecType x, VecType y) +inline VecType sub(const VecType& x, const VecType& y) { return _mm_sub_epi16(x, y); } template -inline VecType mul(VecType x, VecType y); +inline VecType mul(const VecType& x, const VecType& y); template <> -inline VecType mul(VecType x, VecType y) +inline VecType mul(const VecType& x, const VecType& y) { return _mm_mullo_epi32(x, y); } template <> -inline VecType mul(VecType x, VecType y) +inline VecType mul(const VecType& x, const VecType& y) { return _mm_mullo_epi16(x, y); } template -inline VecType compare_eq(VecType x, VecType y); +inline VecType compare_eq(const VecType& x, const VecType& y); template <> -inline VecType compare_eq(VecType x, VecType y) +inline VecType compare_eq(const VecType& x, const VecType& y) { return _mm_cmpeq_epi32(x, y); } template <> -inline VecType compare_eq(VecType x, VecType y) +inline VecType compare_eq(const VecType& x, const VecType& y) { return _mm_cmpeq_epi16(x, y); } template -inline VecType min(VecType x, VecType y); +inline VecType min(const VecType& x, const VecType& y); template <> -inline VecType min(VecType x, VecType y) +inline VecType min(const VecType& x, const VecType& y) { return _mm_min_epu32(x, y); } template <> -inline VecType min(VecType x, VecType y) +inline VecType min(const VecType& x, const VecType& y) { return _mm_min_epu16(x, y); } diff --git a/src/simd_256.h b/src/simd_256.h index 8084d95e..7497cb32 100644 --- a/src/simd_256.h +++ b/src/simd_256.h @@ -53,29 +53,23 @@ typedef __m128i HalfVecType; /* ============= Constant variable ============ */ -// @note: using const leads to an lint error of initialization of 'variable' -// with static storage duration may throw an exception that cannot be caught - -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F4_U32 = _mm256_set1_epi32(65537); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F4_MINUS_ONE_U32 = _mm256_set1_epi32(65536); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_U32 = _mm256_set1_epi32(257); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_MINUS_ONE_U32 = _mm256_set1_epi32(256); - -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_U16 = _mm256_set1_epi16(257); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_MINUS_ONE_U16 = _mm256_set1_epi16(256); +template +inline VecType one(); +template <> +inline VecType one() +{ + return _mm256_set1_epi16(1); +} +template <> +inline VecType one() +{ + return _mm256_set1_epi32(1); +} -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ZERO = _mm256_setzero_si256(); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ONE_U16 = _mm256_set1_epi16(1); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ONE_U32 = _mm256_set1_epi32(1); +inline VecType zero() +{ + return _mm256_setzero_si256(); +} // NOLINTNEXTLINE(cert-err58-cpp) const VecType MASK8_LO = _mm256_set1_epi16(0x80); @@ -86,30 +80,30 @@ inline VecType load_to_reg(VecType* address) { return _mm256_load_si256(address); } -inline void store_to_mem(VecType* address, VecType reg) +inline void store_to_mem(VecType* address, const VecType& reg) { _mm256_store_si256(address, reg); } -inline VecType bit_and(VecType x, VecType y) +inline VecType bit_and(const VecType& x, const VecType& y) { return _mm256_and_si256(x, y); } -inline VecType bit_xor(VecType x, VecType y) +inline VecType bit_xor(const VecType& x, const VecType& y) { return _mm256_xor_si256(x, y); } -inline uint32_t msb8_mask(VecType x) +inline uint32_t msb8_mask(const VecType& x) { return _mm256_movemask_epi8(x); } -inline bool and_is_zero(VecType x, VecType y) +inline bool and_is_zero(const VecType& x, const VecType& y) { return _mm256_testz_si256(x, y); } -inline bool is_zero(VecType x) +inline bool is_zero(const VecType& x) { - return _mm256_testc_si256(ZERO, x); + return _mm256_testc_si256(zero(), x); } #define SHIFTR(x, imm8) (_mm256_srli_si256(x, imm8)) @@ -132,66 +126,66 @@ inline VecType set_one(uint16_t val) } template -inline VecType add(VecType x, VecType y); +inline VecType add(const VecType& x, const VecType& y); template <> -inline VecType add(VecType x, VecType y) +inline VecType add(const VecType& x, const VecType& y) { return _mm256_add_epi32(x, y); } template <> -inline VecType add(VecType x, VecType y) +inline VecType add(const VecType& x, const VecType& y) { return _mm256_add_epi16(x, y); } template -inline VecType sub(VecType x, VecType y); +inline VecType sub(const VecType& x, const VecType& y); template <> -inline VecType sub(VecType x, VecType y) +inline VecType sub(const VecType& x, const VecType& y) { return _mm256_sub_epi32(x, y); } template <> -inline VecType sub(VecType x, VecType y) +inline VecType sub(const VecType& x, const VecType& y) { return _mm256_sub_epi16(x, y); } template -inline VecType mul(VecType x, VecType y); +inline VecType mul(const VecType& x, const VecType& y); template <> -inline VecType mul(VecType x, VecType y) +inline VecType mul(const VecType& x, const VecType& y) { return _mm256_mullo_epi32(x, y); } template <> -inline VecType mul(VecType x, VecType y) +inline VecType mul(const VecType& x, const VecType& y) { return _mm256_mullo_epi16(x, y); } template -inline VecType compare_eq(VecType x, VecType y); +inline VecType compare_eq(const VecType& x, const VecType& y); template <> -inline VecType compare_eq(VecType x, VecType y) +inline VecType compare_eq(const VecType& x, const VecType& y) { return _mm256_cmpeq_epi32(x, y); } template <> -inline VecType compare_eq(VecType x, VecType y) +inline VecType compare_eq(const VecType& x, const VecType& y) { return _mm256_cmpeq_epi16(x, y); } template -inline VecType min(VecType x, VecType y); +inline VecType min(const VecType& x, const VecType& y); template <> -inline VecType min(VecType x, VecType y) +inline VecType min(const VecType& x, const VecType& y) { return _mm256_min_epu32(x, y); } template <> -inline VecType min(VecType x, VecType y) +inline VecType min(const VecType& x, const VecType& y) { return _mm256_min_epu16(x, y); } diff --git a/src/simd_fnt.h b/src/simd_fnt.h index 59e52410..a20471dd 100644 --- a/src/simd_fnt.h +++ b/src/simd_fnt.h @@ -37,42 +37,57 @@ namespace quadiron { namespace simd { template -inline VecType card(T q); +inline VecType card(); template <> -inline VecType card(uint16_t) +inline VecType card() { - return F3_U16; + return set_one(257); } template <> -inline VecType card(uint32_t q) +inline VecType card() { - return (q == F3) ? F3_U32 : F4_U32; + return set_one(65537); } template -inline VecType card_minus_one(T q); +inline VecType card_minus_one(); template <> -inline VecType card_minus_one(uint16_t) +inline VecType card_minus_one() { - return F3_MINUS_ONE_U16; + return set_one(256); } template <> -inline VecType card_minus_one(uint32_t q) +inline VecType card_minus_one() { - return (q == F3) ? F3_MINUS_ONE_U32 : F4_MINUS_ONE_U32; + return set_one(65536); } +const int I_MASK8_LO = 0b01010101; + template -inline VecType get_low_half(VecType x, T q) +inline VecType get_low_half(const VecType& x); +template <> +inline VecType get_low_half(const VecType& x) +{ + return BLEND8(zero(), x, MASK8_LO); +} +template <> +inline VecType get_low_half(const VecType& x) { - return (q == F3) ? BLEND8(ZERO, x, MASK8_LO) : BLEND16(ZERO, x, 0x55); + return BLEND16(zero(), x, I_MASK8_LO); } template -inline VecType get_high_half(VecType x, T q) +inline VecType get_high_half(const VecType& x); +template <> +inline VecType get_high_half(const VecType& x) +{ + return BLEND8(zero(), SHIFTR(x, 1), MASK8_LO); +} +template <> +inline VecType get_high_half(const VecType& x) { - return (q == F3) ? BLEND8(ZERO, SHIFTR(x, 1), MASK8_LO) - : BLEND16(ZERO, SHIFTR(x, 2), 0x55); + return BLEND16(zero(), SHIFTR(x, 2), I_MASK8_LO); } /* ================= Basic Operations ================= */ @@ -82,14 +97,13 @@ inline VecType get_high_half(VecType x, T q) * * @param x input register * @param y input register - * @param q modulo * @return (x + y) mod q */ template -inline VecType mod_add(VecType x, VecType y, T q) +inline VecType mod_add(const VecType& x, const VecType& y) { const VecType res = add(x, y); - return min(res, sub(res, card(q))); + return min(res, sub(res, card())); } /** @@ -97,28 +111,26 @@ inline VecType mod_add(VecType x, VecType y, T q) * * @param x input register * @param y input register - * @param q modulo * @return (x - y) mod q */ template -inline VecType mod_sub(VecType x, VecType y, T q) +inline VecType mod_sub(const VecType& x, const VecType& y) { const VecType res = sub(x, y); - return min(res, add(res, card(q))); + return min(res, add(res, card())); } /** * Modular negation for packed unsigned 32-bit integers * * @param x input register - * @param q modulo * @return (-x) mod q */ template -inline VecType mod_neg(VecType x, T q) +inline VecType mod_neg(const VecType& x) { - const VecType res = sub(card(q), x); - return min(res, sub(res, card(q))); + const VecType res = sub(card(), x); + return min(res, sub(res, card())); } /** @@ -129,16 +141,15 @@ inline VecType mod_neg(VecType x, T q) * * @param x input register * @param y input register - * @param q modulo * @return (x * y) mod q */ template -inline VecType mod_mul(VecType x, VecType y, T q) +inline VecType mod_mul(const VecType& x, const VecType& y) { const VecType res = mul(x, y); - const VecType lo = get_low_half(res, q); - const VecType hi = get_high_half(res, q); - return mod_sub(lo, hi, q); + const VecType lo = get_low_half(res); + const VecType hi = get_high_half(res); + return mod_sub(lo, hi); } /** @@ -148,24 +159,22 @@ inline VecType mod_mul(VecType x, VecType y, T q) * * @param x input register * @param y input register - * @param q modulo * @return (x * y) mod q */ template -inline VecType mod_mul_safe(VecType x, VecType y, T q) +inline VecType mod_mul_safe(const VecType& x, const VecType& y) { - const VecType res = mod_mul(x, y, q); + const VecType res = mod_mul(x, y); // filter elements of both of a & b = card-1 const VecType cmp = bit_and( - compare_eq(x, card_minus_one(q)), - compare_eq(y, card_minus_one(q))); + compare_eq(x, card_minus_one()), + compare_eq(y, card_minus_one())); if (is_zero(cmp)) { return res; } - return (q == F3) ? bit_xor(res, bit_and(F4_U32, cmp)) - : add(res, bit_and(ONE_U32, cmp)); + return add(res, bit_and(one(), cmp)); } /** diff --git a/src/simd_nf4.h b/src/simd_nf4.h index f9ec1172..06a46ebf 100644 --- a/src/simd_nf4.h +++ b/src/simd_nf4.h @@ -180,7 +180,7 @@ inline __uint128_t add(__uint128_t a, __uint128_t b) HalfVecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_low_half_to_mem(&res, mod_add(vec_a, vec_b, F4)); + store_low_half_to_mem(&res, mod_add(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -189,7 +189,7 @@ inline __uint128_t sub(__uint128_t a, __uint128_t b) HalfVecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_low_half_to_mem(&res, mod_sub(vec_a, vec_b, F4)); + store_low_half_to_mem(&res, mod_sub(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -198,7 +198,7 @@ inline __uint128_t mul(__uint128_t a, __uint128_t b) HalfVecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_low_half_to_mem(&res, mod_mul_safe(vec_a, vec_b, F4)); + store_low_half_to_mem(&res, mod_mul_safe(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -217,8 +217,8 @@ inline void add_buf_to_two_bufs_rem( VecType _x_next_p = load_to_reg(_x_half[i]); VecType _y_p = load_to_reg(_y[i]); - store_low_half_to_mem(_x + i, mod_add(_x_p, _y_p, F4)); - store_low_half_to_mem(_x_half + i, mod_add(_x_next_p, _y_p, F4)); + store_low_half_to_mem(_x + i, mod_add(_x_p, _y_p)); + store_low_half_to_mem(_x_half + i, mod_add(_x_next_p, _y_p)); } } @@ -230,7 +230,7 @@ inline void hadamard_mul_rem(unsigned n, __uint128_t* x, __uint128_t* y) VecType _x_p = load_to_reg(_x[i]); VecType _y_p = load_to_reg(_y[i]); - store_low_half_to_mem(_x + i, mod_mul_safe(_x_p, _y_p, F4)); + store_low_half_to_mem(_x + i, mod_mul_safe(_x_p, _y_p)); } } @@ -248,8 +248,9 @@ inline void hadamard_mul_doubled_rem( VecType _x_next_p = load_to_reg(_x_half[i]); VecType _y_p = load_to_reg(_y[i]); - store_low_half_to_mem(_x + i, mod_mul_safe(_x_p, _y_p, F4)); - store_low_half_to_mem(_x_half + i, mod_mul_safe(_x_next_p, _y_p, F4)); + store_low_half_to_mem(_x + i, mod_mul_safe(_x_p, _y_p)); + store_low_half_to_mem( + _x_half + i, mod_mul_safe(_x_next_p, _y_p)); } } @@ -266,7 +267,7 @@ inline __uint128_t add(__uint128_t a, __uint128_t b) VecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_to_mem(&res, mod_add(vec_a, vec_b, F4)); + store_to_mem(&res, mod_add(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -275,7 +276,7 @@ inline __uint128_t sub(__uint128_t a, __uint128_t b) VecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_to_mem(&res, mod_sub(vec_a, vec_b, F4)); + store_to_mem(&res, mod_sub(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -284,7 +285,7 @@ inline __uint128_t mul(__uint128_t a, __uint128_t b) VecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_to_mem(&res, mod_mul_safe(vec_a, vec_b, F4)); + store_to_mem(&res, mod_mul_safe(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -327,12 +328,12 @@ inline void add_buf_to_two_bufs(unsigned n, __uint128_t* _x, __uint128_t* _y) // add y to the first half of `x` for (i = 0; i < vec_len; ++i) { - x[i] = mod_add(x[i], y[i], F4); + x[i] = mod_add(x[i], y[i]); } // add y to the second half of `x` for (i = 0; i < vec_len; ++i) { - x_next[i] = mod_add(x_next[i], y[i], F4); + x_next[i] = mod_add(x_next[i], y[i]); } if (rem_len > 0) { @@ -354,7 +355,7 @@ inline void hadamard_mul(unsigned n, __uint128_t* _x, __uint128_t* _y) // multiply y to the first half of `x` for (i = 0; i < vec_len; ++i) { - x[i] = mod_mul_safe(x[i], y[i], F4); + x[i] = mod_mul_safe(x[i], y[i]); } if (rem_len > 0) { diff --git a/src/simd_radix2_fft.h b/src/simd_radix2_fft.h index 8405f8f0..66dafc17 100644 --- a/src/simd_radix2_fft.h +++ b/src/simd_radix2_fft.h @@ -36,6 +36,24 @@ namespace quadiron { namespace simd { +enum class CtGsCase { + SIMPLE, + NORMAL, + EXTREME, +}; + +template +constexpr CtGsCase get_case(T r, T q) +{ + if (r == 1) { + return CtGsCase::SIMPLE; + } else if (r < q - 1) { + return CtGsCase::NORMAL; + } else { + return CtGsCase::EXTREME; + } +} + /* ================= Vectorized Operations ================= */ /** @@ -44,22 +62,30 @@ namespace simd { * x <- x + r * y * y <- x - r * y * - * @param rp1 coefficient `r` plus one + * @param ct_case coefficient case * @param c a register stores coefficient `r` * @param x working register * @param y working register - * @param q modular */ template -inline void butterfly_ct(T rp1, VecType c, VecType* x, VecType* y, T q) +inline void +butterfly_ct(CtGsCase ct_case, const VecType& c, VecType& x, VecType& y) { - VecType z = (rp1 == 2) ? *y : mod_mul(c, *y, q); - if (rp1 < q) { - *y = mod_sub(*x, z, q); - *x = mod_add(*x, z, q); - } else { // i.e. r == q - 1 - *y = mod_add(*x, z, q); - *x = mod_sub(*x, z, q); + VecType z = y; + switch (ct_case) { + case CtGsCase::SIMPLE: + y = mod_sub(x, z); + x = mod_add(x, z); + break; + case CtGsCase::EXTREME: + y = mod_add(x, z); + x = mod_sub(x, z); + break; + case CtGsCase::NORMAL: + z = mod_mul(c, y); + y = mod_sub(x, z); + x = mod_add(x, z); + break; } } @@ -69,25 +95,29 @@ inline void butterfly_ct(T rp1, VecType c, VecType* x, VecType* y, T q) * x <- x + y * y <- r * (x - y) * - * @param rp1 coefficient `r` plus one + * @param gs_case coefficient case * @param c a register stores coefficient `r` * @param x working register * @param y working register - * @param q modular */ template -inline void butterfly_gs(T rp1, VecType c, VecType* x, VecType* y, T q) +inline void +butterfly_gs(CtGsCase gs_case, const VecType& c, VecType& x, VecType& y) { - VecType add = mod_add(*x, *y, q); - if (rp1 == 2) { - *y = mod_sub(*x, *y, q); - } else if (rp1 < q) { - VecType sub = mod_sub(*x, *y, q); - *y = mod_mul(c, sub, q); - } else { // i.e. r == q - 1 - *y = mod_sub(*y, *x, q); + VecType add = mod_add(x, y); + switch (gs_case) { + case CtGsCase::SIMPLE: + y = mod_sub(x, y); + break; + case CtGsCase::EXTREME: + y = mod_sub(y, x); + break; + case CtGsCase::NORMAL: + VecType sub = mod_sub(x, y); + y = mod_mul(c, sub); + break; } - *x = add; + x = add; } /** @@ -96,24 +126,24 @@ inline void butterfly_gs(T rp1, VecType c, VecType* x, VecType* y, T q) * x <- x, i.e. no operation * y <- r * x * - * @param rp1 coefficient `r` plus one + * @param gs_case coefficient case * @param c a register stores coefficient `r` * @param x working register - * @param q modular - * @return r * x */ template -inline VecType butterfly_simple_gs(T rp1, VecType c, VecType x, T q) +inline void butterfly_simple_gs(CtGsCase gs_case, const VecType& c, VecType& x) { - if (rp1 == 2) { - return x; - } else if (rp1 < q) { - return mod_mul(c, x, q); - } else { - return mod_neg(x, q); + switch (gs_case) { + case CtGsCase::EXTREME: + x = mod_neg(x); + break; + case CtGsCase::NORMAL: + x = mod_mul(c, x); + break; + case CtGsCase::SIMPLE: + break; } } - /** * Vectorized butterfly CT step * @@ -139,54 +169,49 @@ inline void butterfly_ct_step( size_t len, T card) { - if (len == 0) { - return; - } - const T rp1 = r + 1; - VecType c = set_one(r); + const CtGsCase ct_case = get_case(r, card); + const VecType c = set_one(r); const size_t end = (len > 1) ? len - 1 : 0; const unsigned bufs_nb = buf.get_n(); const std::vector& mem = buf.get_mem(); for (unsigned i = start; i < bufs_nb; i += step) { - VecType x1, y1; - VecType x2, y2; VecType* p = reinterpret_cast(mem[i]); VecType* q = reinterpret_cast(mem[i + m]); size_t j = 0; for (; j < end; j += 2) { - x1 = load_to_reg(p + j); - y1 = load_to_reg(q + j); + VecType x1 = load_to_reg(p); + VecType y1 = load_to_reg(q); - butterfly_ct(rp1, c, &x1, &y1, card); + butterfly_ct(ct_case, c, x1, y1); - x2 = load_to_reg(p + j + 1); - y2 = load_to_reg(q + j + 1); + VecType x2 = load_to_reg(p + 1); + VecType y2 = load_to_reg(q + 1); - butterfly_ct(rp1, c, &x2, &y2, card); + butterfly_ct(ct_case, c, x2, y2); // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(p + j + 1, x2); - store_to_mem(q + j, y1); - store_to_mem(q + j + 1, y2); + store_to_mem(p++, x1); + store_to_mem(p++, x2); + store_to_mem(q++, y1); + store_to_mem(q++, y2); } for (; j < len; ++j) { - x1 = load_to_reg(p + j); - y1 = load_to_reg(q + j); + VecType x1 = load_to_reg(p); + VecType y1 = load_to_reg(q); - butterfly_ct(rp1, c, &x1, &y1, card); + butterfly_ct(ct_case, c, x1, y1); // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(q + j, y1); + store_to_mem(p++, x1); + store_to_mem(q++, y1); } } } template -inline static void do_butterfly_ct_2_layers( +inline void do_butterfly_ct_2_layers( const std::vector& mem, T r1, T r2, @@ -196,9 +221,9 @@ inline static void do_butterfly_ct_2_layers( size_t len, T card) { - const T r1p1 = r1 + 1; - const T r2p1 = r2 + 1; - const T r3p1 = r3 + 1; + const CtGsCase case1 = get_case(r1, card); + const CtGsCase case2 = get_case(r2, card); + const CtGsCase case3 = get_case(r3, card); VecType c1 = set_one(r1); VecType c2 = set_one(r2); @@ -212,65 +237,52 @@ inline static void do_butterfly_ct_2_layers( size_t j = 0; const size_t end = (len > 1) ? len - 1 : 0; while (j < end) { - // First layer (c1, x, y) & (c1, u, v) VecType x1 = load_to_reg(p); - VecType x2 = load_to_reg(p + 1); VecType y1 = load_to_reg(q); - VecType y2 = load_to_reg(q + 1); + VecType u1 = load_to_reg(r); + VecType v1 = load_to_reg(s); - butterfly_ct(r1p1, c1, &x1, &y1, card); - butterfly_ct(r1p1, c1, &x2, &y2, card); + butterfly_ct(case1, c1, x1, y1); + butterfly_ct(case1, c1, u1, v1); + butterfly_ct(case2, c2, x1, u1); + butterfly_ct(case3, c3, y1, v1); - VecType u1 = load_to_reg(r); + VecType x2 = load_to_reg(p + 1); + VecType y2 = load_to_reg(q + 1); VecType u2 = load_to_reg(r + 1); - VecType v1 = load_to_reg(s); VecType v2 = load_to_reg(s + 1); - butterfly_ct(r1p1, c1, &u1, &v1, card); - butterfly_ct(r1p1, c1, &u2, &v2, card); - - // Second layer (c2, x, u) & (c3, y, v) - butterfly_ct(r2p1, c2, &x1, &u1, card); - butterfly_ct(r2p1, c2, &x2, &u2, card); - - butterfly_ct(r3p1, c3, &y1, &v1, card); - butterfly_ct(r3p1, c3, &y2, &v2, card); - - // Store back to memory - store_to_mem(p, x1); - store_to_mem(p + 1, x2); - store_to_mem(q, y1); - store_to_mem(q + 1, y2); - - store_to_mem(r, u1); - store_to_mem(r + 1, u2); - store_to_mem(s, v1); - store_to_mem(s + 1, v2); - p = p + 2; - q = q + 2; - r = r + 2; - s = s + 2; - j = j + 2; - }; - + butterfly_ct(case1, c1, x2, y2); + butterfly_ct(case1, c1, u2, v2); + butterfly_ct(case2, c2, x2, u2); + butterfly_ct(case3, c3, y2, v2); + + store_to_mem(p++, x1); + store_to_mem(p++, x2); + store_to_mem(q++, y1); + store_to_mem(q++, y2); + store_to_mem(r++, u1); + store_to_mem(r++, u2); + store_to_mem(s++, v1); + store_to_mem(s++, v2); + + j += 2; + } for (; j < len; ++j) { - // First layer (c1, x, y) & (c1, u, v) - VecType x1 = load_to_reg(p + j); - VecType y1 = load_to_reg(q + j); - VecType u1 = load_to_reg(r + j); - VecType v1 = load_to_reg(s + j); - - // BUTTERFLY_3_test(c1, &x1, &y1, &u1, &v1, card); - butterfly_ct(r1p1, c1, &x1, &y1, card); - butterfly_ct(r1p1, c1, &u1, &v1, card); - butterfly_ct(r2p1, c2, &x1, &u1, card); - butterfly_ct(r3p1, c3, &y1, &v1, card); - - // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(q + j, y1); - store_to_mem(r + j, u1); - store_to_mem(s + j, v1); + VecType x1 = load_to_reg(p); + VecType y1 = load_to_reg(q); + VecType u1 = load_to_reg(r); + VecType v1 = load_to_reg(s); + + butterfly_ct(case1, c1, x1, y1); + butterfly_ct(case1, c1, u1, v1); + butterfly_ct(case2, c2, x1, u1); + butterfly_ct(case3, c3, y1, v1); + + store_to_mem(p++, x1); + store_to_mem(q++, y1); + store_to_mem(r++, u1); + store_to_mem(s++, v1); } } @@ -352,53 +364,23 @@ inline void butterfly_gs_step( return; } const unsigned step = m << 1; - const T rp1 = r + 1; + const CtGsCase gs_case = get_case(r, card); VecType c = set_one(r); - const size_t end = (len > 3) ? len - 3 : 0; const unsigned bufs_nb = buf.get_n(); const std::vector& mem = buf.get_mem(); for (unsigned i = start; i < bufs_nb; i += step) { - VecType x1, x2, x3, x4; - VecType y1, y2, y3, y4; VecType* p = reinterpret_cast(mem[i]); VecType* q = reinterpret_cast(mem[i + m]); - size_t j = 0; - for (; j < end; j += 4) { - x1 = load_to_reg(p + j); - x2 = load_to_reg(p + j + 1); - x3 = load_to_reg(p + j + 2); - x4 = load_to_reg(p + j + 3); - y1 = load_to_reg(q + j); - y2 = load_to_reg(q + j + 1); - y3 = load_to_reg(q + j + 2); - y4 = load_to_reg(q + j + 3); - - butterfly_gs(rp1, c, &x1, &y1, card); - butterfly_gs(rp1, c, &x2, &y2, card); - butterfly_gs(rp1, c, &x3, &y3, card); - butterfly_gs(rp1, c, &x4, &y4, card); - - // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(p + j + 1, x2); - store_to_mem(p + j + 2, x3); - store_to_mem(p + j + 3, x4); - store_to_mem(q + j, y1); - store_to_mem(q + j + 1, y2); - store_to_mem(q + j + 2, y3); - store_to_mem(q + j + 3, y4); - } - for (; j < len; ++j) { - x1 = load_to_reg(p + j); - y1 = load_to_reg(q + j); + for (size_t j = 0; j < len; ++j) { + VecType x1 = load_to_reg(p); + VecType y1 = load_to_reg(q); - butterfly_gs(rp1, c, &x1, &y1, card); + butterfly_gs(gs_case, c, x1, y1); - // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(q + j, y1); + store_to_mem(p++, x1); + store_to_mem(q++, y1); } } } @@ -429,37 +411,21 @@ inline void butterfly_gs_step_simple( return; } const unsigned step = m << 1; - const T rp1 = r + 1; + const CtGsCase gs_case = get_case(r, card); VecType c = set_one(r); - const size_t end = (len > 1) ? len - 1 : 0; const unsigned bufs_nb = buf.get_n(); const std::vector& mem = buf.get_mem(); for (unsigned i = start; i < bufs_nb; i += step) { - VecType x1, y1; - VecType x2, y2; VecType* p = reinterpret_cast(mem[i]); VecType* q = reinterpret_cast(mem[i + m]); - size_t j = 0; - for (; j < end; j += 2) { - x1 = load_to_reg(p + j); - x2 = load_to_reg(p + j + 1); + for (size_t j = 0; j < len; ++j) { + VecType x = load_to_reg(p++); - y1 = butterfly_simple_gs(rp1, c, x1, card); - y2 = butterfly_simple_gs(rp1, c, x2, card); + butterfly_simple_gs(gs_case, c, x); - // Store back to memory - store_to_mem(q + j, y1); - store_to_mem(q + j + 1, y2); - } - for (; j < len; ++j) { - x1 = load_to_reg(p + j); - - y1 = butterfly_simple_gs(rp1, c, x1, card); - - // Store back to memory - store_to_mem(q + j, y1); + store_to_mem(q++, x); } } } diff --git a/src/simd_ring.h b/src/simd_ring.h index 47edce99..a25176ac 100644 --- a/src/simd_ring.h +++ b/src/simd_ring.h @@ -56,13 +56,13 @@ inline void mul_coef_to_buf(const T a, T* src, T* dest, size_t len, T card) size_t i = 0; const size_t end = (_len > 3) ? _len - 3 : 0; for (; i < end; i += 4) { - _dest[i] = mod_mul(coef, _src[i], card); - _dest[i + 1] = mod_mul(coef, _src[i + 1], card); - _dest[i + 2] = mod_mul(coef, _src[i + 2], card); - _dest[i + 3] = mod_mul(coef, _src[i + 3], card); + _dest[i] = mod_mul(coef, _src[i]); + _dest[i + 1] = mod_mul(coef, _src[i + 1]); + _dest[i + 2] = mod_mul(coef, _src[i + 2]); + _dest[i + 3] = mod_mul(coef, _src[i + 3]); } for (; i < _len; ++i) { - _dest[i] = mod_mul(coef, _src[i], card); + _dest[i] = mod_mul(coef, _src[i]); } if (_last_len > 0) { @@ -84,7 +84,7 @@ inline void add_two_bufs(T* src, T* dest, size_t len, T card) size_t i; for (i = 0; i < _len; i++) { - _dest[i] = mod_add(_src[i], _dest[i], card); + _dest[i] = mod_add(_src[i], _dest[i]); } if (_last_len > 0) { for (i = _len * ratio; i < len; i++) { @@ -107,7 +107,7 @@ inline void sub_two_bufs(T* bufa, T* bufb, T* res, size_t len, T card) size_t i; for (i = 0; i < _len; i++) { // perform subtraction - _res[i] = mod_sub(_bufa[i], _bufb[i], card); + _res[i] = mod_sub(_bufa[i], _bufb[i]); } if (_last_len > 0) { for (i = _len * ratio; i < len; i++) { @@ -133,7 +133,7 @@ inline void mul_two_bufs(T* src, T* dest, size_t len, T card) size_t i; for (i = 0; i < _len; i++) { // perform multiplicaton - _dest[i] = mod_mul_safe(_src[i], _dest[i], card); + _dest[i] = mod_mul_safe(_src[i], _dest[i]); } if (_last_len > 0) { for (i = _len * ratio; i < len; i++) { @@ -155,7 +155,7 @@ inline void neg(size_t len, T* buf, T card) size_t i; for (i = 0; i < _len; i++) { - _buf[i] = mod_neg(_buf[i], card); + _buf[i] = mod_neg(_buf[i]); } if (_last_len > 0) { for (i = _len * ratio; i < len; i++) { diff --git a/test/ec_driver.cpp b/test/ec_driver.cpp index 0166f848..1661d88d 100644 --- a/test/ec_driver.cpp +++ b/test/ec_driver.cpp @@ -51,6 +51,13 @@ static void xusage() std::exit(EXIT_FAILURE); } +[[noreturn]] +static void xfnt_bad_word_size() +{ + std::cerr << "FEC using FNT supports word_size = 1 or 2\n"; + std::exit(EXIT_FAILURE); +} + [[noreturn]] static void xperror(const char* str) { @@ -582,36 +589,46 @@ int main(int argc, char** argv) data_zpad = count_digits(n_data - 1); if (eflag == EC_TYPE_RS_FNT) { - if (word_size <= 4) { - run_fec_rs_fnt( + switch (word_size) { + case 1: + run_fec_rs_fnt( word_size, n_data, n_parities, rflag, quadiron::fec::FecType::NON_SYSTEMATIC); - } else if (word_size <= 8) { - run_fec_rs_fnt( + break; + case 2: + run_fec_rs_fnt( word_size, n_data, n_parities, rflag, quadiron::fec::FecType::NON_SYSTEMATIC); + break; + default: + xfnt_bad_word_size(); } } else if (eflag == EC_TYPE_RS_FNT_SYS) { - if (word_size <= 4) { - run_fec_rs_fnt( + switch (word_size) { + case 1: + run_fec_rs_fnt( word_size, n_data, n_parities, rflag, quadiron::fec::FecType::SYSTEMATIC); - } else if (word_size <= 8) { - run_fec_rs_fnt( + break; + case 2: + run_fec_rs_fnt( word_size, n_data, n_parities, rflag, quadiron::fec::FecType::SYSTEMATIC); + break; + default: + xfnt_bad_word_size(); } } else if (eflag == EC_TYPE_RS_NF4) { if (word_size <= 2) { diff --git a/test/fec_utest.cpp b/test/fec_utest.cpp index 72e17a14..b0e97420 100644 --- a/test/fec_utest.cpp +++ b/test/fec_utest.cpp @@ -130,42 +130,41 @@ TYPED_TEST(FecTestCommon, TestGf2nFftAdd) // NOLINT } template -class FecTestNo128 : public FecTestCommon { +class FecTestFnt : public FecTestCommon { }; -using No128 = ::testing::Types; -TYPED_TEST_CASE(FecTestNo128, No128); +using FntType = ::testing::Types; +TYPED_TEST_CASE(FecTestFnt, FntType); -TYPED_TEST(FecTestNo128, TestFnt) // NOLINT +TYPED_TEST(FecTestFnt, TestFnt) // NOLINT { - for (unsigned word_size = 1; word_size <= 2; ++word_size) { - fec::RsFnt fec( - fec::FecType::NON_SYSTEMATIC, - word_size, - this->n_data, - this->n_parities); - this->run_test(fec, true); - } + const size_t word_size = sizeof(TypeParam) / 2; + fec::RsFnt fec( + fec::FecType::NON_SYSTEMATIC, + word_size, + this->n_data, + this->n_parities); + this->run_test(fec, true); } -TYPED_TEST(FecTestNo128, TestFntSys) // NOLINT +TYPED_TEST(FecTestFnt, TestFntSys) // NOLINT { - for (unsigned word_size = 1; word_size <= 2; ++word_size) { - fec::RsFnt fec( - fec::FecType::SYSTEMATIC, - word_size, - this->n_data, - this->n_parities); - this->run_test(fec, true); - } + const size_t word_size = sizeof(TypeParam) / 2; + fec::RsFnt fec( + fec::FecType::SYSTEMATIC, word_size, this->n_data, this->n_parities); + this->run_test(fec, true); } +template +class FecTestNo128 : public FecTestCommon { +}; + +using No128 = ::testing::Types; +TYPED_TEST_CASE(FecTestNo128, No128); + TYPED_TEST(FecTestNo128, TestGfpFft) // NOLINT { - for (size_t word_size = 1; word_size <= 4 && word_size < sizeof(TypeParam); - word_size *= 2) { - fec::RsGfpFft fec(word_size, this->n_data, this->n_parities); - - this->run_test(fec, true); - } + const size_t word_size = sizeof(TypeParam) / 2; + fec::RsGfpFft fec(word_size, this->n_data, this->n_parities); + this->run_test(fec, true); } diff --git a/test/quadiron_c_utest.cpp b/test/quadiron_c_utest.cpp index f87a0e38..7e6cb39e 100644 --- a/test/quadiron_c_utest.cpp +++ b/test/quadiron_c_utest.cpp @@ -127,12 +127,16 @@ class QuadironCTest : public ::testing::Test { } else { n_outputs = n_data + n_parities; } - std::vector> parity(n_outputs); - std::vector _parity(n_outputs); // for C API + std::vector> parity(n_parities); + std::vector _parity(n_parities); // for C API std::vector wanted_idxs(n_outputs); + std::vector> encoded_data(n_outputs); + std::vector _encoded_data(n_outputs); // for C API + + const size_t full_block_size = block_size + metadata_size; for (int i = 0; i < n_data; i++) { - data.at(i).resize(block_size + metadata_size); + data.at(i).resize(full_block_size); _data[i] = data.at(i).data(); ref_data.at(i).resize(block_size); _ref_data[i] = ref_data.at(i).data(); @@ -140,8 +144,8 @@ class QuadironCTest : public ::testing::Test { std::copy_n(_data[i] + metadata_size, block_size, _ref_data[i]); } - for (int i = 0; i < n_outputs; i++) { - parity.at(i).resize(block_size + metadata_size); + for (int i = 0; i < n_parities; i++) { + parity.at(i).resize(full_block_size); _parity[i] = parity.at(i).data(); } @@ -157,15 +161,34 @@ class QuadironCTest : public ::testing::Test { block_size), 0); + // get encoded data + for (int i = 0; i < n_outputs; i++) { + encoded_data.at(i).resize(full_block_size); + _encoded_data[i] = encoded_data.at(i).data(); + } + if (systematic) { + for (int i = 0; i < n_parities; i++) { + std::copy_n(_parity[i], full_block_size, _encoded_data[i]); + } + } else { + for (int i = 0; i < n_data; i++) { + std::copy_n(_data[i], full_block_size, _encoded_data[i]); + } + for (int i = 0; i < n_parities; i++) { + std::copy_n( + _parity[i], full_block_size, _encoded_data[i + n_data]); + } + } + + // delete missing fragments for (int i = 0; i < n_data; i++) { if (missing_idxs[i]) { - std::fill_n(_data[i], block_size + metadata_size, 0); + std::fill_n(_data[i], full_block_size, 0); } } - for (int i = 0; i < n_parities; i++) { if (missing_idxs[n_data + i]) { - std::fill_n(_parity[i], block_size + metadata_size, 0); + std::fill_n(_parity[i], full_block_size, 0); } } @@ -185,6 +208,25 @@ class QuadironCTest : public ::testing::Test { _data[i] + metadata_size)); } + // get back first n_data parities for the non-systematic case + if (!systematic) { + for (int i = 0; i < n_data; i++) { + std::copy_n(_encoded_data[i], full_block_size, _data[i]); + } + } + + // delete missing fragments + for (int i = 0; i < n_data; i++) { + if (missing_idxs[i]) { + std::fill_n(_data[i], full_block_size, 0); + } + } + for (int i = 0; i < n_parities; i++) { + if (missing_idxs[n_data + i]) { + std::fill_n(_parity[i], full_block_size, 0); + } + } + for (int i = 0; i < n_data; i++) { if (missing_idxs[i]) { ASSERT_EQ( @@ -196,6 +238,17 @@ class QuadironCTest : public ::testing::Test { i, block_size), 0); + if (systematic) { + ASSERT_TRUE(std::equal( + _ref_data[i], + _ref_data[i] + block_size, + _data[i] + metadata_size)); + } else { + ASSERT_TRUE(std::equal( + _encoded_data[i], + _encoded_data[i] + full_block_size, + _data[i])); + } } } @@ -210,6 +263,17 @@ class QuadironCTest : public ::testing::Test { n_data + i, block_size), 0); + if (systematic) { + ASSERT_TRUE(std::equal( + _encoded_data[i], + _encoded_data[i] + full_block_size, + _parity[i])); + } else { + ASSERT_TRUE(std::equal( + _encoded_data[n_data + i], + _encoded_data[n_data + i] + full_block_size, + _parity[i])); + } } }