From 7bc804965f8a0ba91e60c580749b6ce1d40e6b9a Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Wed, 12 Jun 2019 17:15:03 +0200 Subject: [PATCH 01/11] Vectorized essential operations: FNT supports only w=T/2 Some global variables are no longer used as we support only FNT for `w=T/2`, i.e. - FNT(257) using uint16_t - FNT(65537) using uint32_t To clarify reader, we use const reference for VecType variable if it's necessary. --- src/simd_128.h | 80 +++++++++++++++++++++++------------------------- src/simd_256.h | 82 +++++++++++++++++++++++--------------------------- 2 files changed, 75 insertions(+), 87 deletions(-) diff --git a/src/simd_128.h b/src/simd_128.h index 856588ec..7f034c2c 100644 --- a/src/simd_128.h +++ b/src/simd_128.h @@ -40,29 +40,23 @@ typedef __m128i VecType; /* ============= Constant variable ============ */ -// @note: using const leads to an lint error of initialization of 'variable' -// with static storage duration may throw an exception that cannot be caught - -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F4_U32 = _mm_set1_epi32(65537); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F4_MINUS_ONE_U32 = _mm_set1_epi32(65536); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_U32 = _mm_set1_epi32(257); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_MINUS_ONE_U32 = _mm_set1_epi32(256); - -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_U16 = _mm_set1_epi16(257); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_MINUS_ONE_U16 = _mm_set1_epi16(256); +template +inline VecType one(); +template <> +inline VecType one() +{ + return _mm_set1_epi16(1); +} +template <> +inline VecType one() +{ + return _mm_set1_epi32(1); +} -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ZERO = _mm_setzero_si128(); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ONE_U16 = _mm_set1_epi16(1); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ONE_U32 = _mm_set1_epi32(1); +inline VecType zero() +{ + return _mm_setzero_si128(); +} // NOLINTNEXTLINE(cert-err58-cpp) const VecType MASK8_LO = _mm_set1_epi16(0x80); @@ -78,25 +72,25 @@ inline void store_to_mem(VecType* address, VecType reg) _mm_store_si128(address, reg); } -inline VecType bit_and(VecType x, VecType y) +inline VecType bit_and(const VecType& x, const VecType& y) { return _mm_and_si128(x, y); } -inline VecType bit_xor(VecType x, VecType y) +inline VecType bit_xor(const VecType& x, const VecType& y) { return _mm_xor_si128(x, y); } -inline uint16_t msb8_mask(VecType x) +inline uint16_t msb8_mask(const VecType& x) { return _mm_movemask_epi8(x); } -inline bool and_is_zero(VecType x, VecType y) +inline bool and_is_zero(const VecType& x, const VecType& y) { return _mm_testz_si128(x, y); } -inline bool is_zero(VecType x) +inline bool is_zero(const VecType& x) { - return _mm_testc_si128(ZERO, x); + return _mm_testc_si128(zero(), x); } #define SHIFTR(x, imm8) (_mm_srli_si128(x, imm8)) @@ -119,66 +113,66 @@ inline VecType set_one(uint16_t val) } template -inline VecType add(VecType x, VecType y); +inline VecType add(const VecType& x, const VecType& y); template <> -inline VecType add(VecType x, VecType y) +inline VecType add(const VecType& x, const VecType& y) { return _mm_add_epi32(x, y); } template <> -inline VecType add(VecType x, VecType y) +inline VecType add(const VecType& x, const VecType& y) { return _mm_add_epi16(x, y); } template -inline VecType sub(VecType x, VecType y); +inline VecType sub(const VecType& x, const VecType& y); template <> -inline VecType sub(VecType x, VecType y) +inline VecType sub(const VecType& x, const VecType& y) { return _mm_sub_epi32(x, y); } template <> -inline VecType sub(VecType x, VecType y) +inline VecType sub(const VecType& x, const VecType& y) { return _mm_sub_epi16(x, y); } template -inline VecType mul(VecType x, VecType y); +inline VecType mul(const VecType& x, const VecType& y); template <> -inline VecType mul(VecType x, VecType y) +inline VecType mul(const VecType& x, const VecType& y) { return _mm_mullo_epi32(x, y); } template <> -inline VecType mul(VecType x, VecType y) +inline VecType mul(const VecType& x, const VecType& y) { return _mm_mullo_epi16(x, y); } template -inline VecType compare_eq(VecType x, VecType y); +inline VecType compare_eq(const VecType& x, const VecType& y); template <> -inline VecType compare_eq(VecType x, VecType y) +inline VecType compare_eq(const VecType& x, const VecType& y) { return _mm_cmpeq_epi32(x, y); } template <> -inline VecType compare_eq(VecType x, VecType y) +inline VecType compare_eq(const VecType& x, const VecType& y) { return _mm_cmpeq_epi16(x, y); } template -inline VecType min(VecType x, VecType y); +inline VecType min(const VecType& x, const VecType& y); template <> -inline VecType min(VecType x, VecType y) +inline VecType min(const VecType& x, const VecType& y) { return _mm_min_epu32(x, y); } template <> -inline VecType min(VecType x, VecType y) +inline VecType min(const VecType& x, const VecType& y) { return _mm_min_epu16(x, y); } diff --git a/src/simd_256.h b/src/simd_256.h index 8084d95e..7497cb32 100644 --- a/src/simd_256.h +++ b/src/simd_256.h @@ -53,29 +53,23 @@ typedef __m128i HalfVecType; /* ============= Constant variable ============ */ -// @note: using const leads to an lint error of initialization of 'variable' -// with static storage duration may throw an exception that cannot be caught - -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F4_U32 = _mm256_set1_epi32(65537); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F4_MINUS_ONE_U32 = _mm256_set1_epi32(65536); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_U32 = _mm256_set1_epi32(257); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_MINUS_ONE_U32 = _mm256_set1_epi32(256); - -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_U16 = _mm256_set1_epi16(257); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType F3_MINUS_ONE_U16 = _mm256_set1_epi16(256); +template +inline VecType one(); +template <> +inline VecType one() +{ + return _mm256_set1_epi16(1); +} +template <> +inline VecType one() +{ + return _mm256_set1_epi32(1); +} -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ZERO = _mm256_setzero_si256(); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ONE_U16 = _mm256_set1_epi16(1); -// NOLINTNEXTLINE(cert-err58-cpp) -const VecType ONE_U32 = _mm256_set1_epi32(1); +inline VecType zero() +{ + return _mm256_setzero_si256(); +} // NOLINTNEXTLINE(cert-err58-cpp) const VecType MASK8_LO = _mm256_set1_epi16(0x80); @@ -86,30 +80,30 @@ inline VecType load_to_reg(VecType* address) { return _mm256_load_si256(address); } -inline void store_to_mem(VecType* address, VecType reg) +inline void store_to_mem(VecType* address, const VecType& reg) { _mm256_store_si256(address, reg); } -inline VecType bit_and(VecType x, VecType y) +inline VecType bit_and(const VecType& x, const VecType& y) { return _mm256_and_si256(x, y); } -inline VecType bit_xor(VecType x, VecType y) +inline VecType bit_xor(const VecType& x, const VecType& y) { return _mm256_xor_si256(x, y); } -inline uint32_t msb8_mask(VecType x) +inline uint32_t msb8_mask(const VecType& x) { return _mm256_movemask_epi8(x); } -inline bool and_is_zero(VecType x, VecType y) +inline bool and_is_zero(const VecType& x, const VecType& y) { return _mm256_testz_si256(x, y); } -inline bool is_zero(VecType x) +inline bool is_zero(const VecType& x) { - return _mm256_testc_si256(ZERO, x); + return _mm256_testc_si256(zero(), x); } #define SHIFTR(x, imm8) (_mm256_srli_si256(x, imm8)) @@ -132,66 +126,66 @@ inline VecType set_one(uint16_t val) } template -inline VecType add(VecType x, VecType y); +inline VecType add(const VecType& x, const VecType& y); template <> -inline VecType add(VecType x, VecType y) +inline VecType add(const VecType& x, const VecType& y) { return _mm256_add_epi32(x, y); } template <> -inline VecType add(VecType x, VecType y) +inline VecType add(const VecType& x, const VecType& y) { return _mm256_add_epi16(x, y); } template -inline VecType sub(VecType x, VecType y); +inline VecType sub(const VecType& x, const VecType& y); template <> -inline VecType sub(VecType x, VecType y) +inline VecType sub(const VecType& x, const VecType& y) { return _mm256_sub_epi32(x, y); } template <> -inline VecType sub(VecType x, VecType y) +inline VecType sub(const VecType& x, const VecType& y) { return _mm256_sub_epi16(x, y); } template -inline VecType mul(VecType x, VecType y); +inline VecType mul(const VecType& x, const VecType& y); template <> -inline VecType mul(VecType x, VecType y) +inline VecType mul(const VecType& x, const VecType& y) { return _mm256_mullo_epi32(x, y); } template <> -inline VecType mul(VecType x, VecType y) +inline VecType mul(const VecType& x, const VecType& y) { return _mm256_mullo_epi16(x, y); } template -inline VecType compare_eq(VecType x, VecType y); +inline VecType compare_eq(const VecType& x, const VecType& y); template <> -inline VecType compare_eq(VecType x, VecType y) +inline VecType compare_eq(const VecType& x, const VecType& y) { return _mm256_cmpeq_epi32(x, y); } template <> -inline VecType compare_eq(VecType x, VecType y) +inline VecType compare_eq(const VecType& x, const VecType& y) { return _mm256_cmpeq_epi16(x, y); } template -inline VecType min(VecType x, VecType y); +inline VecType min(const VecType& x, const VecType& y); template <> -inline VecType min(VecType x, VecType y) +inline VecType min(const VecType& x, const VecType& y) { return _mm256_min_epu32(x, y); } template <> -inline VecType min(VecType x, VecType y) +inline VecType min(const VecType& x, const VecType& y) { return _mm256_min_epu16(x, y); } From 5c799eb9f432c27ad77af7b28bc21c4adda9cffc Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Wed, 12 Jun 2019 17:16:19 +0200 Subject: [PATCH 02/11] Vectorized modular arithmetics: FNT supports only w=T/2 The only support FNT with `w=T/2`, i.e. - FNT(257) using uint16_t - FNT(65537) using uint32_t gives advantages: - Operations are simplified by avoiding the argument cardinal. - Some branches can be avoided. --- src/simd_fnt.h | 83 ++++++++++++++++++++++++++++---------------------- 1 file changed, 46 insertions(+), 37 deletions(-) diff --git a/src/simd_fnt.h b/src/simd_fnt.h index 59e52410..a20471dd 100644 --- a/src/simd_fnt.h +++ b/src/simd_fnt.h @@ -37,42 +37,57 @@ namespace quadiron { namespace simd { template -inline VecType card(T q); +inline VecType card(); template <> -inline VecType card(uint16_t) +inline VecType card() { - return F3_U16; + return set_one(257); } template <> -inline VecType card(uint32_t q) +inline VecType card() { - return (q == F3) ? F3_U32 : F4_U32; + return set_one(65537); } template -inline VecType card_minus_one(T q); +inline VecType card_minus_one(); template <> -inline VecType card_minus_one(uint16_t) +inline VecType card_minus_one() { - return F3_MINUS_ONE_U16; + return set_one(256); } template <> -inline VecType card_minus_one(uint32_t q) +inline VecType card_minus_one() { - return (q == F3) ? F3_MINUS_ONE_U32 : F4_MINUS_ONE_U32; + return set_one(65536); } +const int I_MASK8_LO = 0b01010101; + template -inline VecType get_low_half(VecType x, T q) +inline VecType get_low_half(const VecType& x); +template <> +inline VecType get_low_half(const VecType& x) +{ + return BLEND8(zero(), x, MASK8_LO); +} +template <> +inline VecType get_low_half(const VecType& x) { - return (q == F3) ? BLEND8(ZERO, x, MASK8_LO) : BLEND16(ZERO, x, 0x55); + return BLEND16(zero(), x, I_MASK8_LO); } template -inline VecType get_high_half(VecType x, T q) +inline VecType get_high_half(const VecType& x); +template <> +inline VecType get_high_half(const VecType& x) +{ + return BLEND8(zero(), SHIFTR(x, 1), MASK8_LO); +} +template <> +inline VecType get_high_half(const VecType& x) { - return (q == F3) ? BLEND8(ZERO, SHIFTR(x, 1), MASK8_LO) - : BLEND16(ZERO, SHIFTR(x, 2), 0x55); + return BLEND16(zero(), SHIFTR(x, 2), I_MASK8_LO); } /* ================= Basic Operations ================= */ @@ -82,14 +97,13 @@ inline VecType get_high_half(VecType x, T q) * * @param x input register * @param y input register - * @param q modulo * @return (x + y) mod q */ template -inline VecType mod_add(VecType x, VecType y, T q) +inline VecType mod_add(const VecType& x, const VecType& y) { const VecType res = add(x, y); - return min(res, sub(res, card(q))); + return min(res, sub(res, card())); } /** @@ -97,28 +111,26 @@ inline VecType mod_add(VecType x, VecType y, T q) * * @param x input register * @param y input register - * @param q modulo * @return (x - y) mod q */ template -inline VecType mod_sub(VecType x, VecType y, T q) +inline VecType mod_sub(const VecType& x, const VecType& y) { const VecType res = sub(x, y); - return min(res, add(res, card(q))); + return min(res, add(res, card())); } /** * Modular negation for packed unsigned 32-bit integers * * @param x input register - * @param q modulo * @return (-x) mod q */ template -inline VecType mod_neg(VecType x, T q) +inline VecType mod_neg(const VecType& x) { - const VecType res = sub(card(q), x); - return min(res, sub(res, card(q))); + const VecType res = sub(card(), x); + return min(res, sub(res, card())); } /** @@ -129,16 +141,15 @@ inline VecType mod_neg(VecType x, T q) * * @param x input register * @param y input register - * @param q modulo * @return (x * y) mod q */ template -inline VecType mod_mul(VecType x, VecType y, T q) +inline VecType mod_mul(const VecType& x, const VecType& y) { const VecType res = mul(x, y); - const VecType lo = get_low_half(res, q); - const VecType hi = get_high_half(res, q); - return mod_sub(lo, hi, q); + const VecType lo = get_low_half(res); + const VecType hi = get_high_half(res); + return mod_sub(lo, hi); } /** @@ -148,24 +159,22 @@ inline VecType mod_mul(VecType x, VecType y, T q) * * @param x input register * @param y input register - * @param q modulo * @return (x * y) mod q */ template -inline VecType mod_mul_safe(VecType x, VecType y, T q) +inline VecType mod_mul_safe(const VecType& x, const VecType& y) { - const VecType res = mod_mul(x, y, q); + const VecType res = mod_mul(x, y); // filter elements of both of a & b = card-1 const VecType cmp = bit_and( - compare_eq(x, card_minus_one(q)), - compare_eq(y, card_minus_one(q))); + compare_eq(x, card_minus_one()), + compare_eq(y, card_minus_one())); if (is_zero(cmp)) { return res; } - return (q == F3) ? bit_xor(res, bit_and(F4_U32, cmp)) - : add(res, bit_and(ONE_U32, cmp)); + return add(res, bit_and(one(), cmp)); } /** From 3f0f4c562323f61dcdbd353f21e1c0c85ccb4364 Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Wed, 12 Jun 2019 17:29:06 +0200 Subject: [PATCH 03/11] [SIMD] More readable in Butterfly operations In butterfly operations, there are three cases depending on the coefficient `r`: - r = 1 - r = q - 1 - 1 < r < q - 1 We use an enum class to clarify such cases. --- src/simd_radix2_fft.h | 302 +++++++++++++++++++----------------------- 1 file changed, 134 insertions(+), 168 deletions(-) diff --git a/src/simd_radix2_fft.h b/src/simd_radix2_fft.h index 8405f8f0..66dafc17 100644 --- a/src/simd_radix2_fft.h +++ b/src/simd_radix2_fft.h @@ -36,6 +36,24 @@ namespace quadiron { namespace simd { +enum class CtGsCase { + SIMPLE, + NORMAL, + EXTREME, +}; + +template +constexpr CtGsCase get_case(T r, T q) +{ + if (r == 1) { + return CtGsCase::SIMPLE; + } else if (r < q - 1) { + return CtGsCase::NORMAL; + } else { + return CtGsCase::EXTREME; + } +} + /* ================= Vectorized Operations ================= */ /** @@ -44,22 +62,30 @@ namespace simd { * x <- x + r * y * y <- x - r * y * - * @param rp1 coefficient `r` plus one + * @param ct_case coefficient case * @param c a register stores coefficient `r` * @param x working register * @param y working register - * @param q modular */ template -inline void butterfly_ct(T rp1, VecType c, VecType* x, VecType* y, T q) +inline void +butterfly_ct(CtGsCase ct_case, const VecType& c, VecType& x, VecType& y) { - VecType z = (rp1 == 2) ? *y : mod_mul(c, *y, q); - if (rp1 < q) { - *y = mod_sub(*x, z, q); - *x = mod_add(*x, z, q); - } else { // i.e. r == q - 1 - *y = mod_add(*x, z, q); - *x = mod_sub(*x, z, q); + VecType z = y; + switch (ct_case) { + case CtGsCase::SIMPLE: + y = mod_sub(x, z); + x = mod_add(x, z); + break; + case CtGsCase::EXTREME: + y = mod_add(x, z); + x = mod_sub(x, z); + break; + case CtGsCase::NORMAL: + z = mod_mul(c, y); + y = mod_sub(x, z); + x = mod_add(x, z); + break; } } @@ -69,25 +95,29 @@ inline void butterfly_ct(T rp1, VecType c, VecType* x, VecType* y, T q) * x <- x + y * y <- r * (x - y) * - * @param rp1 coefficient `r` plus one + * @param gs_case coefficient case * @param c a register stores coefficient `r` * @param x working register * @param y working register - * @param q modular */ template -inline void butterfly_gs(T rp1, VecType c, VecType* x, VecType* y, T q) +inline void +butterfly_gs(CtGsCase gs_case, const VecType& c, VecType& x, VecType& y) { - VecType add = mod_add(*x, *y, q); - if (rp1 == 2) { - *y = mod_sub(*x, *y, q); - } else if (rp1 < q) { - VecType sub = mod_sub(*x, *y, q); - *y = mod_mul(c, sub, q); - } else { // i.e. r == q - 1 - *y = mod_sub(*y, *x, q); + VecType add = mod_add(x, y); + switch (gs_case) { + case CtGsCase::SIMPLE: + y = mod_sub(x, y); + break; + case CtGsCase::EXTREME: + y = mod_sub(y, x); + break; + case CtGsCase::NORMAL: + VecType sub = mod_sub(x, y); + y = mod_mul(c, sub); + break; } - *x = add; + x = add; } /** @@ -96,24 +126,24 @@ inline void butterfly_gs(T rp1, VecType c, VecType* x, VecType* y, T q) * x <- x, i.e. no operation * y <- r * x * - * @param rp1 coefficient `r` plus one + * @param gs_case coefficient case * @param c a register stores coefficient `r` * @param x working register - * @param q modular - * @return r * x */ template -inline VecType butterfly_simple_gs(T rp1, VecType c, VecType x, T q) +inline void butterfly_simple_gs(CtGsCase gs_case, const VecType& c, VecType& x) { - if (rp1 == 2) { - return x; - } else if (rp1 < q) { - return mod_mul(c, x, q); - } else { - return mod_neg(x, q); + switch (gs_case) { + case CtGsCase::EXTREME: + x = mod_neg(x); + break; + case CtGsCase::NORMAL: + x = mod_mul(c, x); + break; + case CtGsCase::SIMPLE: + break; } } - /** * Vectorized butterfly CT step * @@ -139,54 +169,49 @@ inline void butterfly_ct_step( size_t len, T card) { - if (len == 0) { - return; - } - const T rp1 = r + 1; - VecType c = set_one(r); + const CtGsCase ct_case = get_case(r, card); + const VecType c = set_one(r); const size_t end = (len > 1) ? len - 1 : 0; const unsigned bufs_nb = buf.get_n(); const std::vector& mem = buf.get_mem(); for (unsigned i = start; i < bufs_nb; i += step) { - VecType x1, y1; - VecType x2, y2; VecType* p = reinterpret_cast(mem[i]); VecType* q = reinterpret_cast(mem[i + m]); size_t j = 0; for (; j < end; j += 2) { - x1 = load_to_reg(p + j); - y1 = load_to_reg(q + j); + VecType x1 = load_to_reg(p); + VecType y1 = load_to_reg(q); - butterfly_ct(rp1, c, &x1, &y1, card); + butterfly_ct(ct_case, c, x1, y1); - x2 = load_to_reg(p + j + 1); - y2 = load_to_reg(q + j + 1); + VecType x2 = load_to_reg(p + 1); + VecType y2 = load_to_reg(q + 1); - butterfly_ct(rp1, c, &x2, &y2, card); + butterfly_ct(ct_case, c, x2, y2); // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(p + j + 1, x2); - store_to_mem(q + j, y1); - store_to_mem(q + j + 1, y2); + store_to_mem(p++, x1); + store_to_mem(p++, x2); + store_to_mem(q++, y1); + store_to_mem(q++, y2); } for (; j < len; ++j) { - x1 = load_to_reg(p + j); - y1 = load_to_reg(q + j); + VecType x1 = load_to_reg(p); + VecType y1 = load_to_reg(q); - butterfly_ct(rp1, c, &x1, &y1, card); + butterfly_ct(ct_case, c, x1, y1); // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(q + j, y1); + store_to_mem(p++, x1); + store_to_mem(q++, y1); } } } template -inline static void do_butterfly_ct_2_layers( +inline void do_butterfly_ct_2_layers( const std::vector& mem, T r1, T r2, @@ -196,9 +221,9 @@ inline static void do_butterfly_ct_2_layers( size_t len, T card) { - const T r1p1 = r1 + 1; - const T r2p1 = r2 + 1; - const T r3p1 = r3 + 1; + const CtGsCase case1 = get_case(r1, card); + const CtGsCase case2 = get_case(r2, card); + const CtGsCase case3 = get_case(r3, card); VecType c1 = set_one(r1); VecType c2 = set_one(r2); @@ -212,65 +237,52 @@ inline static void do_butterfly_ct_2_layers( size_t j = 0; const size_t end = (len > 1) ? len - 1 : 0; while (j < end) { - // First layer (c1, x, y) & (c1, u, v) VecType x1 = load_to_reg(p); - VecType x2 = load_to_reg(p + 1); VecType y1 = load_to_reg(q); - VecType y2 = load_to_reg(q + 1); + VecType u1 = load_to_reg(r); + VecType v1 = load_to_reg(s); - butterfly_ct(r1p1, c1, &x1, &y1, card); - butterfly_ct(r1p1, c1, &x2, &y2, card); + butterfly_ct(case1, c1, x1, y1); + butterfly_ct(case1, c1, u1, v1); + butterfly_ct(case2, c2, x1, u1); + butterfly_ct(case3, c3, y1, v1); - VecType u1 = load_to_reg(r); + VecType x2 = load_to_reg(p + 1); + VecType y2 = load_to_reg(q + 1); VecType u2 = load_to_reg(r + 1); - VecType v1 = load_to_reg(s); VecType v2 = load_to_reg(s + 1); - butterfly_ct(r1p1, c1, &u1, &v1, card); - butterfly_ct(r1p1, c1, &u2, &v2, card); - - // Second layer (c2, x, u) & (c3, y, v) - butterfly_ct(r2p1, c2, &x1, &u1, card); - butterfly_ct(r2p1, c2, &x2, &u2, card); - - butterfly_ct(r3p1, c3, &y1, &v1, card); - butterfly_ct(r3p1, c3, &y2, &v2, card); - - // Store back to memory - store_to_mem(p, x1); - store_to_mem(p + 1, x2); - store_to_mem(q, y1); - store_to_mem(q + 1, y2); - - store_to_mem(r, u1); - store_to_mem(r + 1, u2); - store_to_mem(s, v1); - store_to_mem(s + 1, v2); - p = p + 2; - q = q + 2; - r = r + 2; - s = s + 2; - j = j + 2; - }; - + butterfly_ct(case1, c1, x2, y2); + butterfly_ct(case1, c1, u2, v2); + butterfly_ct(case2, c2, x2, u2); + butterfly_ct(case3, c3, y2, v2); + + store_to_mem(p++, x1); + store_to_mem(p++, x2); + store_to_mem(q++, y1); + store_to_mem(q++, y2); + store_to_mem(r++, u1); + store_to_mem(r++, u2); + store_to_mem(s++, v1); + store_to_mem(s++, v2); + + j += 2; + } for (; j < len; ++j) { - // First layer (c1, x, y) & (c1, u, v) - VecType x1 = load_to_reg(p + j); - VecType y1 = load_to_reg(q + j); - VecType u1 = load_to_reg(r + j); - VecType v1 = load_to_reg(s + j); - - // BUTTERFLY_3_test(c1, &x1, &y1, &u1, &v1, card); - butterfly_ct(r1p1, c1, &x1, &y1, card); - butterfly_ct(r1p1, c1, &u1, &v1, card); - butterfly_ct(r2p1, c2, &x1, &u1, card); - butterfly_ct(r3p1, c3, &y1, &v1, card); - - // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(q + j, y1); - store_to_mem(r + j, u1); - store_to_mem(s + j, v1); + VecType x1 = load_to_reg(p); + VecType y1 = load_to_reg(q); + VecType u1 = load_to_reg(r); + VecType v1 = load_to_reg(s); + + butterfly_ct(case1, c1, x1, y1); + butterfly_ct(case1, c1, u1, v1); + butterfly_ct(case2, c2, x1, u1); + butterfly_ct(case3, c3, y1, v1); + + store_to_mem(p++, x1); + store_to_mem(q++, y1); + store_to_mem(r++, u1); + store_to_mem(s++, v1); } } @@ -352,53 +364,23 @@ inline void butterfly_gs_step( return; } const unsigned step = m << 1; - const T rp1 = r + 1; + const CtGsCase gs_case = get_case(r, card); VecType c = set_one(r); - const size_t end = (len > 3) ? len - 3 : 0; const unsigned bufs_nb = buf.get_n(); const std::vector& mem = buf.get_mem(); for (unsigned i = start; i < bufs_nb; i += step) { - VecType x1, x2, x3, x4; - VecType y1, y2, y3, y4; VecType* p = reinterpret_cast(mem[i]); VecType* q = reinterpret_cast(mem[i + m]); - size_t j = 0; - for (; j < end; j += 4) { - x1 = load_to_reg(p + j); - x2 = load_to_reg(p + j + 1); - x3 = load_to_reg(p + j + 2); - x4 = load_to_reg(p + j + 3); - y1 = load_to_reg(q + j); - y2 = load_to_reg(q + j + 1); - y3 = load_to_reg(q + j + 2); - y4 = load_to_reg(q + j + 3); - - butterfly_gs(rp1, c, &x1, &y1, card); - butterfly_gs(rp1, c, &x2, &y2, card); - butterfly_gs(rp1, c, &x3, &y3, card); - butterfly_gs(rp1, c, &x4, &y4, card); - - // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(p + j + 1, x2); - store_to_mem(p + j + 2, x3); - store_to_mem(p + j + 3, x4); - store_to_mem(q + j, y1); - store_to_mem(q + j + 1, y2); - store_to_mem(q + j + 2, y3); - store_to_mem(q + j + 3, y4); - } - for (; j < len; ++j) { - x1 = load_to_reg(p + j); - y1 = load_to_reg(q + j); + for (size_t j = 0; j < len; ++j) { + VecType x1 = load_to_reg(p); + VecType y1 = load_to_reg(q); - butterfly_gs(rp1, c, &x1, &y1, card); + butterfly_gs(gs_case, c, x1, y1); - // Store back to memory - store_to_mem(p + j, x1); - store_to_mem(q + j, y1); + store_to_mem(p++, x1); + store_to_mem(q++, y1); } } } @@ -429,37 +411,21 @@ inline void butterfly_gs_step_simple( return; } const unsigned step = m << 1; - const T rp1 = r + 1; + const CtGsCase gs_case = get_case(r, card); VecType c = set_one(r); - const size_t end = (len > 1) ? len - 1 : 0; const unsigned bufs_nb = buf.get_n(); const std::vector& mem = buf.get_mem(); for (unsigned i = start; i < bufs_nb; i += step) { - VecType x1, y1; - VecType x2, y2; VecType* p = reinterpret_cast(mem[i]); VecType* q = reinterpret_cast(mem[i + m]); - size_t j = 0; - for (; j < end; j += 2) { - x1 = load_to_reg(p + j); - x2 = load_to_reg(p + j + 1); + for (size_t j = 0; j < len; ++j) { + VecType x = load_to_reg(p++); - y1 = butterfly_simple_gs(rp1, c, x1, card); - y2 = butterfly_simple_gs(rp1, c, x2, card); + butterfly_simple_gs(gs_case, c, x); - // Store back to memory - store_to_mem(q + j, y1); - store_to_mem(q + j + 1, y2); - } - for (; j < len; ++j) { - x1 = load_to_reg(p + j); - - y1 = butterfly_simple_gs(rp1, c, x1, card); - - // Store back to memory - store_to_mem(q + j, y1); + store_to_mem(q++, x); } } } From f0bcce76e2657c38de89327c2a5e38b1254fbcd2 Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Wed, 12 Jun 2019 17:30:02 +0200 Subject: [PATCH 04/11] Update vectorized nf4 and RingModN operations according changes in modular arithmetics --- src/simd_nf4.h | 29 +++++++++++++++-------------- src/simd_ring.h | 18 +++++++++--------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/src/simd_nf4.h b/src/simd_nf4.h index f9ec1172..06a46ebf 100644 --- a/src/simd_nf4.h +++ b/src/simd_nf4.h @@ -180,7 +180,7 @@ inline __uint128_t add(__uint128_t a, __uint128_t b) HalfVecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_low_half_to_mem(&res, mod_add(vec_a, vec_b, F4)); + store_low_half_to_mem(&res, mod_add(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -189,7 +189,7 @@ inline __uint128_t sub(__uint128_t a, __uint128_t b) HalfVecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_low_half_to_mem(&res, mod_sub(vec_a, vec_b, F4)); + store_low_half_to_mem(&res, mod_sub(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -198,7 +198,7 @@ inline __uint128_t mul(__uint128_t a, __uint128_t b) HalfVecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_low_half_to_mem(&res, mod_mul_safe(vec_a, vec_b, F4)); + store_low_half_to_mem(&res, mod_mul_safe(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -217,8 +217,8 @@ inline void add_buf_to_two_bufs_rem( VecType _x_next_p = load_to_reg(_x_half[i]); VecType _y_p = load_to_reg(_y[i]); - store_low_half_to_mem(_x + i, mod_add(_x_p, _y_p, F4)); - store_low_half_to_mem(_x_half + i, mod_add(_x_next_p, _y_p, F4)); + store_low_half_to_mem(_x + i, mod_add(_x_p, _y_p)); + store_low_half_to_mem(_x_half + i, mod_add(_x_next_p, _y_p)); } } @@ -230,7 +230,7 @@ inline void hadamard_mul_rem(unsigned n, __uint128_t* x, __uint128_t* y) VecType _x_p = load_to_reg(_x[i]); VecType _y_p = load_to_reg(_y[i]); - store_low_half_to_mem(_x + i, mod_mul_safe(_x_p, _y_p, F4)); + store_low_half_to_mem(_x + i, mod_mul_safe(_x_p, _y_p)); } } @@ -248,8 +248,9 @@ inline void hadamard_mul_doubled_rem( VecType _x_next_p = load_to_reg(_x_half[i]); VecType _y_p = load_to_reg(_y[i]); - store_low_half_to_mem(_x + i, mod_mul_safe(_x_p, _y_p, F4)); - store_low_half_to_mem(_x_half + i, mod_mul_safe(_x_next_p, _y_p, F4)); + store_low_half_to_mem(_x + i, mod_mul_safe(_x_p, _y_p)); + store_low_half_to_mem( + _x_half + i, mod_mul_safe(_x_next_p, _y_p)); } } @@ -266,7 +267,7 @@ inline __uint128_t add(__uint128_t a, __uint128_t b) VecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_to_mem(&res, mod_add(vec_a, vec_b, F4)); + store_to_mem(&res, mod_add(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -275,7 +276,7 @@ inline __uint128_t sub(__uint128_t a, __uint128_t b) VecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_to_mem(&res, mod_sub(vec_a, vec_b, F4)); + store_to_mem(&res, mod_sub(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -284,7 +285,7 @@ inline __uint128_t mul(__uint128_t a, __uint128_t b) VecType res; VecType vec_a = load_to_reg(a); VecType vec_b = load_to_reg(b); - store_to_mem(&res, mod_mul_safe(vec_a, vec_b, F4)); + store_to_mem(&res, mod_mul_safe(vec_a, vec_b)); return reinterpret_cast<__uint128_t>(res); } @@ -327,12 +328,12 @@ inline void add_buf_to_two_bufs(unsigned n, __uint128_t* _x, __uint128_t* _y) // add y to the first half of `x` for (i = 0; i < vec_len; ++i) { - x[i] = mod_add(x[i], y[i], F4); + x[i] = mod_add(x[i], y[i]); } // add y to the second half of `x` for (i = 0; i < vec_len; ++i) { - x_next[i] = mod_add(x_next[i], y[i], F4); + x_next[i] = mod_add(x_next[i], y[i]); } if (rem_len > 0) { @@ -354,7 +355,7 @@ inline void hadamard_mul(unsigned n, __uint128_t* _x, __uint128_t* _y) // multiply y to the first half of `x` for (i = 0; i < vec_len; ++i) { - x[i] = mod_mul_safe(x[i], y[i], F4); + x[i] = mod_mul_safe(x[i], y[i]); } if (rem_len > 0) { diff --git a/src/simd_ring.h b/src/simd_ring.h index 47edce99..a25176ac 100644 --- a/src/simd_ring.h +++ b/src/simd_ring.h @@ -56,13 +56,13 @@ inline void mul_coef_to_buf(const T a, T* src, T* dest, size_t len, T card) size_t i = 0; const size_t end = (_len > 3) ? _len - 3 : 0; for (; i < end; i += 4) { - _dest[i] = mod_mul(coef, _src[i], card); - _dest[i + 1] = mod_mul(coef, _src[i + 1], card); - _dest[i + 2] = mod_mul(coef, _src[i + 2], card); - _dest[i + 3] = mod_mul(coef, _src[i + 3], card); + _dest[i] = mod_mul(coef, _src[i]); + _dest[i + 1] = mod_mul(coef, _src[i + 1]); + _dest[i + 2] = mod_mul(coef, _src[i + 2]); + _dest[i + 3] = mod_mul(coef, _src[i + 3]); } for (; i < _len; ++i) { - _dest[i] = mod_mul(coef, _src[i], card); + _dest[i] = mod_mul(coef, _src[i]); } if (_last_len > 0) { @@ -84,7 +84,7 @@ inline void add_two_bufs(T* src, T* dest, size_t len, T card) size_t i; for (i = 0; i < _len; i++) { - _dest[i] = mod_add(_src[i], _dest[i], card); + _dest[i] = mod_add(_src[i], _dest[i]); } if (_last_len > 0) { for (i = _len * ratio; i < len; i++) { @@ -107,7 +107,7 @@ inline void sub_two_bufs(T* bufa, T* bufb, T* res, size_t len, T card) size_t i; for (i = 0; i < _len; i++) { // perform subtraction - _res[i] = mod_sub(_bufa[i], _bufb[i], card); + _res[i] = mod_sub(_bufa[i], _bufb[i]); } if (_last_len > 0) { for (i = _len * ratio; i < len; i++) { @@ -133,7 +133,7 @@ inline void mul_two_bufs(T* src, T* dest, size_t len, T card) size_t i; for (i = 0; i < _len; i++) { // perform multiplicaton - _dest[i] = mod_mul_safe(_src[i], _dest[i], card); + _dest[i] = mod_mul_safe(_src[i], _dest[i]); } if (_last_len > 0) { for (i = _len * ratio; i < len; i++) { @@ -155,7 +155,7 @@ inline void neg(size_t len, T* buf, T card) size_t i; for (i = 0; i < _len; i++) { - _buf[i] = mod_neg(_buf[i], card); + _buf[i] = mod_neg(_buf[i]); } if (_last_len > 0) { for (i = _len * ratio; i < len; i++) { From 9437297e24fa320b677702e1df9bdf6f3977590f Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Wed, 12 Jun 2019 17:31:06 +0200 Subject: [PATCH 05/11] Update FecTest: FecFnt supports only w=T/2 --- test/fec_utest.cpp | 53 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/test/fec_utest.cpp b/test/fec_utest.cpp index 72e17a14..b0e97420 100644 --- a/test/fec_utest.cpp +++ b/test/fec_utest.cpp @@ -130,42 +130,41 @@ TYPED_TEST(FecTestCommon, TestGf2nFftAdd) // NOLINT } template -class FecTestNo128 : public FecTestCommon { +class FecTestFnt : public FecTestCommon { }; -using No128 = ::testing::Types; -TYPED_TEST_CASE(FecTestNo128, No128); +using FntType = ::testing::Types; +TYPED_TEST_CASE(FecTestFnt, FntType); -TYPED_TEST(FecTestNo128, TestFnt) // NOLINT +TYPED_TEST(FecTestFnt, TestFnt) // NOLINT { - for (unsigned word_size = 1; word_size <= 2; ++word_size) { - fec::RsFnt fec( - fec::FecType::NON_SYSTEMATIC, - word_size, - this->n_data, - this->n_parities); - this->run_test(fec, true); - } + const size_t word_size = sizeof(TypeParam) / 2; + fec::RsFnt fec( + fec::FecType::NON_SYSTEMATIC, + word_size, + this->n_data, + this->n_parities); + this->run_test(fec, true); } -TYPED_TEST(FecTestNo128, TestFntSys) // NOLINT +TYPED_TEST(FecTestFnt, TestFntSys) // NOLINT { - for (unsigned word_size = 1; word_size <= 2; ++word_size) { - fec::RsFnt fec( - fec::FecType::SYSTEMATIC, - word_size, - this->n_data, - this->n_parities); - this->run_test(fec, true); - } + const size_t word_size = sizeof(TypeParam) / 2; + fec::RsFnt fec( + fec::FecType::SYSTEMATIC, word_size, this->n_data, this->n_parities); + this->run_test(fec, true); } +template +class FecTestNo128 : public FecTestCommon { +}; + +using No128 = ::testing::Types; +TYPED_TEST_CASE(FecTestNo128, No128); + TYPED_TEST(FecTestNo128, TestGfpFft) // NOLINT { - for (size_t word_size = 1; word_size <= 4 && word_size < sizeof(TypeParam); - word_size *= 2) { - fec::RsGfpFft fec(word_size, this->n_data, this->n_parities); - - this->run_test(fec, true); - } + const size_t word_size = sizeof(TypeParam) / 2; + fec::RsGfpFft fec(word_size, this->n_data, this->n_parities); + this->run_test(fec, true); } From 2792ac05aa8981b5e85fe0b93a1125cef4f50469 Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Wed, 12 Jun 2019 17:31:46 +0200 Subject: [PATCH 06/11] Update ecdriver: FecFnt supports only w=T/2 --- test/ec_driver.cpp | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/test/ec_driver.cpp b/test/ec_driver.cpp index 0166f848..1661d88d 100644 --- a/test/ec_driver.cpp +++ b/test/ec_driver.cpp @@ -51,6 +51,13 @@ static void xusage() std::exit(EXIT_FAILURE); } +[[noreturn]] +static void xfnt_bad_word_size() +{ + std::cerr << "FEC using FNT supports word_size = 1 or 2\n"; + std::exit(EXIT_FAILURE); +} + [[noreturn]] static void xperror(const char* str) { @@ -582,36 +589,46 @@ int main(int argc, char** argv) data_zpad = count_digits(n_data - 1); if (eflag == EC_TYPE_RS_FNT) { - if (word_size <= 4) { - run_fec_rs_fnt( + switch (word_size) { + case 1: + run_fec_rs_fnt( word_size, n_data, n_parities, rflag, quadiron::fec::FecType::NON_SYSTEMATIC); - } else if (word_size <= 8) { - run_fec_rs_fnt( + break; + case 2: + run_fec_rs_fnt( word_size, n_data, n_parities, rflag, quadiron::fec::FecType::NON_SYSTEMATIC); + break; + default: + xfnt_bad_word_size(); } } else if (eflag == EC_TYPE_RS_FNT_SYS) { - if (word_size <= 4) { - run_fec_rs_fnt( + switch (word_size) { + case 1: + run_fec_rs_fnt( word_size, n_data, n_parities, rflag, quadiron::fec::FecType::SYSTEMATIC); - } else if (word_size <= 8) { - run_fec_rs_fnt( + break; + case 2: + run_fec_rs_fnt( word_size, n_data, n_parities, rflag, quadiron::fec::FecType::SYSTEMATIC); + break; + default: + xfnt_bad_word_size(); } } else if (eflag == EC_TYPE_RS_NF4) { if (word_size <= 2) { From a63a6bbf9781358d30e527a7cbe858cd666736fc Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Wed, 12 Jun 2019 17:42:45 +0200 Subject: [PATCH 07/11] [CI] Update benchmark script: FNT supports only w=T/2 --- scripts/benchmark.sh | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh index 9658c255..a95c8150 100755 --- a/scripts/benchmark.sh +++ b/scripts/benchmark.sh @@ -44,24 +44,22 @@ fi chunk_size=51200 # for rs-fnt with different packet sizes -word_size=2 for word_size in 1 2; do - for type_size in 2 4; do - max_len=$((256**word_size)) - if ((type_size>word_size)); then - for ec_type in rs-fnt rs-fnt-sys; do - for k in 16 64; do - for n in 32 256 1024; do - if ((nk)); then - m=$((n-k)) - for pkt_size in 512; do - ${bin} -e ${ec_type} -w ${word_size} -t ${type_size} -k ${k} -m ${m} -c ${chunk_size} -s ${sce_type} -g ${threads_nb} -f ${show_type} -p ${pkt_size} -n ${samples_nb} - show_type=0 - done - fi + type_size=$((word_size*2)) + max_len=$((256**word_size)) + if ((type_size>word_size)); then + for ec_type in rs-fnt rs-fnt-sys; do + for k in 16 64; do + for n in 32 256 1024; do + if ((nk)); then + m=$((n-k)) + for pkt_size in 32 64 128 256 512 1024; do + ${bin} -e ${ec_type} -w ${word_size} -t ${type_size} -k ${k} -m ${m} -c ${chunk_size} -s ${sce_type} -g ${threads_nb} -f ${show_type} -p ${pkt_size} -n ${samples_nb} + show_type=0 done - done + fi done - fi - done + done + done + fi done From 7c26d5cd11ccbfe95e7b14535d42459c3bfd3b26 Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Mon, 27 May 2019 14:47:13 +0200 Subject: [PATCH 08/11] Remove long codelength in ec_driver's test --- scripts/test_ec.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/test_ec.sh b/scripts/test_ec.sh index 4b133150..4b7e2b47 100755 --- a/scripts/test_ec.sh +++ b/scripts/test_ec.sh @@ -174,7 +174,6 @@ do fec_type=$(echo $i|cut -d_ -f1) word_size=$(echo $i|cut -d_ -f2) - do_test enconly ${fec_type} ${word_size} 50 50 "" "" do_test all ${fec_type} ${word_size} 3 3 "" "" do_test all ${fec_type} ${word_size} 3 3 "0 1" "0" do_test all ${fec_type} ${word_size} 3 5 "0 1" "0" From 00e7637cad728bc41e089555f9d6f366d4ebc27c Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Thu, 13 Jun 2019 11:19:42 +0200 Subject: [PATCH 09/11] Add link time optimization flag --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8879b400..1f0937f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -191,7 +191,7 @@ endif() # Release flags set(RELEASE_CXX_FLAGS - -O3 -DNDEBUG + -O3 -DNDEBUG -flto=full ) foreach(flag ${RELEASE_CXX_FLAGS}) check_cxx_compiler_flag(${flag} has_flag_${flag}) From 321c7c9e8d1105e68f7b0fa660fbf30560cc59aa Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Fri, 14 Jun 2019 05:20:20 +0200 Subject: [PATCH 10/11] QuadIron C API: make it safer - Reset metadata of decoded data: for non-systematic code, we use first k parities to store decoded data. These metadata should be reset. - Remove an useless initialisation of data_vec for non-systematic codes. --- src/quadiron_c.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/quadiron_c.cpp b/src/quadiron_c.cpp index 82542445..3400eebd 100644 --- a/src/quadiron_c.cpp +++ b/src/quadiron_c.cpp @@ -215,6 +215,16 @@ int quadiron_fnt32_decode( if (!res) return -1; + // reset metadata of data + for (unsigned i = 0; i < fec->n_data; i++) { + parities_props[i].clear(); + uint32_t* metadata = reinterpret_cast(data[i]); + int ret = parities_props[i].fnt_serialize(metadata, metadata_size / 4); + if (ret == -1) { + return -1; + } + } + return 0; } @@ -262,7 +272,6 @@ int quadiron_fnt32_reconstruct( return -1; } parities_vec[i] = data[i] + metadata_size; - data_vec[i] = data[i] + metadata_size; } for (unsigned i = 0; i < fec->n_parities; i++) { if (!missing_idxs[fec->n_data + i]) { From 7147e6ffcdca7f9254026c9363920cdf3b783393 Mon Sep 17 00:00:00 2001 From: Lam Pham-Sy Date: Fri, 14 Jun 2019 05:26:51 +0200 Subject: [PATCH 11/11] QuadIron C Test: fix & assert reconstructed fragments Note that for non-systematic FNT, in encoding and decoding of QuadIron C API, input data will be overwritten by output data: - `quadiron_fnt32_encode` will store first `k` parities in the input data buffers, and the next `m` parities in the usual parity buffers. - `quadiron_fnt32_decode` will overwrite input data pointers (that stores actually encoded fragments) by decoded data. In the test, coded fragments will be stored to use correct fragments. They will be used to check reconstructed data. --- test/quadiron_c_utest.cpp | 80 +++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/test/quadiron_c_utest.cpp b/test/quadiron_c_utest.cpp index f87a0e38..7e6cb39e 100644 --- a/test/quadiron_c_utest.cpp +++ b/test/quadiron_c_utest.cpp @@ -127,12 +127,16 @@ class QuadironCTest : public ::testing::Test { } else { n_outputs = n_data + n_parities; } - std::vector> parity(n_outputs); - std::vector _parity(n_outputs); // for C API + std::vector> parity(n_parities); + std::vector _parity(n_parities); // for C API std::vector wanted_idxs(n_outputs); + std::vector> encoded_data(n_outputs); + std::vector _encoded_data(n_outputs); // for C API + + const size_t full_block_size = block_size + metadata_size; for (int i = 0; i < n_data; i++) { - data.at(i).resize(block_size + metadata_size); + data.at(i).resize(full_block_size); _data[i] = data.at(i).data(); ref_data.at(i).resize(block_size); _ref_data[i] = ref_data.at(i).data(); @@ -140,8 +144,8 @@ class QuadironCTest : public ::testing::Test { std::copy_n(_data[i] + metadata_size, block_size, _ref_data[i]); } - for (int i = 0; i < n_outputs; i++) { - parity.at(i).resize(block_size + metadata_size); + for (int i = 0; i < n_parities; i++) { + parity.at(i).resize(full_block_size); _parity[i] = parity.at(i).data(); } @@ -157,15 +161,34 @@ class QuadironCTest : public ::testing::Test { block_size), 0); + // get encoded data + for (int i = 0; i < n_outputs; i++) { + encoded_data.at(i).resize(full_block_size); + _encoded_data[i] = encoded_data.at(i).data(); + } + if (systematic) { + for (int i = 0; i < n_parities; i++) { + std::copy_n(_parity[i], full_block_size, _encoded_data[i]); + } + } else { + for (int i = 0; i < n_data; i++) { + std::copy_n(_data[i], full_block_size, _encoded_data[i]); + } + for (int i = 0; i < n_parities; i++) { + std::copy_n( + _parity[i], full_block_size, _encoded_data[i + n_data]); + } + } + + // delete missing fragments for (int i = 0; i < n_data; i++) { if (missing_idxs[i]) { - std::fill_n(_data[i], block_size + metadata_size, 0); + std::fill_n(_data[i], full_block_size, 0); } } - for (int i = 0; i < n_parities; i++) { if (missing_idxs[n_data + i]) { - std::fill_n(_parity[i], block_size + metadata_size, 0); + std::fill_n(_parity[i], full_block_size, 0); } } @@ -185,6 +208,25 @@ class QuadironCTest : public ::testing::Test { _data[i] + metadata_size)); } + // get back first n_data parities for the non-systematic case + if (!systematic) { + for (int i = 0; i < n_data; i++) { + std::copy_n(_encoded_data[i], full_block_size, _data[i]); + } + } + + // delete missing fragments + for (int i = 0; i < n_data; i++) { + if (missing_idxs[i]) { + std::fill_n(_data[i], full_block_size, 0); + } + } + for (int i = 0; i < n_parities; i++) { + if (missing_idxs[n_data + i]) { + std::fill_n(_parity[i], full_block_size, 0); + } + } + for (int i = 0; i < n_data; i++) { if (missing_idxs[i]) { ASSERT_EQ( @@ -196,6 +238,17 @@ class QuadironCTest : public ::testing::Test { i, block_size), 0); + if (systematic) { + ASSERT_TRUE(std::equal( + _ref_data[i], + _ref_data[i] + block_size, + _data[i] + metadata_size)); + } else { + ASSERT_TRUE(std::equal( + _encoded_data[i], + _encoded_data[i] + full_block_size, + _data[i])); + } } } @@ -210,6 +263,17 @@ class QuadironCTest : public ::testing::Test { n_data + i, block_size), 0); + if (systematic) { + ASSERT_TRUE(std::equal( + _encoded_data[i], + _encoded_data[i] + full_block_size, + _parity[i])); + } else { + ASSERT_TRUE(std::equal( + _encoded_data[n_data + i], + _encoded_data[n_data + i] + full_block_size, + _parity[i])); + } } }