From 7649c91ae5c9a90ad84675726795f1b95a91802c Mon Sep 17 00:00:00 2001 From: Zachary Dremann Date: Fri, 9 Oct 2026 14:21:11 -0400 Subject: [PATCH] perf: use a perfect-shuffle bitonic merge for the SSE union and xor sse_merge rotated one register through the other with 8 dependent alignr/min/max rounds. Replace it with the perfect-shuffle form of the bitonic merge: reverse the incoming block, then 4 rounds of lane-wise min/max and a 16-bit interleave. Fewer uops (17 vs ~24), no blends, and the chain through vecMax drops from ~15 to ~8 cycles since the reverse is applied to the freshly loaded block rather than the carry. --- src/array_util.c | 88 +++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 49 deletions(-) diff --git a/src/array_util.c b/src/array_util.c index e56133216..1800825a2 100644 --- a/src/array_util.c +++ b/src/array_util.c @@ -1276,39 +1276,32 @@ int32_t xor_uint16(const uint16_t *array_1, int32_t card_1, */ CROARING_TARGET_AVX2 -// Assuming that vInput1 and vInput2 are sorted, produces a sorted output going -// from vecMin all the way to vecMax -// developed originally for merge sort using SIMD instructions. -// Standard merge. See, e.g., Inoue and Taura, SIMD- and Cache-Friendly -// Algorithm for Sorting an Array of Structures -static inline void sse_merge(const __m128i *vInput1, - const __m128i *vInput2, // input 1 & 2 - __m128i *vecMin, __m128i *vecMax) { // output - __m128i vecTmp; - vecTmp = _mm_min_epu16(*vInput1, *vInput2); - *vecMax = _mm_max_epu16(*vInput1, *vInput2); - vecTmp = _mm_alignr_epi8(vecTmp, vecTmp, 2); - *vecMin = _mm_min_epu16(vecTmp, *vecMax); - *vecMax = _mm_max_epu16(vecTmp, *vecMax); - vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); - *vecMin = _mm_min_epu16(vecTmp, *vecMax); - *vecMax = _mm_max_epu16(vecTmp, *vecMax); - vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); - *vecMin = _mm_min_epu16(vecTmp, *vecMax); - *vecMax = _mm_max_epu16(vecTmp, *vecMax); - vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); - *vecMin = _mm_min_epu16(vecTmp, *vecMax); - *vecMax = _mm_max_epu16(vecTmp, *vecMax); - vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); - *vecMin = _mm_min_epu16(vecTmp, *vecMax); - *vecMax = _mm_max_epu16(vecTmp, *vecMax); - vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); - *vecMin = _mm_min_epu16(vecTmp, *vecMax); - *vecMax = _mm_max_epu16(vecTmp, *vecMax); - vecTmp = _mm_alignr_epi8(*vecMin, *vecMin, 2); - *vecMin = _mm_min_epu16(vecTmp, *vecMax); - *vecMax = _mm_max_epu16(vecTmp, *vecMax); - *vecMin = _mm_alignr_epi8(*vecMin, *vecMin, 2); +// Merge two ascending vectors: *vecMin receives the 8 smallest values in +// ascending order, *vecMax the 8 largest, also ascending. `b` is reversed, so +// pass the operand that is not on a loop-carried chain there. +// +// This is the perfect-shuffle form of the bitonic merge (Stone, "Parallel +// processing with the perfect shuffle", 1971): a lane-wise min/max between the +// two registers followed by a 16-bit interleave, four times. A bitonic merge of +// 16 values takes log2(16) = 4 compare-exchange stages, at distances 8, 4, 2 +// and 1. The interleave lines up the next stage's pairs in the same lanes, so +// no stage needs a blend, and after the fourth interleave the 16 values are +// back in sorted lane order. Only two operations per stage sit on the chain +// through *vecMax. +static inline void sse_merge(__m128i a, __m128i b, __m128i *vecMin, + __m128i *vecMax) { + const __m128i rev = + _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); + __m128i x = a; + __m128i y = _mm_shuffle_epi8(b, rev); + for (int i = 0; i < 4; i++) { + __m128i mn = _mm_min_epu16(x, y); + __m128i mx = _mm_max_epu16(x, y); + x = _mm_unpacklo_epi16(mn, mx); + y = _mm_unpackhi_epi16(mn, mx); + } + *vecMin = x; + *vecMax = y; } CROARING_UNTARGET_AVX2 // used by store_unique, generated by simdunion.py @@ -1719,7 +1712,7 @@ uint32_t union_vector16(const uint16_t *array1, uint32_t length1, pos1++; vB = _mm_lddqu_si128((const __m128i *)array2 + pos2); pos2++; - sse_merge(&vA, &vB, &vecMin, &vecMax); + sse_merge(vA, vB, &vecMin, &vecMax); laststore = _mm_set1_epi16(-1); output += store_unique(laststore, vecMin, output); laststore = vecMin; @@ -1745,11 +1738,11 @@ uint32_t union_vector16(const uint16_t *array1, uint32_t length1, break; } } - sse_merge(&V, &vecMax, &vecMin, &vecMax); + sse_merge(vecMax, V, &vecMin, &vecMax); output += store_unique(laststore, vecMin, output); laststore = vecMin; } - sse_merge(&V, &vecMax, &vecMin, &vecMax); + sse_merge(vecMax, V, &vecMin, &vecMax); output += store_unique(laststore, vecMin, output); laststore = vecMin; } @@ -1790,16 +1783,13 @@ CROARING_UNTARGET_AVX2 /** * Start of the AVX-512 16-bit union code. * - * union_vector16 above merges 8 lanes at a time with an odd-even transposition - * network: 8 dependent min/max stages per 8 output values, which measures at - * ~2.3 cycles per input element -- no better than the scalar union_uint16. - * - * With AVX-512 we can merge 32+32 lanes with a Batcher bitonic network: one - * reverse plus 5 compare-exchange stages per sorted half, i.e. ~4x fewer - * operations per output value. This needs cross-lane 16-bit permutes (vpermw, - * vpermt2w) and a 16-bit compress store (vpcompressw); AVX2 has none of these, - * so the algorithm is genuinely AVX-512-only rather than a wider rerun of the - * SSE code. + * union_vector16 above merges 8 lanes at a time: 4 compare-exchange stages per + * 8 output values. With AVX-512 we can merge 32+32 lanes with a Batcher bitonic + * network: one reverse plus 5 compare-exchange stages per sorted half, i.e. + * fewer operations per output value. This needs cross-lane 16-bit permutes + * (vpermw, vpermt2w) and a 16-bit compress store (vpcompressw); AVX2 has none + * of these, so the algorithm is genuinely AVX-512-only rather than a wider + * rerun of the SSE code. */ #if CROARING_COMPILER_SUPPORTS_AVX512 @@ -2029,7 +2019,7 @@ uint32_t xor_vector16(const uint16_t *array1, uint32_t length1, pos1++; vB = _mm_lddqu_si128((const __m128i *)array2 + pos2); pos2++; - sse_merge(&vA, &vB, &vecMin, &vecMax); + sse_merge(vA, vB, &vecMin, &vecMax); laststore = _mm_set1_epi16(-1); uint16_t buffer[17]; output += store_unique_xor(laststore, vecMin, output); @@ -2057,14 +2047,14 @@ uint32_t xor_vector16(const uint16_t *array1, uint32_t length1, break; } } - sse_merge(&V, &vecMax, &vecMin, &vecMax); + sse_merge(vecMax, V, &vecMin, &vecMax); // conditionally stores the last value of laststore as well as all // but the // last value of vecMin output += store_unique_xor(laststore, vecMin, output); laststore = vecMin; } - sse_merge(&V, &vecMax, &vecMin, &vecMax); + sse_merge(vecMax, V, &vecMin, &vecMax); // conditionally stores the last value of laststore as well as all but // the // last value of vecMin