From d72d514ded59727bafa4c00e379be3a5ae69fcce Mon Sep 17 00:00:00 2001 From: Niklas Schilli Date: Thu, 17 Sep 2026 11:34:22 +0200 Subject: [PATCH 1/5] Use a block reduction in TensorPrimitives.MinMaxCore MinMaxCore backs Min, Max, MinMagnitude, MaxMagnitude, MinNumber, MaxNumber, MinMagnitudeNumber, MaxMagnitudeNumber (and the Half path through HalfAsInt16AggregationOperator). Its vector loops used one accumulator, one vector per iteration and, for float/double, a NaN test with a branch on every vector. On AVX-512 hardware Vector.Min/Max also lowers to vrangeps plus two dependent vfixupimmps, so the float loop was bound by a 3-instruction latency chain plus 5-6 instructions of NaN checking per vector. The vector paths (512/256/128, kept symmetric) now reduce the whole vectors in blocks of 32 vectors with two independent accumulators, then handle any remaining elements with one final vector that overlaps the last whole one, as before. NaN is decided once per block: the IsNaN masks of the block's vectors are OR-accumulated (branch-free) and, if any is set, a cold NoInlining helper rescans the block for the first NaN, so the first NaN of the input is still the one returned (blocks are visited in order and the running result is only combined after the check). The mask is gathered from the elements rather than from the result because the Number operators discard NaN; the previous behavior of returning the first NaN for those operators too is preserved. Min and Max over float and double take a fast path that reduces each block with Vector.MinNative/MaxNative (vminps/vmaxps, one instruction, no NaN or signed-zero guarantees) and restores the IEEE 754:2019 minimum/maximum rules once per block: the NaN mask above, and an OR (Min) or AND (Max) of the raw element bits. The native minimum never returns a value below every input, so when a block's minimum compares equal to zero and the block holds no NaN, no element is negative and any set sign bit is a -0; the maximum case is mirrored. The Magnitude and Number operators, Half and the integer types use the generic block reduction with the operator's own vector Invoke (for integers that is already two vpminsd with memory operands per iteration). Block results are combined with the operator's scalar Invoke, first block first, so the operators' tie rules are unchanged. The per-width methods are NoInlining so that each is its own JIT root with enough inlining budget for the block reduction and the horizontal aggregates; the old shape was inlined into the public entry point, whose small budget left HorizontalAggregate and the typeof helpers as real calls. Hot loop for float on AVX-512 (32 floats per iteration): 2 loads, 2 vminps, 1 vpternlogd, 2 vcmpneqps, korw, vpmovm2d, vorps, plus the loop overhead. Measured on a Ryzen 7 7800X3D (Zen 4, AVX-512), 65536 elements, package 10.0.12 vs this change, GElem/s: Min 11.1-11.5 -> 27.0 Max 11.4-11.5 -> 26.2-27.2 Min 5.6-5.7 -> 13.5-14.1 Min 31-35 -> 34-35 (load ceiling either way) Verified bit-exact against the unmodified library over a differential matrix (8 element types x 8 reductions x 29 lengths up to 65539 x random/all-equal/ all-zero/signed-zero/NaN-with-payload/two-NaN/infinity/magnitude-tie/MinValue cases, on the 512, 256 and 128-bit paths). The only difference is which of two distinct NaN payloads Half returns: the Half path never had a "first NaN" rule (it depends on the lane pairing of the vector operator, and the previous code returned the second NaN's payload in two thirds of those cases), and the block pairing changes it in about 5% of them. Tests cover lengths past 256, the first-NaN rule, signed zeros and all-zero inputs at block boundaries for all six span reductions. Co-Authored-By: Claude Fable 5.1 --- .../Tensors/netcore/TensorPrimitives.Max.cs | 832 +++++++++++++++--- .../tests/TensorPrimitives.Generic.cs | 50 ++ .../tests/TensorPrimitivesTests.cs | 204 +++++ 3 files changed, 964 insertions(+), 122 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs index 1e7249bd3fc751..d1462c91c2cebe 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -119,6 +120,9 @@ public static void Max(ReadOnlySpan x, T y, Span destination) public static T Invoke(Vector512 x) => HorizontalAggregate>(x); } + /// Vectors per block in the vectorized paths of . + private const int MinMaxBlockVectors = 32; + /// /// This is the same as /// with an identity transform, except it early exits on NaN. @@ -139,219 +143,803 @@ private static T MinMaxCore(ReadOnlySpan x) if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count) { - ref T xRef = ref MemoryMarshal.GetReference(x); + return MinMaxVectorized512(x); + } - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector512 result = Vector512.LoadUnsafe(ref xRef, 0); - Vector512 current; + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) + { + return MinMaxVectorized256(x); + } - Vector512 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) + { + return MinMaxVectorized128(x); + } + + // Scalar path used when either vectorization is not supported or the input is too small to vectorize. + T curResult = x[0]; + if (T.IsNaN(curResult)) + { + return curResult; + } + + for (int i = 1; i < x.Length; i++) + { + T current = x[i]; + if (T.IsNaN(current)) { - // Check for NaNs - nanMask = Vector512.IsNaN(result); - if (nanMask != Vector512.Zero) - { - return result.GetElement(IndexOfFirstMatch(nanMask)); - } + return current; } - int oneVectorFromEnd = x.Length - Vector512.Count; - int i = Vector512.Count; + curResult = TMinMaxOperator.Invoke(curResult, current); + } + + return curResult; + } + + /// + /// Whether is or over + /// or : the reductions whose blocks are reduced with the native vector minimum/maximum + /// (, one instruction) plus masks that restore the IEEE 754:2019 rules once per block, + /// rather than with the operator itself (, several dependent instructions). + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsNativeMinMax() => + (typeof(T) == typeof(float) && (typeof(TMinMaxOperator) == typeof(MinOperator) || typeof(TMinMaxOperator) == typeof(MaxOperator))) || + (typeof(T) == typeof(double) && (typeof(TMinMaxOperator) == typeof(MinOperator) || typeof(TMinMaxOperator) == typeof(MaxOperator))); + + /// Whether is a minimum rather than a maximum. Only meaningful when . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsNativeMin() => + typeof(TMinMaxOperator) == typeof(MinOperator) || typeof(TMinMaxOperator) == typeof(MinOperator); + + /// + /// Restores the IEEE 754:2019 `minimum`/`maximum` result of a block reduced with the native vector minimum/maximum, which does not + /// order the signed zeros. The native operation never returns a value below (above) every input, so if the block minimum (maximum) + /// compares equal to zero and the block holds no NaN, no element is negative (positive), and an element whose sign bit is set (clear) + /// can only be -0 (+0). + /// + /// The result of the native reduction of a block that contains no NaN. + /// For a minimum, whether any element of the block has its sign bit set; for a maximum, whether any has it clear. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T FixUpNativeSignedZero(T blockResult, bool anyOppositeSignBit) + where T : INumberBase + { + Debug.Assert(IsNativeMinMax()); + Debug.Assert(!T.IsNaN(blockResult)); + + if (anyOppositeSignBit && blockResult == T.Zero) + { + return IsNativeMin() ? -T.Zero : T.Zero; + } - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) + return blockResult; + } + + /// Min(x, y) with no NaN or signed-zero guarantees: the hardware minimum where there is one. Used for the horizontal reductions of the native blocks. + private readonly struct MinNativeOperator : IBinaryOperator + where T : INumberBase + { + public static bool Vectorizable => true; + + public static T Invoke(T x, T y) => Vector128.MinNative(Vector128.CreateScalar(x), Vector128.CreateScalar(y)).ToScalar(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => Vector128.MinNative(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => Vector256.MinNative(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => Vector512.MinNative(x, y); + } + + /// Max(x, y) with no NaN or signed-zero guarantees: the hardware maximum where there is one. Used for the horizontal reductions of the native blocks. + private readonly struct MaxNativeOperator : IBinaryOperator + where T : INumberBase + { + public static bool Vectorizable => true; + + public static T Invoke(T x, T y) => Vector128.MaxNative(Vector128.CreateScalar(x), Vector128.CreateScalar(y)).ToScalar(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) => Vector128.MaxNative(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) => Vector256.MaxNative(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) => Vector512.MaxNative(x, y); + } + + /// The 512-bit path of : a block reduction of the whole vectors followed by one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is reduced with two independent accumulators and a single NaN decision + /// (the OR of the elements' NaN masks), so the hot loop contains no branch on the data. Blocks are visited in order and the running + /// result is only combined after a block has been checked, so the first NaN of the input is the one returned. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per reduction; its own inlining budget keeps the block reduction and the horizontal aggregates inlined + private static T MinMaxVectorized512(ReadOnlySpan x) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(Vector512.IsHardwareAccelerated && Vector512.IsSupported); + Debug.Assert(x.Length >= Vector512.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + int length = x.Length; + int blockSize = MinMaxBlockVectors * Vector512.Count; + int wholeVectorsLength = length - (length % Vector512.Count); + + // Reduce the whole vectors block by block. Starting from the first element is harmless: it is part of the first block, + // and combining an element with a result that already accounts for it does not change that result. + T result = xRef; + for (int i = 0; i < wholeVectorsLength; i += blockSize) + { + int blockLength = Math.Min(blockSize, wholeVectorsLength - i); + T blockResult; + bool anyNaN; + + if (IsNativeMinMax()) { - // Load the next vector, and early exit on NaN. - current = Vector512.LoadUnsafe(ref xRef, (uint)i); + blockResult = BlockReduceNative512(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN, out bool anyOppositeSignBit); + if (anyNaN) + { + return FirstNaN512(ref Unsafe.Add(ref xRef, i), blockLength); + } + blockResult = FixUpNativeSignedZero(blockResult, anyOppositeSignBit); + } + else + { + blockResult = BlockReduce512(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) + if (anyNaN) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return FirstNaN512(ref Unsafe.Add(ref xRef, i), blockLength); } } - - result = TMinMaxOperator.Invoke(result, current); - i += Vector512.Count; } - // If any elements remain, handle them in one final vector. - if (i != x.Length) + result = TMinMaxOperator.Invoke(result, blockResult); + } + + // If any elements remain, handle them in one final vector. Its NaN check does not change which NaN is returned: + // a NaN among the elements it revisits would already have been returned by their block. + if (wholeVectorsLength != length) + { + Vector512 last = Vector512.LoadUnsafe(ref xRef, (uint)(length - Vector512.Count)); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); + Vector512 nanMask = Vector512.IsNaN(last); + if (nanMask != Vector512.Zero) + { + return last.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, TMinMaxOperator.Invoke(last)); + } + return result; + } + + /// Reduces the elements at , a whole number of 512-bit vectors, with . + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// + /// For and , whether any element is NaN, in which case the returned value is meaningless. + /// It is gathered from the elements rather than from the result because the Number operators discard NaN. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduce512(ref T xRef, int length, out bool anyNaN) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(length >= Vector512.Count && length % Vector512.Count == 0); + + Vector512 acc1 = Vector512.LoadUnsafe(ref xRef); + Vector512 nan = Vector512.IsNaN(acc1); + int i = Vector512.Count; + + if (length >= 2 * Vector512.Count) + { + Vector512 acc2 = Vector512.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector512.IsNaN(acc2); + i += Vector512.Count; + + int twoVectorsFromEnd = length - (2 * Vector512.Count); + while (i <= twoVectorsFromEnd) + { + Vector512 current1 = Vector512.LoadUnsafe(ref xRef, (uint)i); + Vector512 current2 = Vector512.LoadUnsafe(ref xRef, (uint)(i + Vector512.Count)); + acc1 = TMinMaxOperator.Invoke(acc1, current1); + acc2 = TMinMaxOperator.Invoke(acc2, current2); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } + nan |= Vector512.IsNaN(current1) | Vector512.IsNaN(current2); } - result = TMinMaxOperator.Invoke(result, current); + i += 2 * Vector512.Count; } - // Aggregate the lanes in the vector to create the final scalar result. - return TMinMaxOperator.Invoke(result); + acc1 = TMinMaxOperator.Invoke(acc1, acc2); } - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) + if (i != length) { - ref T xRef = ref MemoryMarshal.GetReference(x); + Vector512 current = Vector512.LoadUnsafe(ref xRef, (uint)i); + acc1 = TMinMaxOperator.Invoke(acc1, current); + nan |= Vector512.IsNaN(current); + } - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector256 result = Vector256.LoadUnsafe(ref xRef, 0); - Vector256 current; + anyNaN = (typeof(T) == typeof(float) || typeof(T) == typeof(double)) && nan != Vector512.Zero; + return TMinMaxOperator.Invoke(acc1); + } - Vector256 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + /// + /// Reduces the elements at , a whole number of 512-bit vectors, with the native vector + /// minimum/maximum, gathering alongside the two facts that let the caller restore the IEEE 754:2019 result (see ). + /// + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// Whether any element is NaN, in which case the returned value is meaningless. + /// For a minimum, whether any element has its sign bit set; for a maximum, whether any has it clear. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduceNative512(ref T xRef, int length, out bool anyNaN, out bool anyOppositeSignBit) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(IsNativeMinMax()); + Debug.Assert(length >= Vector512.Count && length % Vector512.Count == 0); + + // For a minimum, OR the elements together so that any set sign bit survives; for a maximum, AND them so that any clear one does. + Vector512 acc1 = Vector512.LoadUnsafe(ref xRef); + Vector512 nan = Vector512.IsNaN(acc1); + Vector512 signBits = acc1; + int i = Vector512.Count; + + if (length >= 2 * Vector512.Count) + { + Vector512 acc2 = Vector512.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector512.IsNaN(acc2); + signBits = IsNativeMin() ? signBits | acc2 : signBits & acc2; + i += Vector512.Count; + + int twoVectorsFromEnd = length - (2 * Vector512.Count); + while (i <= twoVectorsFromEnd) { - // Check for NaNs - nanMask = ~Vector256.Equals(result, result); - if (nanMask != Vector256.Zero) + Vector512 current1 = Vector512.LoadUnsafe(ref xRef, (uint)i); + Vector512 current2 = Vector512.LoadUnsafe(ref xRef, (uint)(i + Vector512.Count)); + if (IsNativeMin()) + { + acc1 = Vector512.MinNative(acc1, current1); + acc2 = Vector512.MinNative(acc2, current2); + signBits |= current1 | current2; + } + else { - return result.GetElement(IndexOfFirstMatch(nanMask)); + acc1 = Vector512.MaxNative(acc1, current1); + acc2 = Vector512.MaxNative(acc2, current2); + signBits &= current1 & current2; } + + nan |= Vector512.IsNaN(current1) | Vector512.IsNaN(current2); + i += 2 * Vector512.Count; + } + + acc1 = IsNativeMin() ? Vector512.MinNative(acc1, acc2) : Vector512.MaxNative(acc1, acc2); + } + + if (i != length) + { + Vector512 current = Vector512.LoadUnsafe(ref xRef, (uint)i); + if (IsNativeMin()) + { + acc1 = Vector512.MinNative(acc1, current); + signBits |= current; + } + else + { + acc1 = Vector512.MaxNative(acc1, current); + signBits &= current; } - int oneVectorFromEnd = x.Length - Vector256.Count; - int i = Vector256.Count; + nan |= Vector512.IsNaN(current); + } + + anyNaN = nan != Vector512.Zero; + anyOppositeSignBit = IsNativeMin() ? + signBits.ExtractMostSignificantBits() != 0 : + (~signBits).ExtractMostSignificantBits() != 0; + + return IsNativeMin() ? + HorizontalAggregate>(acc1) : + HorizontalAggregate>(acc1); + } + + /// Returns the first NaN among the elements at , a whole number of 512-bit vectors that contains one. + [MethodImpl(MethodImplOptions.NoInlining)] // cold: called at most once per reduction; keeps the caller within the inlining budget + private static T FirstNaN512(ref T xRef, int length) + where T : INumberBase + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + Debug.Assert(length >= Vector512.Count && length % Vector512.Count == 0); + + for (int i = 0; i < length; i += Vector512.Count) + { + Vector512 current = Vector512.LoadUnsafe(ref xRef, (uint)i); + Vector512 nanMask = Vector512.IsNaN(current); + if (nanMask != Vector512.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + Debug.Fail("The block must contain a NaN."); + return default!; + } + + /// The 256-bit path of : a block reduction of the whole vectors followed by one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is reduced with two independent accumulators and a single NaN decision + /// (the OR of the elements' NaN masks), so the hot loop contains no branch on the data. Blocks are visited in order and the running + /// result is only combined after a block has been checked, so the first NaN of the input is the one returned. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per reduction; its own inlining budget keeps the block reduction and the horizontal aggregates inlined + private static T MinMaxVectorized256(ReadOnlySpan x) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(Vector256.IsHardwareAccelerated && Vector256.IsSupported); + Debug.Assert(x.Length >= Vector256.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + int length = x.Length; + int blockSize = MinMaxBlockVectors * Vector256.Count; + int wholeVectorsLength = length - (length % Vector256.Count); + + // Reduce the whole vectors block by block. Starting from the first element is harmless: it is part of the first block, + // and combining an element with a result that already accounts for it does not change that result. + T result = xRef; + for (int i = 0; i < wholeVectorsLength; i += blockSize) + { + int blockLength = Math.Min(blockSize, wholeVectorsLength - i); + T blockResult; + bool anyNaN; - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) + if (IsNativeMinMax()) { - // Load the next vector, and early exit on NaN. - current = Vector256.LoadUnsafe(ref xRef, (uint)i); + blockResult = BlockReduceNative256(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN, out bool anyOppositeSignBit); + if (anyNaN) + { + return FirstNaN256(ref Unsafe.Add(ref xRef, i), blockLength); + } + blockResult = FixUpNativeSignedZero(blockResult, anyOppositeSignBit); + } + else + { + blockResult = BlockReduce256(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) + if (anyNaN) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return FirstNaN256(ref Unsafe.Add(ref xRef, i), blockLength); } } - - result = TMinMaxOperator.Invoke(result, current); - i += Vector256.Count; } - // If any elements remain, handle them in one final vector. - if (i != x.Length) + result = TMinMaxOperator.Invoke(result, blockResult); + } + + // If any elements remain, handle them in one final vector. Its NaN check does not change which NaN is returned: + // a NaN among the elements it revisits would already have been returned by their block. + if (wholeVectorsLength != length) + { + Vector256 last = Vector256.LoadUnsafe(ref xRef, (uint)(length - Vector256.Count)); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); + Vector256 nanMask = Vector256.IsNaN(last); + if (nanMask != Vector256.Zero) + { + return last.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, TMinMaxOperator.Invoke(last)); + } + + return result; + } + /// Reduces the elements at , a whole number of 256-bit vectors, with . + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// + /// For and , whether any element is NaN, in which case the returned value is meaningless. + /// It is gathered from the elements rather than from the result because the Number operators discard NaN. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduce256(ref T xRef, int length, out bool anyNaN) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(length >= Vector256.Count && length % Vector256.Count == 0); + Vector256 acc1 = Vector256.LoadUnsafe(ref xRef); + Vector256 nan = Vector256.IsNaN(acc1); + int i = Vector256.Count; + + if (length >= 2 * Vector256.Count) + { + Vector256 acc2 = Vector256.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector256.IsNaN(acc2); + i += Vector256.Count; + + int twoVectorsFromEnd = length - (2 * Vector256.Count); + while (i <= twoVectorsFromEnd) + { + Vector256 current1 = Vector256.LoadUnsafe(ref xRef, (uint)i); + Vector256 current2 = Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256.Count)); + acc1 = TMinMaxOperator.Invoke(acc1, current1); + acc2 = TMinMaxOperator.Invoke(acc2, current2); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } + nan |= Vector256.IsNaN(current1) | Vector256.IsNaN(current2); } - result = TMinMaxOperator.Invoke(result, current); + i += 2 * Vector256.Count; } - // Aggregate the lanes in the vector to create the final scalar result. - return TMinMaxOperator.Invoke(result); + acc1 = TMinMaxOperator.Invoke(acc1, acc2); } - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) + if (i != length) { - ref T xRef = ref MemoryMarshal.GetReference(x); + Vector256 current = Vector256.LoadUnsafe(ref xRef, (uint)i); + acc1 = TMinMaxOperator.Invoke(acc1, current); + nan |= Vector256.IsNaN(current); + } - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector128 result = Vector128.LoadUnsafe(ref xRef, 0); - Vector128 current; + anyNaN = (typeof(T) == typeof(float) || typeof(T) == typeof(double)) && nan != Vector256.Zero; + return TMinMaxOperator.Invoke(acc1); + } - Vector128 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + /// + /// Reduces the elements at , a whole number of 256-bit vectors, with the native vector + /// minimum/maximum, gathering alongside the two facts that let the caller restore the IEEE 754:2019 result (see ). + /// + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// Whether any element is NaN, in which case the returned value is meaningless. + /// For a minimum, whether any element has its sign bit set; for a maximum, whether any has it clear. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduceNative256(ref T xRef, int length, out bool anyNaN, out bool anyOppositeSignBit) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(IsNativeMinMax()); + Debug.Assert(length >= Vector256.Count && length % Vector256.Count == 0); + + // For a minimum, OR the elements together so that any set sign bit survives; for a maximum, AND them so that any clear one does. + Vector256 acc1 = Vector256.LoadUnsafe(ref xRef); + Vector256 nan = Vector256.IsNaN(acc1); + Vector256 signBits = acc1; + int i = Vector256.Count; + + if (length >= 2 * Vector256.Count) + { + Vector256 acc2 = Vector256.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector256.IsNaN(acc2); + signBits = IsNativeMin() ? signBits | acc2 : signBits & acc2; + i += Vector256.Count; + + int twoVectorsFromEnd = length - (2 * Vector256.Count); + while (i <= twoVectorsFromEnd) { - // Check for NaNs - nanMask = Vector128.IsNaN(result); - if (nanMask != Vector128.Zero) + Vector256 current1 = Vector256.LoadUnsafe(ref xRef, (uint)i); + Vector256 current2 = Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256.Count)); + if (IsNativeMin()) + { + acc1 = Vector256.MinNative(acc1, current1); + acc2 = Vector256.MinNative(acc2, current2); + signBits |= current1 | current2; + } + else { - return result.GetElement(IndexOfFirstMatch(nanMask)); + acc1 = Vector256.MaxNative(acc1, current1); + acc2 = Vector256.MaxNative(acc2, current2); + signBits &= current1 & current2; } + + nan |= Vector256.IsNaN(current1) | Vector256.IsNaN(current2); + i += 2 * Vector256.Count; } - int oneVectorFromEnd = x.Length - Vector128.Count; - int i = Vector128.Count; + acc1 = IsNativeMin() ? Vector256.MinNative(acc1, acc2) : Vector256.MaxNative(acc1, acc2); + } - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) + if (i != length) + { + Vector256 current = Vector256.LoadUnsafe(ref xRef, (uint)i); + if (IsNativeMin()) + { + acc1 = Vector256.MinNative(acc1, current); + signBits |= current; + } + else { - // Load the next vector, and early exit on NaN. - current = Vector128.LoadUnsafe(ref xRef, (uint)i); + acc1 = Vector256.MaxNative(acc1, current); + signBits &= current; + } + + nan |= Vector256.IsNaN(current); + } + + anyNaN = nan != Vector256.Zero; + anyOppositeSignBit = IsNativeMin() ? + signBits.ExtractMostSignificantBits() != 0 : + (~signBits).ExtractMostSignificantBits() != 0; + return IsNativeMin() ? + HorizontalAggregate>(acc1) : + HorizontalAggregate>(acc1); + } + + /// Returns the first NaN among the elements at , a whole number of 256-bit vectors that contains one. + [MethodImpl(MethodImplOptions.NoInlining)] // cold: called at most once per reduction; keeps the caller within the inlining budget + private static T FirstNaN256(ref T xRef, int length) + where T : INumberBase + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + Debug.Assert(length >= Vector256.Count && length % Vector256.Count == 0); + + for (int i = 0; i < length; i += Vector256.Count) + { + Vector256 current = Vector256.LoadUnsafe(ref xRef, (uint)i); + Vector256 nanMask = Vector256.IsNaN(current); + if (nanMask != Vector256.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + Debug.Fail("The block must contain a NaN."); + return default!; + } + + /// The 128-bit path of : a block reduction of the whole vectors followed by one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is reduced with two independent accumulators and a single NaN decision + /// (the OR of the elements' NaN masks), so the hot loop contains no branch on the data. Blocks are visited in order and the running + /// result is only combined after a block has been checked, so the first NaN of the input is the one returned. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per reduction; its own inlining budget keeps the block reduction and the horizontal aggregates inlined + private static T MinMaxVectorized128(ReadOnlySpan x) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(Vector128.IsHardwareAccelerated && Vector128.IsSupported); + Debug.Assert(x.Length >= Vector128.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + int length = x.Length; + int blockSize = MinMaxBlockVectors * Vector128.Count; + int wholeVectorsLength = length - (length % Vector128.Count); + + // Reduce the whole vectors block by block. Starting from the first element is harmless: it is part of the first block, + // and combining an element with a result that already accounts for it does not change that result. + T result = xRef; + for (int i = 0; i < wholeVectorsLength; i += blockSize) + { + int blockLength = Math.Min(blockSize, wholeVectorsLength - i); + T blockResult; + bool anyNaN; + + if (IsNativeMinMax()) + { + blockResult = BlockReduceNative128(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN, out bool anyOppositeSignBit); + if (anyNaN) + { + return FirstNaN128(ref Unsafe.Add(ref xRef, i), blockLength); + } + + blockResult = FixUpNativeSignedZero(blockResult, anyOppositeSignBit); + } + else + { + blockResult = BlockReduce128(ref Unsafe.Add(ref xRef, i), blockLength, out anyNaN); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = Vector128.IsNaN(current); - if (nanMask != Vector128.Zero) + if (anyNaN) { - return current.GetElement(IndexOfFirstMatch(nanMask)); + return FirstNaN128(ref Unsafe.Add(ref xRef, i), blockLength); } } - - result = TMinMaxOperator.Invoke(result, current); - i += Vector128.Count; } - // If any elements remain, handle them in one final vector. - if (i != x.Length) + result = TMinMaxOperator.Invoke(result, blockResult); + } + + // If any elements remain, handle them in one final vector. Its NaN check does not change which NaN is returned: + // a NaN among the elements it revisits would already have been returned by their block. + if (wholeVectorsLength != length) + { + Vector128 last = Vector128.LoadUnsafe(ref xRef, (uint)(length - Vector128.Count)); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); + Vector128 nanMask = Vector128.IsNaN(last); + if (nanMask != Vector128.Zero) + { + return last.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, TMinMaxOperator.Invoke(last)); + } + + return result; + } + /// Reduces the elements at , a whole number of 128-bit vectors, with . + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// + /// For and , whether any element is NaN, in which case the returned value is meaningless. + /// It is gathered from the elements rather than from the result because the Number operators discard NaN. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduce128(ref T xRef, int length, out bool anyNaN) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(length >= Vector128.Count && length % Vector128.Count == 0); + + Vector128 acc1 = Vector128.LoadUnsafe(ref xRef); + Vector128 nan = Vector128.IsNaN(acc1); + int i = Vector128.Count; + + if (length >= 2 * Vector128.Count) + { + Vector128 acc2 = Vector128.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector128.IsNaN(acc2); + i += Vector128.Count; + + int twoVectorsFromEnd = length - (2 * Vector128.Count); + while (i <= twoVectorsFromEnd) + { + Vector128 current1 = Vector128.LoadUnsafe(ref xRef, (uint)i); + Vector128 current2 = Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128.Count)); + acc1 = TMinMaxOperator.Invoke(acc1, current1); + acc2 = TMinMaxOperator.Invoke(acc2, current2); if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) { - // Check for NaNs - nanMask = Vector128.IsNaN(current); - if (nanMask != Vector128.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } + nan |= Vector128.IsNaN(current1) | Vector128.IsNaN(current2); } - result = TMinMaxOperator.Invoke(result, current); + i += 2 * Vector128.Count; } - // Aggregate the lanes in the vector to create the final scalar result. - return TMinMaxOperator.Invoke(result); + acc1 = TMinMaxOperator.Invoke(acc1, acc2); } - // Scalar path used when either vectorization is not supported or the input is too small to vectorize. - T curResult = x[0]; - if (T.IsNaN(curResult)) + if (i != length) { - return curResult; + Vector128 current = Vector128.LoadUnsafe(ref xRef, (uint)i); + acc1 = TMinMaxOperator.Invoke(acc1, current); + nan |= Vector128.IsNaN(current); } - for (int i = 1; i < x.Length; i++) + anyNaN = (typeof(T) == typeof(float) || typeof(T) == typeof(double)) && nan != Vector128.Zero; + return TMinMaxOperator.Invoke(acc1); + } + + /// + /// Reduces the elements at , a whole number of 128-bit vectors, with the native vector + /// minimum/maximum, gathering alongside the two facts that let the caller restore the IEEE 754:2019 result (see ). + /// + /// The first element of the block. + /// The number of elements in the block, a positive multiple of . + /// Whether any element is NaN, in which case the returned value is meaningless. + /// For a minimum, whether any element has its sign bit set; for a maximum, whether any has it clear. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T BlockReduceNative128(ref T xRef, int length, out bool anyNaN, out bool anyOppositeSignBit) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + Debug.Assert(IsNativeMinMax()); + Debug.Assert(length >= Vector128.Count && length % Vector128.Count == 0); + + // For a minimum, OR the elements together so that any set sign bit survives; for a maximum, AND them so that any clear one does. + Vector128 acc1 = Vector128.LoadUnsafe(ref xRef); + Vector128 nan = Vector128.IsNaN(acc1); + Vector128 signBits = acc1; + int i = Vector128.Count; + + if (length >= 2 * Vector128.Count) { - T current = x[i]; - if (T.IsNaN(current)) + Vector128 acc2 = Vector128.LoadUnsafe(ref xRef, (uint)i); + nan |= Vector128.IsNaN(acc2); + signBits = IsNativeMin() ? signBits | acc2 : signBits & acc2; + i += Vector128.Count; + + int twoVectorsFromEnd = length - (2 * Vector128.Count); + while (i <= twoVectorsFromEnd) { - return current; + Vector128 current1 = Vector128.LoadUnsafe(ref xRef, (uint)i); + Vector128 current2 = Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128.Count)); + if (IsNativeMin()) + { + acc1 = Vector128.MinNative(acc1, current1); + acc2 = Vector128.MinNative(acc2, current2); + signBits |= current1 | current2; + } + else + { + acc1 = Vector128.MaxNative(acc1, current1); + acc2 = Vector128.MaxNative(acc2, current2); + signBits &= current1 & current2; + } + + nan |= Vector128.IsNaN(current1) | Vector128.IsNaN(current2); + i += 2 * Vector128.Count; } - curResult = TMinMaxOperator.Invoke(curResult, current); + acc1 = IsNativeMin() ? Vector128.MinNative(acc1, acc2) : Vector128.MaxNative(acc1, acc2); } - return curResult; + if (i != length) + { + Vector128 current = Vector128.LoadUnsafe(ref xRef, (uint)i); + if (IsNativeMin()) + { + acc1 = Vector128.MinNative(acc1, current); + signBits |= current; + } + else + { + acc1 = Vector128.MaxNative(acc1, current); + signBits &= current; + } + + nan |= Vector128.IsNaN(current); + } + + anyNaN = nan != Vector128.Zero; + anyOppositeSignBit = IsNativeMin() ? + signBits.ExtractMostSignificantBits() != 0 : + (~signBits).ExtractMostSignificantBits() != 0; + + return IsNativeMin() ? + HorizontalAggregate>(acc1) : + HorizontalAggregate>(acc1); + } + + /// Returns the first NaN among the elements at , a whole number of 128-bit vectors that contains one. + [MethodImpl(MethodImplOptions.NoInlining)] // cold: called at most once per reduction; keeps the caller within the inlining budget + private static T FirstNaN128(ref T xRef, int length) + where T : INumberBase + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + Debug.Assert(length >= Vector128.Count && length % Vector128.Count == 0); + + for (int i = 0; i < length; i += Vector128.Count) + { + Vector128 current = Vector128.LoadUnsafe(ref xRef, (uint)i); + Vector128 nanMask = Vector128.IsNaN(current); + if (nanMask != Vector128.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + Debug.Fail("The block must contain a NaN."); + return default!; } } } diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs index 295955d9497ea6..8a6374546cdc25 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs @@ -3115,6 +3115,56 @@ public void ScalarSpanDestination_ThrowsForOverlappingInputsWithOutputs(ScalarSp } #endregion + #region MinNumber/MaxNumber + // The multi-block coverage of Min_LongLengths_* for the Number variants, which share the block-reduction core. + [Fact] + public void MinNumber_LongLengths() => + AssertMinMaxLongLengths(TensorPrimitives.MinNumber, T.MinNumber); + + [Fact] + public void MaxNumber_LongLengths() => + AssertMinMaxLongLengths(TensorPrimitives.MaxNumber, T.MaxNumber); + + [Fact] + public void MinNumber_LongLengths_MinimumAtBlockBoundaries() => + AssertMinMaxLongLengthsValues(TensorPrimitives.MinNumber, fill: ConvertFromSingle(2), best: ConvertFromSingle(1)); + + [Fact] + public void MaxNumber_LongLengths_MaximumAtBlockBoundaries() => + AssertMinMaxLongLengthsValues(TensorPrimitives.MaxNumber, fill: ConvertFromSingle(1), best: ConvertFromSingle(2)); + + [Fact] + public void MinNumber_LongLengths_Negative0LesserThanPositive0() => + AssertMinMaxLongLengthsValues(TensorPrimitives.MinNumber, fill: Zero, best: NegativeZero); + + [Fact] + public void MaxNumber_LongLengths_Positive0GreaterThanNegative0() => + AssertMinMaxLongLengthsValues(TensorPrimitives.MaxNumber, fill: NegativeZero, best: Zero); + + [Fact] + public void MinNumber_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(TensorPrimitives.MinNumber); + + [Fact] + public void MaxNumber_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(TensorPrimitives.MaxNumber); + + // The vectorized float/double paths return the first NaN of the input, as Min/Max do; this pins that behavior across block boundaries. + [Fact] + public void MinNumber_LongLengths_FirstNaNReturned() + { + if (typeof(T) != typeof(float) && typeof(T) != typeof(double)) return; + AssertMinMaxLongLengthsFirstNaN(TensorPrimitives.MinNumber, fill: ConvertFromSingle(1), better: ConvertFromSingle(-1)); + } + + [Fact] + public void MaxNumber_LongLengths_FirstNaNReturned() + { + if (typeof(T) != typeof(float) && typeof(T) != typeof(double)) return; + AssertMinMaxLongLengthsFirstNaN(TensorPrimitives.MaxNumber, fill: ConvertFromSingle(1), better: ConvertFromSingle(2)); + } + #endregion + #region IsXx public static IEnumerable SpanDestinationIsFunctionsToTest() { diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs index f428af07f79d64..3a3c52214b68d1 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs @@ -1825,6 +1825,27 @@ public void Max_TwoTensors_ThrowsForOverlappingInputsWithOutputs() AssertExtensions.Throws("destination", () => Max(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(3, 2))); AssertExtensions.Throws("destination", () => Max(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(5, 2))); } + + // The same multi-block coverage as Min_LongLengths_* for the other reductions that share the block-reduction core. + [Fact] + public void Max_LongLengths() => + AssertMinMaxLongLengths(Max, Max); + + [Fact] + public void Max_LongLengths_MaximumAtBlockBoundaries() => + AssertMinMaxLongLengthsValues(Max, fill: ConvertFromSingle(1), best: ConvertFromSingle(2)); + + [Fact] + public void Max_LongLengths_FirstNaNReturned() => + AssertMinMaxLongLengthsFirstNaN(Max, fill: ConvertFromSingle(1), better: ConvertFromSingle(2)); // a larger value in an earlier block must not beat the NaN + + [Fact] + public void Max_LongLengths_Positive0GreaterThanNegative0() => + AssertMinMaxLongLengthsValues(Max, fill: NegativeZero, best: Zero); + + [Fact] + public void Max_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(Max); #endregion #region MaxMagnitude @@ -2010,6 +2031,32 @@ public void MaxMagnitude_TwoTensors_ThrowsForOverlappingInputsWithOutputs() AssertExtensions.Throws("destination", () => MaxMagnitude(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(3, 2))); AssertExtensions.Throws("destination", () => MaxMagnitude(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(5, 2))); } + + [Fact] + public void MaxMagnitude_LongLengths() => + AssertMinMaxLongLengths(MaxMagnitude, MaxMagnitude); + + [Fact] + public void MaxMagnitude_LongLengths_MaximumAtBlockBoundaries() + { + AssertMinMaxLongLengthsValues(MaxMagnitude, fill: ConvertFromSingle(1), best: ConvertFromSingle(2)); + if (HasNegativeValues) + { + AssertMinMaxLongLengthsValues(MaxMagnitude, fill: ConvertFromSingle(1), best: ConvertFromSingle(-2)); + } + } + + [Fact] + public void MaxMagnitude_LongLengths_FirstNaNReturned() => + AssertMinMaxLongLengthsFirstNaN(MaxMagnitude, fill: ConvertFromSingle(1), better: ConvertFromSingle(-2)); + + [Fact] + public void MaxMagnitude_LongLengths_Positive0GreaterThanNegative0() => + AssertMinMaxLongLengthsValues(MaxMagnitude, fill: NegativeZero, best: Zero); + + [Fact] + public void MaxMagnitude_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(MaxMagnitude); #endregion #region Min @@ -2195,6 +2242,137 @@ public void Min_TwoTensors_ThrowsForOverlappingInputsWithOutputs() AssertExtensions.Throws("destination", () => Min(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(3, 2))); AssertExtensions.Throws("destination", () => Min(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(5, 2))); } + + // Lengths that span several blocks of the block reduction behind Min/Max/MinMagnitude/MaxMagnitude/MinNumber/MaxNumber + // (Helpers.TensorLengths stops at 256). + protected static readonly int[] s_minMaxLongLengths = [255, 256, 257, 511, 512, 513, 1023, 1024, 1025, 2047, 2048, 2049, 4097, 65539]; + + protected static IEnumerable MinMaxLongPositions(int tensorLength) => + new[] { 0, 1, 15, 16, 17, 255, 256, 257, 511, 512, 513, tensorLength / 2, tensorLength - 2, tensorLength - 1 }.Where(i => i < tensorLength).Distinct(); + + protected delegate T MinMaxReduction(ReadOnlySpan x); + + /// Whether represents -1 (the magnitude tests place a negative value only then). + protected bool HasNegativeValues => !ConvertFromSingle(-1).Equals(Zero) && Min(ConvertFromSingle(-1), Zero).Equals(ConvertFromSingle(-1)); + + /// A NaN whose payload identifies it, so that a test can tell which NaN of the input was returned. + protected T NaNWithPayload(int payload) => ConvertFromSingle(BitConverter.Int32BitsToSingle(0x7FC00000 | payload)); + + /// Asserts bit-for-bit equality, which unlike tells -0 from +0 and one NaN payload from another. + protected static void AssertEqualBits(T expected, T actual) + { + if (typeof(T) == typeof(float)) + { + Assert.Equal(BitConverter.SingleToInt32Bits((float)(object)expected), BitConverter.SingleToInt32Bits((float)(object)actual)); + } + else if (typeof(T) == typeof(double)) + { + Assert.Equal(BitConverter.DoubleToInt64Bits((double)(object)expected), BitConverter.DoubleToInt64Bits((double)(object)actual)); + } + else if (typeof(T) == typeof(Half)) + { + Assert.Equal(BitConverter.HalfToInt16Bits((Half)(object)expected), BitConverter.HalfToInt16Bits((Half)(object)actual)); + } + else + { + Assert.Equal(expected, actual); + } + } + + /// The reduction of random data of every long length must equal the scalar fold with the operator. + protected void AssertMinMaxLongLengths(MinMaxReduction reduction, Func scalar) + { + Assert.All(s_minMaxLongLengths, tensorLength => + { + using BoundedMemory x = CreateAndFillTensor(tensorLength); + T expected = x[0]; + for (int i = 1; i < tensorLength; i++) + { + expected = scalar(expected, x[i]); + } + + AssertEqualBits(expected, reduction(x.Span)); + }); + } + + /// Fills with and places at each block-boundary position; the reduction must return bit for bit. + protected void AssertMinMaxLongLengthsValues(MinMaxReduction reduction, T fill, T best) + { + Assert.All(s_minMaxLongLengths, tensorLength => + { + foreach (int position in MinMaxLongPositions(tensorLength)) + { + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(fill); + x[position] = best; + AssertEqualBits(best, reduction(x.Span)); + } + }); + } + + /// + /// A NaN at any block-boundary position must be returned although a better value sits in an earlier block and a second NaN at the end; + /// for float and double it must be that first NaN, payload included. + /// + protected void AssertMinMaxLongLengthsFirstNaN(MinMaxReduction reduction, T fill, T better) + { + if (!IsFloatingPoint) return; + + Assert.All(s_minMaxLongLengths, tensorLength => + { + foreach (int position in MinMaxLongPositions(tensorLength)) + { + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(fill); + x[0] = better; + x[tensorLength - 1] = NaNWithPayload(2); + x[position] = NaNWithPayload(1); + + T actual = reduction(x.Span); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + AssertEqualBits(NaNWithPayload(1), actual); + } + else + { + Assert.Equal(ConvertFromSingle(float.NaN), actual); + } + } + }); + } + + /// All +0 must reduce to +0 and all -0 to -0, bit for bit. + protected void AssertMinMaxLongLengthsAllZero(MinMaxReduction reduction) + { + Assert.All(s_minMaxLongLengths, tensorLength => + { + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(Zero); + AssertEqualBits(Zero, reduction(x.Span)); + x.Span.Fill(NegativeZero); + AssertEqualBits(NegativeZero, reduction(x.Span)); + }); + } + + [Fact] + public void Min_LongLengths() => + AssertMinMaxLongLengths(Min, Min); + + [Fact] + public void Min_LongLengths_MinimumAtBlockBoundaries() => + AssertMinMaxLongLengthsValues(Min, fill: ConvertFromSingle(2), best: ConvertFromSingle(1)); + + [Fact] + public void Min_LongLengths_FirstNaNReturned() => + AssertMinMaxLongLengthsFirstNaN(Min, fill: ConvertFromSingle(1), better: ConvertFromSingle(-1)); // a smaller value in an earlier block must not beat the NaN + + [Fact] + public void Min_LongLengths_Negative0LesserThanPositive0() => + AssertMinMaxLongLengthsValues(Min, fill: Zero, best: NegativeZero); + + [Fact] + public void Min_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(Min); #endregion #region MinMagnitude @@ -2378,6 +2556,32 @@ public void MinMagnitude_TwoTensors_ThrowsForOverlappingInputsWithOutputs() AssertExtensions.Throws("destination", () => MinMagnitude(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(3, 2))); AssertExtensions.Throws("destination", () => MinMagnitude(array.AsSpan(1, 2), array.AsSpan(4, 2), array.AsSpan(5, 2))); } + + [Fact] + public void MinMagnitude_LongLengths() => + AssertMinMaxLongLengths(MinMagnitude, MinMagnitude); + + [Fact] + public void MinMagnitude_LongLengths_MinimumAtBlockBoundaries() + { + AssertMinMaxLongLengthsValues(MinMagnitude, fill: ConvertFromSingle(2), best: ConvertFromSingle(1)); + if (HasNegativeValues) + { + AssertMinMaxLongLengthsValues(MinMagnitude, fill: ConvertFromSingle(2), best: ConvertFromSingle(-1)); + } + } + + [Fact] + public void MinMagnitude_LongLengths_FirstNaNReturned() => + AssertMinMaxLongLengthsFirstNaN(MinMagnitude, fill: ConvertFromSingle(2), better: ConvertFromSingle(-1)); + + [Fact] + public void MinMagnitude_LongLengths_Negative0LesserThanPositive0() => + AssertMinMaxLongLengthsValues(MinMagnitude, fill: Zero, best: NegativeZero); + + [Fact] + public void MinMagnitude_LongLengths_AllZero() => + AssertMinMaxLongLengthsAllZero(MinMagnitude); #endregion #region Multiply From 0e25b3ecb5cf9410a97417af5a91cb610ca986bb Mon Sep 17 00:00:00 2001 From: Niklas Schilli Date: Thu, 17 Sep 2026 12:24:46 +0200 Subject: [PATCH 2/5] Use block accumulation in TensorPrimitives.AggregateAnyAll AggregateAnyAll backs the 36 IsXxAny / IsXxAll methods (IsNaNAny, IsFiniteAll, IsNegativeAny, IsZeroAll, ...). Its vector loops tested and branched on every vector: load, the operator's compare, a horizontal test (vptest, or for All a compare against zero and vptest) and a conditional branch, plus the loop overhead, so the loop was front-end bound at 8 instructions and two branches per vector, and its 512-bit path was no faster than the 256-bit one. The vector paths (512/256/128, kept symmetric) now fold the whole vectors in blocks of 32 vectors into two independent accumulators with no branch on the data and decide once per block, then fold the remaining whole vectors (fewer than a block) the same way, and finally handle any remaining elements with one vector that overlaps the last whole one, as before. The index is a nuint so that the loads need no zero extension. IAnyAllAggregator gains Accumulate(accumulator, result) per width, the fold of an operator result into a block accumulator, in which a lane with all bits set means that the block settles the result. AnyAggregator ORs the result in; AllAggregator ORs its complement in (a lane where the operator was false), so both use a zero identity, an OR to merge the two accumulators and the same block test (AnyWhereAllBitsSet). Accumulating the complement rather than AND-ing the results also lets the JIT fold the negation into an operator that ends in one, such as IsFinite. The per-vector ShouldEarlyExit overloads remain in use for the final overlapping vector; the scalar path is unchanged. The results are identical: a hit is still found, only its detection is deferred to the end of its block, and every read stays within the span. The per-width methods are NoInlining roots, as in MinMaxCore, so that the operator and the aggregator stay inlined. Hot loop for float on AVX2 (16 floats per iteration): 2 loads, 2 vcmpneqps, 2 vorps, add, cmp, jb; one vorps, vpcmpeqd x2, vptest, jne per 32 vectors. On AVX-512 the JIT goes through a mask register per vector (vcmpneqps k1 + vpmovm2d + vorps). Measured on a Ryzen 7 7800X3D (Zen 4, AVX-512), 65536 elements, GElem/s: BenchmarkDotNet, HPC AnyAllBenchmarks, package 10.0.12 vs this change, no hit (the whole span is scanned): IsNaNAny 23.4 -> 32.5 (AVX-512 path), 23.9 -> 31.5 (AVX2 path) IsFiniteAll 22.3 -> 28.9, 12.6 -> 28.9 IsNegativeAny 23.0 -> 30.5, 24.5 -> 35.6 IsNaNAny 14.8 -> 17.8, 12.3 -> 18.0 Hit at N/2: IsNaNAny 45 -> 65 (the block shape reads at most one extra block past the hit); hit at N-1: the same as no hit. Stopwatch A/B of 21 vectorized instantiations against the unmodified library, no hit: 1.02x to 1.32x on the AVX-512 path, 1.04x to 2.05x on the AVX2 path (IsFiniteAll 14.3 -> 29.1, IsZeroAll 21.9 -> 33.7, IsOddIntegerAny 29.3 -> 53.4). The one exception is IsSubnormalAny on the AVX-512 path, 20.8 -> 19.1: the old shape tested the compare's mask register directly (kortestw), the block shape has to materialize it (vpmovm2d) to OR it in, and with that operator's four instructions per vector the double-pumped 512-bit units are the bound on Zen 4; on the AVX2 path it is 17.5 -> 23.3. Verified identical to the unmodified library over a differential matrix (all 36 methods x float/double/Half/int/long/byte/ushort x 33 lengths up to 65539 x uniform fills, one deciding element at every vector, block and ragged boundary, two deciding elements, random mixes: 253,555 cases, on the 512, 256 and 128-bit paths). Tests place the deciding element at every block, vector and ragged boundary of long inputs for IsNaNAny/All, IsFiniteAll/Any, IsNegativeAny, IsPositiveAny and IsZeroAll/Any. Co-Authored-By: Claude Fable 5.1 --- .../TensorPrimitives.IBooleanUnaryOperator.cs | 271 ++++++++++++++---- .../tests/TensorPrimitives.Generic.cs | 93 ++++++ 2 files changed, 314 insertions(+), 50 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs index 45c33f35e7c1ab..3f4f6eb7411387 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs @@ -21,6 +21,12 @@ private interface IBooleanUnaryOperator static abstract Vector512 Invoke(Vector512 x); } + /// + /// Combines the results of into an Any/All decision. The vectorized loops of + /// fold the per-vector results of a block of vectors into an accumulator + /// with Accumulate, starting from zero, and decide once per block: a lane of the accumulator with all bits set means + /// that the block contains an element which settles the result, and the loop exits with !DefaultResult. + /// private interface IAnyAllAggregator { static abstract bool DefaultResult { get; } @@ -28,6 +34,13 @@ private interface IAnyAllAggregator static abstract bool ShouldEarlyExit(Vector128 result); static abstract bool ShouldEarlyExit(Vector256 result); static abstract bool ShouldEarlyExit(Vector512 result); + + /// Folds an operator result into a block accumulator: a lane of the accumulator ends up with all bits set if holds for any result folded into it. + static abstract Vector128 Accumulate(Vector128 accumulator, Vector128 result); + /// + static abstract Vector256 Accumulate(Vector256 accumulator, Vector256 result); + /// + static abstract Vector512 Accumulate(Vector512 accumulator, Vector512 result); } private readonly struct AnyAggregator : IAnyAllAggregator @@ -39,6 +52,11 @@ private interface IAnyAllAggregator public static bool ShouldEarlyExit(Vector128 result) => Vector128.AnyWhereAllBitsSet(result); public static bool ShouldEarlyExit(Vector256 result) => Vector256.AnyWhereAllBitsSet(result); public static bool ShouldEarlyExit(Vector512 result) => Vector512.AnyWhereAllBitsSet(result); + + // A lane where the operator was true stays set. + public static Vector128 Accumulate(Vector128 accumulator, Vector128 result) => accumulator | result; + public static Vector256 Accumulate(Vector256 accumulator, Vector256 result) => accumulator | result; + public static Vector512 Accumulate(Vector512 accumulator, Vector512 result) => accumulator | result; } private readonly struct AllAggregator : IAnyAllAggregator @@ -61,6 +79,13 @@ public static bool ShouldEarlyExit(Vector512 result) => typeof(T) == typeof(float) ? Vector512.EqualsAny(result.AsUInt32(), Vector512.Zero) : typeof(T) == typeof(double) ? Vector512.EqualsAny(result.AsUInt64(), Vector512.Zero) : Vector512.EqualsAny(result, Vector512.Zero); + + // A lane where the operator was false (its result is zero) becomes all bits set and stays set. Accumulating the + // complement rather than AND-ing the results keeps the block test the same as for Any and lets the JIT fold the + // complement into an operator that ends in a negation (such as IsFinite). + public static Vector128 Accumulate(Vector128 accumulator, Vector128 result) => accumulator | ~result; + public static Vector256 Accumulate(Vector256 accumulator, Vector256 result) => accumulator | ~result; + public static Vector512 Accumulate(Vector512 accumulator, Vector512 result) => accumulator | ~result; } private static bool All(ReadOnlySpan x) @@ -71,107 +96,253 @@ private static bool Any(ReadOnlySpan x) where TOperator : struct, IBooleanUnaryOperator => AggregateAnyAll>(x); + /// Vectors per block in the vectorized paths of . Must be even. + private const int AnyAllBlockVectors = 32; + private static bool AggregateAnyAll(ReadOnlySpan x) where TOperator : struct, IBooleanUnaryOperator where TAnyAll : struct, IAnyAllAggregator { Debug.Assert(!x.IsEmpty); + if (Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported && x.Length >= Vector512.Count) + { + return AggregateAnyAllVectorized512(x); + } + + if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported && x.Length >= Vector256.Count) + { + return AggregateAnyAllVectorized256(x); + } + + if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported && x.Length >= Vector128.Count) + { + return AggregateAnyAllVectorized128(x); + } + + ref T xRef = ref MemoryMarshal.GetReference(x); + for (int i = 0; i < x.Length; i++) + { + if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Unsafe.Add(ref xRef, i)))) + { + return !TAnyAll.DefaultResult; + } + } + + return TAnyAll.DefaultResult; + } + + /// The 512-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is folded into two independent accumulators with no branch on the + /// data, and the exit decision is made once per block, so a hit is detected after at most one block of extra reads. + /// Blocks are visited in order and the whole input lies within the span, so the result is the same as with a test per vector. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator and the aggregator inlined + private static bool AggregateAnyAllVectorized512(ReadOnlySpan x) + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported); + Debug.Assert(x.Length >= Vector512.Count); + ref T xRef = ref MemoryMarshal.GetReference(x); - int i = 0, oneVectorFromEnd; + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector512.Count; + nuint i = 0; - if (Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported) + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector512.Count); + if (length >= blockLength) { - oneVectorFromEnd = x.Length - Vector512.Count; - if (i <= oneVectorFromEnd) + nuint oneBlockFromEnd = length - blockLength; + do { - // Loop handling one vector at a time. + Vector512 accumulator0 = Vector512.Zero; + Vector512 accumulator1 = Vector512.Zero; + nuint blockEnd = i + blockLength; do { - if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, (uint)i)))) - { - return !TAnyAll.DefaultResult; - } - - i += Vector512.Count; + accumulator0 = TAnyAll.Accumulate(accumulator0, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i))); + accumulator1 = TAnyAll.Accumulate(accumulator1, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i + (uint)Vector512.Count))); + i += (uint)(2 * Vector512.Count); } - while (i <= oneVectorFromEnd); + while (i < blockEnd); - // Handle any remaining elements with a final vector. - if (i != x.Length && - TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count))))) + if (Vector512.AnyWhereAllBitsSet(accumulator0 | accumulator1)) { return !TAnyAll.DefaultResult; } + } + while (i <= oneBlockFromEnd); + } + + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector512 accumulator = Vector512.Zero; + do + { + accumulator = TAnyAll.Accumulate(accumulator, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i))); + i += (uint)Vector512.Count; + } + while (i <= oneVectorFromEnd); - return TAnyAll.DefaultResult; + if (Vector512.AnyWhereAllBitsSet(accumulator)) + { + return !TAnyAll.DefaultResult; } } - if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported) + // Handle any remaining elements with a final vector. + if (i != length && + TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, oneVectorFromEnd)))) + { + return !TAnyAll.DefaultResult; + } + + return TAnyAll.DefaultResult; + } + + /// The 256-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is folded into two independent accumulators with no branch on the + /// data, and the exit decision is made once per block, so a hit is detected after at most one block of extra reads. + /// Blocks are visited in order and the whole input lies within the span, so the result is the same as with a test per vector. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator and the aggregator inlined + private static bool AggregateAnyAllVectorized256(ReadOnlySpan x) + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported); + Debug.Assert(x.Length >= Vector256.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector256.Count; + nuint i = 0; + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector256.Count); + if (length >= blockLength) { - oneVectorFromEnd = x.Length - Vector256.Count; - if (i <= oneVectorFromEnd) + nuint oneBlockFromEnd = length - blockLength; + do { - // Loop handling one vector at a time. + Vector256 accumulator0 = Vector256.Zero; + Vector256 accumulator1 = Vector256.Zero; + nuint blockEnd = i + blockLength; do { - if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)i)))) - { - return !TAnyAll.DefaultResult; - } - - i += Vector256.Count; + accumulator0 = TAnyAll.Accumulate(accumulator0, TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, i))); + accumulator1 = TAnyAll.Accumulate(accumulator1, TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, i + (uint)Vector256.Count))); + i += (uint)(2 * Vector256.Count); } - while (i <= oneVectorFromEnd); + while (i < blockEnd); - // Handle any remaining elements with a final vector. - if (i != x.Length && - TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count))))) + if (Vector256.AnyWhereAllBitsSet(accumulator0 | accumulator1)) { return !TAnyAll.DefaultResult; } + } + while (i <= oneBlockFromEnd); + } + + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector256 accumulator = Vector256.Zero; + do + { + accumulator = TAnyAll.Accumulate(accumulator, TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, i))); + i += (uint)Vector256.Count; + } + while (i <= oneVectorFromEnd); - return TAnyAll.DefaultResult; + if (Vector256.AnyWhereAllBitsSet(accumulator)) + { + return !TAnyAll.DefaultResult; } } - if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported) + // Handle any remaining elements with a final vector. + if (i != length && + TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, oneVectorFromEnd)))) + { + return !TAnyAll.DefaultResult; + } + + return TAnyAll.DefaultResult; + } + + /// The 128-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// Every block of up to vectors is folded into two independent accumulators with no branch on the + /// data, and the exit decision is made once per block, so a hit is detected after at most one block of extra reads. + /// Blocks are visited in order and the whole input lies within the span, so the result is the same as with a test per vector. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator and the aggregator inlined + private static bool AggregateAnyAllVectorized128(ReadOnlySpan x) + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported); + Debug.Assert(x.Length >= Vector128.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector128.Count; + nuint i = 0; + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector128.Count); + if (length >= blockLength) { - oneVectorFromEnd = x.Length - Vector128.Count; - if (i <= oneVectorFromEnd) + nuint oneBlockFromEnd = length - blockLength; + do { - // Loop handling one vector at a time. + Vector128 accumulator0 = Vector128.Zero; + Vector128 accumulator1 = Vector128.Zero; + nuint blockEnd = i + blockLength; do { - if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)i)))) - { - return !TAnyAll.DefaultResult; - } - - i += Vector128.Count; + accumulator0 = TAnyAll.Accumulate(accumulator0, TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, i))); + accumulator1 = TAnyAll.Accumulate(accumulator1, TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, i + (uint)Vector128.Count))); + i += (uint)(2 * Vector128.Count); } - while (i <= oneVectorFromEnd); + while (i < blockEnd); - // Handle any remaining elements with a final vector. - if (i != x.Length && - TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count))))) + if (Vector128.AnyWhereAllBitsSet(accumulator0 | accumulator1)) { return !TAnyAll.DefaultResult; } - - return TAnyAll.DefaultResult; } + while (i <= oneBlockFromEnd); } - while (i < x.Length) + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) { - if (TAnyAll.ShouldEarlyExit(TOperator.Invoke(Unsafe.Add(ref xRef, i)))) + Vector128 accumulator = Vector128.Zero; + do + { + accumulator = TAnyAll.Accumulate(accumulator, TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, i))); + i += (uint)Vector128.Count; + } + while (i <= oneVectorFromEnd); + + if (Vector128.AnyWhereAllBitsSet(accumulator)) { return !TAnyAll.DefaultResult; } + } - i++; + // Handle any remaining elements with a final vector. + if (i != length && + TAnyAll.ShouldEarlyExit(TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, oneVectorFromEnd)))) + { + return !TAnyAll.DefaultResult; } return TAnyAll.DefaultResult; diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs index 8a6374546cdc25..f4f68ce4b4912e 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs @@ -3313,6 +3313,99 @@ public void SpanDestionIsAny_AllLengths(SpanIsAllAnyDelegate tensorPrimitivesMet Assert.Equal(expected, actual); }); } + + // The vectorized Any/All paths decide once per block of vectors. These place the element that settles the result at every + // block, vector and ragged boundary of long inputs, where the block loop, the ragged vectors and the final overlapping vector meet. + private static IEnumerable IsAnyAllLongPositions(int tensorLength) + { + var positions = new List { 0, 1, tensorLength / 2, tensorLength - 2, tensorLength - 1 }; + foreach (int vectorBytes in new[] { 16, 32, 64 }) + { + int vector = vectorBytes / sizeof(T), block = 32 * vector; + int afterLastWholeVector = tensorLength - (tensorLength % vector), afterLastWholeBlock = tensorLength - (tensorLength % block); + positions.AddRange([vector - 1, vector, vector + 1, block - 1, block, block + 1, (2 * block) - 1, 2 * block, + afterLastWholeVector - 1, afterLastWholeVector, afterLastWholeBlock - 1, afterLastWholeBlock]); + } + + return positions.Where(i => i >= 0 && i < tensorLength).Distinct(); + } + + /// + /// Filled with the aggregation must return for every long length; with + /// placed at any boundary position, alone or together with one at the end, it must return the opposite. + /// + private void AssertIsAnyAllLongLengths(SpanIsAllAnyDelegate tensorPrimitivesMethod, T fill, T hit, bool expectedWithoutHit) + { + Assert.All(s_minMaxLongLengths, tensorLength => + { + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(fill); + Assert.Equal(expectedWithoutHit, tensorPrimitivesMethod(x)); + + foreach (int position in IsAnyAllLongPositions(tensorLength)) + { + x[position] = hit; + Assert.Equal(!expectedWithoutHit, tensorPrimitivesMethod(x)); + + x[tensorLength - 1] = hit; + Assert.Equal(!expectedWithoutHit, tensorPrimitivesMethod(x)); + + x[position] = fill; + x[tensorLength - 1] = fill; + } + }); + } + + [Fact] + public void IsNaNAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAny, fill: ConvertFromSingle(1), hit: ConvertFromSingle(float.NaN), expectedWithoutHit: false); + } + + [Fact] + public void IsNaNAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAll, fill: ConvertFromSingle(float.NaN), hit: ConvertFromSingle(1), expectedWithoutHit: true); + } + + [Fact] + public void IsFiniteAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: ConvertFromSingle(1), hit: ConvertFromSingle(float.PositiveInfinity), expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: ConvertFromSingle(-1), hit: ConvertFromSingle(float.NaN), expectedWithoutHit: true); + } + + [Fact] + public void IsFiniteAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAny, fill: ConvertFromSingle(float.NegativeInfinity), hit: Zero, expectedWithoutHit: false); + } + + [Fact] + public void IsNegativeAny_LongLengths() + { + if (!HasNegativeValues) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAny, fill: ConvertFromSingle(1), hit: ConvertFromSingle(-1), expectedWithoutHit: false); + } + + [Fact] + public void IsPositiveAny_LongLengths() + { + if (!HasNegativeValues) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAny, fill: ConvertFromSingle(-1), hit: ConvertFromSingle(1), expectedWithoutHit: false); + } + + [Fact] + public void IsZeroAll_LongLengths() => + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAll, fill: Zero, hit: ConvertFromSingle(1), expectedWithoutHit: true); + + [Fact] + public void IsZeroAny_LongLengths() => + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAny, fill: ConvertFromSingle(1), hit: Zero, expectedWithoutHit: false); #endregion #region HammingDistance From 5724acecee4a0bd4fb0968221d132c18f081fdf0 Mon Sep 17 00:00:00 2001 From: Niklas Schilli Date: Thu, 17 Sep 2026 13:09:04 +0200 Subject: [PATCH 3/5] Test that two distinct NaN payloads reduce to one of the input NaNs Min/Max/MinMagnitude/MaxMagnitude with two NaNs of different payloads at every pair of block-boundary positions over the long lengths: float and double must return the first NaN bit for bit; Half (which reaches the core as short and has no element-level NaN check) must return one of the two input NaNs rather than a canonical NaN or a value. The payloads sit above bit 13 so they survive the narrowing to Half, and the test asserts the two inputs really differ. Passes on the unmodified library as well; nothing in the suite covered this before. Co-Authored-By: Claude Fable 5.1 --- .../tests/TensorPrimitivesTests.cs | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs index 3a3c52214b68d1..f627412a2c9e99 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs @@ -1846,6 +1846,10 @@ public void Max_LongLengths_Positive0GreaterThanNegative0() => [Fact] public void Max_LongLengths_AllZero() => AssertMinMaxLongLengthsAllZero(Max); + + [Fact] + public void Max_LongLengths_TwoDistinctNaNs_ReturnsOneOfThem() => + AssertMinMaxLongLengthsTwoDistinctNaNs(Max, fill: ConvertFromSingle(2)); #endregion #region MaxMagnitude @@ -2057,6 +2061,10 @@ public void MaxMagnitude_LongLengths_Positive0GreaterThanNegative0() => [Fact] public void MaxMagnitude_LongLengths_AllZero() => AssertMinMaxLongLengthsAllZero(MaxMagnitude); + + [Fact] + public void MaxMagnitude_LongLengths_TwoDistinctNaNs_ReturnsOneOfThem() => + AssertMinMaxLongLengthsTwoDistinctNaNs(MaxMagnitude, fill: ConvertFromSingle(-2)); #endregion #region Min @@ -2341,6 +2349,62 @@ protected void AssertMinMaxLongLengthsFirstNaN(MinMaxReduction reduction, T fill }); } + /// + /// A NaN whose payload survives narrowing to (the low 13 bits of a single's payload are truncated), + /// so that two of them stay distinguishable through every floating-point . + /// + protected T NaNWithWidePayload(int payload) => ConvertFromSingle(BitConverter.Int32BitsToSingle(0x7FC00000 | (payload << 13))); + + /// + /// Two NaNs with different payloads at block-boundary positions must reduce to one of those two NaNs, bit for bit. + /// Which one is only specified for float and double (the first); for Half the choice depends on how the lanes pair up, + /// but the result must still be an input NaN and not a canonical NaN or a value. + /// + protected void AssertMinMaxLongLengthsTwoDistinctNaNs(MinMaxReduction reduction, T fill) + { + if (!IsFloatingPoint) return; + + T first = NaNWithWidePayload(1), second = NaNWithWidePayload(2); + Assert.NotEqual(ToBits(first), ToBits(second)); + + Assert.All(s_minMaxLongLengths, tensorLength => + { + int[] positions = MinMaxLongPositions(tensorLength).ToArray(); + foreach (int i in positions) + { + foreach (int j in positions) + { + if (i == j) continue; + + using BoundedMemory x = CreateTensor(tensorLength); + x.Span.Fill(fill); + x[i] = first; + x[j] = second; + + T actual = reduction(x.Span); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + AssertEqualBits(i < j ? first : second, actual); + } + else + { + long bits = ToBits(actual); + Assert.True(bits == ToBits(first) || bits == ToBits(second), + $"Length {tensorLength}, NaNs at {i} and {j}: got 0x{bits:X}, expected 0x{ToBits(first):X} or 0x{ToBits(second):X}"); + } + } + } + }); + } + + /// The bit pattern of a floating-point value, for comparisons that must tell NaN payloads apart. + protected static long ToBits(T value) => + typeof(T) == typeof(float) ? BitConverter.SingleToInt32Bits((float)(object)value) : + typeof(T) == typeof(double) ? BitConverter.DoubleToInt64Bits((double)(object)value) : + typeof(T) == typeof(Half) ? BitConverter.HalfToInt16Bits((Half)(object)value) : + typeof(T) == typeof(NFloat) ? BitConverter.DoubleToInt64Bits((double)(NFloat)(object)value) : + throw new NotSupportedException(typeof(T).Name); + /// All +0 must reduce to +0 and all -0 to -0, bit for bit. protected void AssertMinMaxLongLengthsAllZero(MinMaxReduction reduction) { @@ -2373,6 +2437,10 @@ public void Min_LongLengths_Negative0LesserThanPositive0() => [Fact] public void Min_LongLengths_AllZero() => AssertMinMaxLongLengthsAllZero(Min); + + [Fact] + public void Min_LongLengths_TwoDistinctNaNs_ReturnsOneOfThem() => + AssertMinMaxLongLengthsTwoDistinctNaNs(Min, fill: ConvertFromSingle(-1)); // a smaller value must not beat either NaN #endregion #region MinMagnitude @@ -2582,6 +2650,10 @@ public void MinMagnitude_LongLengths_Negative0LesserThanPositive0() => [Fact] public void MinMagnitude_LongLengths_AllZero() => AssertMinMaxLongLengthsAllZero(MinMagnitude); + + [Fact] + public void MinMagnitude_LongLengths_TwoDistinctNaNs_ReturnsOneOfThem() => + AssertMinMaxLongLengthsTwoDistinctNaNs(MinMagnitude, fill: ConvertFromSingle(-1)); #endregion #region Multiply From 5683256ea29bb2d9d15485d97bc8684a26f33c05 Mon Sep 17 00:00:00 2001 From: Niklas Schilli Date: Thu, 17 Sep 2026 21:21:19 +0200 Subject: [PATCH 4/5] Consume the comparison masks directly in the 512-bit TensorPrimitives.AggregateAnyAll loops On AVX-512 hardware the JIT keeps a 512-bit comparison result in a mask register, and OR-ing it into a vector accumulator first materializes it as a vector (vpmovm2d), so the block loops of AggregateAnyAll cost three vector instructions per vector. A select whose condition is the comparison lowers to a masked blend that consumes the mask register as it is, and a select whose other operand is the zero vector costs no extra instruction (the JIT even folds it into a zero-masking move under the inverted comparison), whereas an all-ones operand is rematerialized inside the loop (vpternlogd). The 512-bit loops therefore accumulate by clearing lanes: the accumulators start from all bits set, IAnyAllAggregator.ClearSettled (which replaces the 512-bit Accumulate) clears the lanes whose result settles the aggregation, the two accumulators are combined with an AND and a block is decided by any lane being zero. Any clears where the result is true (ConditionalSelect(result, Zero, acc)) and All where it is false (ConditionalSelect(result, acc, Zero)). The 128- and 256-bit loops keep vector accumulators, but each is now a result itself: the OR of Any results and the AND of All results, starting from the result that equals DefaultResult, so that ShouldEarlyExit decides the block. This removes the complement from All's accumulation ("acc | ~result"), which on AVX-512VL hardware running 256-bit vectors folded into the predicate of an integer comparison that the JIT can then not rewrite from its EVEX mask form back to a VEX vector comparison, costing a vpmovm2d per vector. The scalar loop moves to AggregateAnyAllScalar; results are unchanged. Ryzen 7 7800X3D, best of three runs, N = 4096 (L1-resident), GElem/s, before -> after, 512-bit path: IsNaNAny 38.6 -> 42.7, IsNaNAll 39.1 -> 45.6, IsFiniteAll 33.0 -> 39.9, IsFiniteAny 32.0 -> 39.8, IsInfinityAny 30.6 -> 36.4, IsIntegerAll 16.4 -> 20.8, IsOddIntegerAny 60.8 -> 72.5, IsNegativeAny 34.5 -> 41.4, IsNaNAny 16.2 -> 17.7. The 256-bit path (with and without AVX-512VL) and the 128-bit path are unchanged within noise. Co-Authored-By: Claude Opus 5 --- .../TensorPrimitives.IBooleanUnaryOperator.cs | 125 +++++++++++------- 1 file changed, 79 insertions(+), 46 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs index 3f4f6eb7411387..e59746c5f46fc0 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs @@ -24,8 +24,12 @@ private interface IBooleanUnaryOperator /// /// Combines the results of into an Any/All decision. The vectorized loops of /// fold the per-vector results of a block of vectors into an accumulator - /// with Accumulate, starting from zero, and decide once per block: a lane of the accumulator with all bits set means - /// that the block contains an element which settles the result, and the loop exits with !DefaultResult. + /// and decide once per block whether the block contains an element which settles the result, in which case they exit with + /// !DefaultResult. At 128 and 256 bits the accumulator is a result itself, the fold of the block's results with + /// Accumulate starting from the result that equals DefaultResult, so ShouldEarlyExit decides. At 512 bits + /// the operator's comparison leaves its result in a mask register, which a select consumes as it is (a masked blend) whereas a + /// bitwise operation would first have to expand the mask into a vector, so the accumulator starts from all bits set and + /// ClearSettled clears the lanes whose result settles the aggregation, which a zero lane then signals. /// private interface IAnyAllAggregator { @@ -35,12 +39,13 @@ private interface IAnyAllAggregator static abstract bool ShouldEarlyExit(Vector256 result); static abstract bool ShouldEarlyExit(Vector512 result); - /// Folds an operator result into a block accumulator: a lane of the accumulator ends up with all bits set if holds for any result folded into it. + /// Folds an operator result into the aggregation of the results folded so far, a result itself: the OR of Any results, the AND of All results. static abstract Vector128 Accumulate(Vector128 accumulator, Vector128 result); /// static abstract Vector256 Accumulate(Vector256 accumulator, Vector256 result); - /// - static abstract Vector512 Accumulate(Vector512 accumulator, Vector512 result); + + /// Clears the lanes of whose lane of settles the aggregation, that is, for which would hold. + static abstract Vector512 ClearSettled(Vector512 accumulator, Vector512 result); } private readonly struct AnyAggregator : IAnyAllAggregator @@ -53,10 +58,12 @@ private interface IAnyAllAggregator public static bool ShouldEarlyExit(Vector256 result) => Vector256.AnyWhereAllBitsSet(result); public static bool ShouldEarlyExit(Vector512 result) => Vector512.AnyWhereAllBitsSet(result); - // A lane where the operator was true stays set. public static Vector128 Accumulate(Vector128 accumulator, Vector128 result) => accumulator | result; public static Vector256 Accumulate(Vector256 accumulator, Vector256 result) => accumulator | result; - public static Vector512 Accumulate(Vector512 accumulator, Vector512 result) => accumulator | result; + + // A lane where the operator was true is cleared: the select's constant is the zero vector, which costs no instruction, + // and the JIT folds the selection of the other lanes into a zero-masking move under the inverted comparison. + public static Vector512 ClearSettled(Vector512 accumulator, Vector512 result) => Vector512.ConditionalSelect(result, Vector512.Zero, accumulator); } private readonly struct AllAggregator : IAnyAllAggregator @@ -65,29 +72,38 @@ private interface IAnyAllAggregator public static bool ShouldEarlyExit(bool result) => !result; - public static bool ShouldEarlyExit(Vector128 result) => - typeof(T) == typeof(float) ? Vector128.EqualsAny(result.AsUInt32(), Vector128.Zero) : - typeof(T) == typeof(double) ? Vector128.EqualsAny(result.AsUInt64(), Vector128.Zero) : - Vector128.EqualsAny(result, Vector128.Zero); - - public static bool ShouldEarlyExit(Vector256 result) => - typeof(T) == typeof(float) ? Vector256.EqualsAny(result.AsUInt32(), Vector256.Zero) : - typeof(T) == typeof(double) ? Vector256.EqualsAny(result.AsUInt64(), Vector256.Zero) : - Vector256.EqualsAny(result, Vector256.Zero); - - public static bool ShouldEarlyExit(Vector512 result) => - typeof(T) == typeof(float) ? Vector512.EqualsAny(result.AsUInt32(), Vector512.Zero) : - typeof(T) == typeof(double) ? Vector512.EqualsAny(result.AsUInt64(), Vector512.Zero) : - Vector512.EqualsAny(result, Vector512.Zero); - - // A lane where the operator was false (its result is zero) becomes all bits set and stays set. Accumulating the - // complement rather than AND-ing the results keeps the block test the same as for Any and lets the JIT fold the - // complement into an operator that ends in a negation (such as IsFinite). - public static Vector128 Accumulate(Vector128 accumulator, Vector128 result) => accumulator | ~result; - public static Vector256 Accumulate(Vector256 accumulator, Vector256 result) => accumulator | ~result; - public static Vector512 Accumulate(Vector512 accumulator, Vector512 result) => accumulator | ~result; + public static bool ShouldEarlyExit(Vector128 result) => AnyLaneZero(result); + public static bool ShouldEarlyExit(Vector256 result) => AnyLaneZero(result); + public static bool ShouldEarlyExit(Vector512 result) => AnyLaneZero(result); + + public static Vector128 Accumulate(Vector128 accumulator, Vector128 result) => accumulator & result; + public static Vector256 Accumulate(Vector256 accumulator, Vector256 result) => accumulator & result; + + // A lane where the operator was false (its result is zero) is cleared (see AnyAggregator). + public static Vector512 ClearSettled(Vector512 accumulator, Vector512 result) => Vector512.ConditionalSelect(result, accumulator, Vector512.Zero); } + /// Whether any lane of is zero. For the floating-point types the lanes are compared as integers: a lane of an operator result or of an accumulator is either all bits set (a NaN) or zero. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool AnyLaneZero(Vector128 vector) => + typeof(T) == typeof(float) ? Vector128.EqualsAny(vector.AsUInt32(), Vector128.Zero) : + typeof(T) == typeof(double) ? Vector128.EqualsAny(vector.AsUInt64(), Vector128.Zero) : + Vector128.EqualsAny(vector, Vector128.Zero); + + /// Whether any lane of is zero. For the floating-point types the lanes are compared as integers: a lane of an operator result or of an accumulator is either all bits set (a NaN) or zero. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool AnyLaneZero(Vector256 vector) => + typeof(T) == typeof(float) ? Vector256.EqualsAny(vector.AsUInt32(), Vector256.Zero) : + typeof(T) == typeof(double) ? Vector256.EqualsAny(vector.AsUInt64(), Vector256.Zero) : + Vector256.EqualsAny(vector, Vector256.Zero); + + /// Whether any lane of is zero. For the floating-point types the lanes are compared as integers: a lane of an operator result or of an accumulator is either all bits set (a NaN) or zero. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool AnyLaneZero(Vector512 vector) => + typeof(T) == typeof(float) ? Vector512.EqualsAny(vector.AsUInt32(), Vector512.Zero) : + typeof(T) == typeof(double) ? Vector512.EqualsAny(vector.AsUInt64(), Vector512.Zero) : + Vector512.EqualsAny(vector, Vector512.Zero); + private static bool All(ReadOnlySpan x) where TOperator : struct, IBooleanUnaryOperator => AggregateAnyAll>(x); @@ -120,6 +136,14 @@ private static bool AggregateAnyAll(ReadOnlySpan x) return AggregateAnyAllVectorized128(x); } + return AggregateAnyAllScalar(x); + } + + /// The scalar path of , used when vectorization is not supported or the input is too small to vectorize. + private static bool AggregateAnyAllScalar(ReadOnlySpan x) + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { ref T xRef = ref MemoryMarshal.GetReference(x); for (int i = 0; i < x.Length; i++) { @@ -137,6 +161,7 @@ private static bool AggregateAnyAll(ReadOnlySpan x) /// Every block of up to vectors is folded into two independent accumulators with no branch on the /// data, and the exit decision is made once per block, so a hit is detected after at most one block of extra reads. /// Blocks are visited in order and the whole input lies within the span, so the result is the same as with a test per vector. + /// The accumulators start from all bits set and have the lanes of the settling results cleared (see ). /// [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator and the aggregator inlined private static bool AggregateAnyAllVectorized512(ReadOnlySpan x) @@ -158,18 +183,18 @@ private static bool AggregateAnyAllVectorized512(ReadOnly nuint oneBlockFromEnd = length - blockLength; do { - Vector512 accumulator0 = Vector512.Zero; - Vector512 accumulator1 = Vector512.Zero; + Vector512 accumulator0 = Vector512.AllBitsSet; + Vector512 accumulator1 = Vector512.AllBitsSet; nuint blockEnd = i + blockLength; do { - accumulator0 = TAnyAll.Accumulate(accumulator0, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i))); - accumulator1 = TAnyAll.Accumulate(accumulator1, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i + (uint)Vector512.Count))); + accumulator0 = TAnyAll.ClearSettled(accumulator0, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i))); + accumulator1 = TAnyAll.ClearSettled(accumulator1, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i + (uint)Vector512.Count))); i += (uint)(2 * Vector512.Count); } while (i < blockEnd); - if (Vector512.AnyWhereAllBitsSet(accumulator0 | accumulator1)) + if (AnyLaneZero(accumulator0 & accumulator1)) { return !TAnyAll.DefaultResult; } @@ -180,15 +205,15 @@ private static bool AggregateAnyAllVectorized512(ReadOnly // The remaining whole vectors, fewer than a block. if (i <= oneVectorFromEnd) { - Vector512 accumulator = Vector512.Zero; + Vector512 accumulator = Vector512.AllBitsSet; do { - accumulator = TAnyAll.Accumulate(accumulator, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i))); + accumulator = TAnyAll.ClearSettled(accumulator, TOperator.Invoke(Vector512.LoadUnsafe(ref xRef, i))); i += (uint)Vector512.Count; } while (i <= oneVectorFromEnd); - if (Vector512.AnyWhereAllBitsSet(accumulator)) + if (AnyLaneZero(accumulator)) { return !TAnyAll.DefaultResult; } @@ -209,6 +234,8 @@ private static bool AggregateAnyAllVectorized512(ReadOnly /// Every block of up to vectors is folded into two independent accumulators with no branch on the /// data, and the exit decision is made once per block, so a hit is detected after at most one block of extra reads. /// Blocks are visited in order and the whole input lies within the span, so the result is the same as with a test per vector. + /// The accumulators are results themselves: the fold of the block's results, starting from the result that equals the default + /// (see ). /// [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator and the aggregator inlined private static bool AggregateAnyAllVectorized256(ReadOnlySpan x) @@ -223,6 +250,8 @@ private static bool AggregateAnyAllVectorized256(ReadOnly nuint oneVectorFromEnd = length - (uint)Vector256.Count; nuint i = 0; + Vector256 defaultResult = TAnyAll.DefaultResult ? Vector256.AllBitsSet : Vector256.Zero; + // Whole blocks: two accumulators, one decision per block. nuint blockLength = (uint)(AnyAllBlockVectors * Vector256.Count); if (length >= blockLength) @@ -230,8 +259,8 @@ private static bool AggregateAnyAllVectorized256(ReadOnly nuint oneBlockFromEnd = length - blockLength; do { - Vector256 accumulator0 = Vector256.Zero; - Vector256 accumulator1 = Vector256.Zero; + Vector256 accumulator0 = defaultResult; + Vector256 accumulator1 = defaultResult; nuint blockEnd = i + blockLength; do { @@ -241,7 +270,7 @@ private static bool AggregateAnyAllVectorized256(ReadOnly } while (i < blockEnd); - if (Vector256.AnyWhereAllBitsSet(accumulator0 | accumulator1)) + if (TAnyAll.ShouldEarlyExit(TAnyAll.Accumulate(accumulator0, accumulator1))) { return !TAnyAll.DefaultResult; } @@ -252,7 +281,7 @@ private static bool AggregateAnyAllVectorized256(ReadOnly // The remaining whole vectors, fewer than a block. if (i <= oneVectorFromEnd) { - Vector256 accumulator = Vector256.Zero; + Vector256 accumulator = defaultResult; do { accumulator = TAnyAll.Accumulate(accumulator, TOperator.Invoke(Vector256.LoadUnsafe(ref xRef, i))); @@ -260,7 +289,7 @@ private static bool AggregateAnyAllVectorized256(ReadOnly } while (i <= oneVectorFromEnd); - if (Vector256.AnyWhereAllBitsSet(accumulator)) + if (TAnyAll.ShouldEarlyExit(accumulator)) { return !TAnyAll.DefaultResult; } @@ -281,6 +310,8 @@ private static bool AggregateAnyAllVectorized256(ReadOnly /// Every block of up to vectors is folded into two independent accumulators with no branch on the /// data, and the exit decision is made once per block, so a hit is detected after at most one block of extra reads. /// Blocks are visited in order and the whole input lies within the span, so the result is the same as with a test per vector. + /// The accumulators are results themselves: the fold of the block's results, starting from the result that equals the default + /// (see ). /// [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator and the aggregator inlined private static bool AggregateAnyAllVectorized128(ReadOnlySpan x) @@ -295,6 +326,8 @@ private static bool AggregateAnyAllVectorized128(ReadOnly nuint oneVectorFromEnd = length - (uint)Vector128.Count; nuint i = 0; + Vector128 defaultResult = TAnyAll.DefaultResult ? Vector128.AllBitsSet : Vector128.Zero; + // Whole blocks: two accumulators, one decision per block. nuint blockLength = (uint)(AnyAllBlockVectors * Vector128.Count); if (length >= blockLength) @@ -302,8 +335,8 @@ private static bool AggregateAnyAllVectorized128(ReadOnly nuint oneBlockFromEnd = length - blockLength; do { - Vector128 accumulator0 = Vector128.Zero; - Vector128 accumulator1 = Vector128.Zero; + Vector128 accumulator0 = defaultResult; + Vector128 accumulator1 = defaultResult; nuint blockEnd = i + blockLength; do { @@ -313,7 +346,7 @@ private static bool AggregateAnyAllVectorized128(ReadOnly } while (i < blockEnd); - if (Vector128.AnyWhereAllBitsSet(accumulator0 | accumulator1)) + if (TAnyAll.ShouldEarlyExit(TAnyAll.Accumulate(accumulator0, accumulator1))) { return !TAnyAll.DefaultResult; } @@ -324,7 +357,7 @@ private static bool AggregateAnyAllVectorized128(ReadOnly // The remaining whole vectors, fewer than a block. if (i <= oneVectorFromEnd) { - Vector128 accumulator = Vector128.Zero; + Vector128 accumulator = defaultResult; do { accumulator = TAnyAll.Accumulate(accumulator, TOperator.Invoke(Vector128.LoadUnsafe(ref xRef, i))); @@ -332,7 +365,7 @@ private static bool AggregateAnyAllVectorized128(ReadOnly } while (i <= oneVectorFromEnd); - if (Vector128.AnyWhereAllBitsSet(accumulator)) + if (TAnyAll.ShouldEarlyExit(accumulator)) { return !TAnyAll.DefaultResult; } From ce7fc23b7938aa6d158c2168ec6036f602686ef4 Mon Sep 17 00:00:00 2001 From: Niklas Schilli Date: Thu, 17 Sep 2026 21:22:41 +0200 Subject: [PATCH 5/5] Fold blocks by an unsigned minimum or maximum of the elements' bits in TensorPrimitives.AggregateAnyAll Most of the IEEE 754 classifications are a single unsigned comparison of the element's bits against a constant: a NaN has an absolute bit pattern above infinity's, a finite value one below it, a negative value has bits above the largest pattern with the sign bit clear, and a normal or subnormal value falls in a range that a subtraction turns into a comparison with wrap-around. For such an operator an Any/All aggregation does not need the comparison per vector: it folds the keys of a block with one unsigned minimum or maximum per vector and compares the fold against the threshold once per block. Any looks for a key on the operator's side of the threshold and All for one on the other side, so the fold is the maximum when the operator is true above the threshold and the aggregation is Any, or true below it and the aggregation is All, and the minimum otherwise. IBooleanUnaryOperator gains the threshold form as optional members with defaults: HasThresholdForm, TrueBelowThreshold, ThresholdBits (the threshold as the bits of an unsigned integer of the element size) and Key per width. The operators that take it are IsNaN, IsFinite and IsRealNumber (the absolute bits against infinity), IsNormal and IsSubnormal (the absolute bits minus the smallest normal value, or minus one, against the width of the range), IsZero (the absolute bits, or the bits for the integers, below one) and IsNegative and IsPositive (the bits against the sign bit), the last three for the primitive integers as well as for float and double. AggregateAnyAll dispatches such an operator to AggregateAnyAllThreshold, generic over the key type (the unsigned integer of the element size) and over the fold, MaxOperator or MinOperator: choosing the fold by type rather than by a branch on its direction leaves a single use of each key in the loop, which the JIT then folds into the fold instruction's memory operand. The 64-bit unsigned minimum and maximum are single instructions only with AVX-512, so double and the 64-bit integers use the threshold form only when Avx512F.VL is supported and keep the operator's comparison otherwise. The per-width loops have the block shape of the generic paths; the final overlapping vector is compared against the threshold as well. Hot loops on AVX-512 for IsNaNAny: vandps with the load as its memory operand and vpmaxud per vector, vpcmpgtud and kortestw per block; for IsNegativeAny a single vpmaxud with a memory operand per vector, and on SSE/AVX2 the same with 128- and 256-bit registers. Ryzen 7 7800X3D, best of three runs, N = 4096 (L1-resident), GElem/s, before the two commits -> after, 512-bit path: IsNaNAny 39.8 -> 43.0, IsNaNAll 40.1 -> 57.5, IsFiniteAll 33.0 -> 43.2, IsFiniteAny 32.4 -> 42.6, IsNormalAll 28.3 -> 41.3, IsSubnormalAny 19.5 -> 28.5 (the regression noted in the block-accumulation change is gone), IsNegativeAny 38.9 -> 59.3, IsNegativeAny 36.8 -> 65.9, IsNegativeAny 17.9 -> 31.2, IsFiniteAll 15.6 -> 21.4, IsNormalAll 14.0 -> 20.3, IsZeroAll 40.2 -> 44.1, IsZeroAny 140.9 -> 172.4. N = 65536 (L2-resident): IsNaNAny 34.3 -> 36.4, IsFiniteAll 31.9 -> 38.5, IsNormalAll 27.7 -> 33.9, IsSubnormalAny 19.0 -> 27.8. AVX2 path (AVX-512 disabled), N = 4096: IsFiniteAny 24.2 -> 41.7, IsPositiveAll 23.4 -> 42.8, IsNormalAll 23.9 -> 40.1, IsZeroAll 29.9 -> 43.1, IsNegativeAny 41.0 -> 58.9; IsNaNAny and IsZeroAny, already two instructions per vector there, vary within +-10% between runs and sizes. 128-bit path: IsFiniteAny 13.1 -> 26.8, IsPositiveAll 13.2 -> 27.1, IsNegativeAny 22.8 -> 32.7. Tests: long-length boundary tests for the operators that gained the threshold form, placing the values whose bits lie next to the thresholds (-0, the smallest and largest subnormal, the smallest normal, the largest finite value, infinity and the NaN whose bits follow it, each with both signs) at every block, vector and ragged boundary; and a theory over all 17 Any/All pairs that checks long spans filled with each special value, and spans in which one special value is placed at every boundary of a span filled with another, against the scalar predicate. The Is* tests pass at 512, 256 and 128 bits, with AVX-512, AVX2 and AVX disabled, and with hardware intrinsics disabled. Co-Authored-By: Claude Opus 5 --- .../TensorPrimitives.IBooleanUnaryOperator.cs | 406 +++++++++++++++++- .../netcore/TensorPrimitives.IsFinite.cs | 8 + .../Tensors/netcore/TensorPrimitives.IsNaN.cs | 8 + .../netcore/TensorPrimitives.IsNegative.cs | 8 + .../netcore/TensorPrimitives.IsNormal.cs | 9 + .../netcore/TensorPrimitives.IsPositive.cs | 8 + .../netcore/TensorPrimitives.IsRealNumber.cs | 8 + .../netcore/TensorPrimitives.IsSubnormal.cs | 9 + .../netcore/TensorPrimitives.IsZero.cs | 8 + .../tests/TensorPrimitives.Generic.cs | 250 +++++++++++ 10 files changed, 721 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs index e59746c5f46fc0..f1a7e5c978dd82 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBooleanUnaryOperator.cs @@ -5,13 +5,19 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; namespace System.Numerics.Tensors { public static unsafe partial class TensorPrimitives { /// Unary operator that produces a Boolean result for each element. - /// For vector-based methods, the Boolean result is either all-bits-set or zero. + /// + /// For vector-based methods, the Boolean result is either all-bits-set or zero. + /// An operator may also have a threshold form (): its result is then a single unsigned comparison of a + /// key derived from the element's bits against a constant, which lets fold a + /// block of vectors with one unsigned minimum or maximum per vector and compare once per block. + /// private interface IBooleanUnaryOperator { static abstract bool Vectorizable { get; } @@ -19,6 +25,77 @@ private interface IBooleanUnaryOperator static abstract Vector128 Invoke(Vector128 x); static abstract Vector256 Invoke(Vector256 x); static abstract Vector512 Invoke(Vector512 x); + + /// + /// Whether, reading the bits of the key and as unsigned integers of the element size, Invoke(x) + /// is Key(x) < ThresholdBits when and Key(x) > ThresholdBits otherwise. + /// + static virtual bool HasThresholdForm => false; + + /// Whether the operator is true for keys below the threshold rather than for keys above it. + static virtual bool TrueBelowThreshold => throw new NotSupportedException(); + + /// The threshold of the threshold form, as the bits of an unsigned integer of the element size. + static virtual ulong ThresholdBits => throw new NotSupportedException(); + + /// The key of the threshold form: the element's bits, transformed so that the operator is a single comparison of them. + static virtual Vector128 Key(Vector128 x) => throw new NotSupportedException(); + /// + static virtual Vector256 Key(Vector256 x) => throw new NotSupportedException(); + /// + static virtual Vector512 Key(Vector512 x) => throw new NotSupportedException(); + } + + /// The bits of the positive infinity of , or . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong PositiveInfinityBits() + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? + BitConverter.SingleToUInt32Bits(float.PositiveInfinity) : + BitConverter.DoubleToUInt64Bits(double.PositiveInfinity); + } + + /// The bits of the smallest positive normal value of , or . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong SmallestNormalBits() + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? 0x0080_0000u : 0x0010_0000_0000_0000ul; + } + + /// The sign bit of , a primitive signed integer, or , as an unsigned integer of the element size. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong SignBit() => 1ul << ((sizeof(T) * 8) - 1); + + /// Subtracts from every element's bits, read as unsigned integers of the element size ( or ). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 SubtractBits(Vector128 x, ulong bits) + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? + (x.AsUInt32() - Vector128.Create((uint)bits)).As() : + (x.AsUInt64() - Vector128.Create(bits)).As(); + } + + /// Subtracts from every element's bits, read as unsigned integers of the element size ( or ). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 SubtractBits(Vector256 x, ulong bits) + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? + (x.AsUInt32() - Vector256.Create((uint)bits)).As() : + (x.AsUInt64() - Vector256.Create(bits)).As(); + } + + /// Subtracts from every element's bits, read as unsigned integers of the element size ( or ). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 SubtractBits(Vector512 x, ulong bits) + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + return typeof(T) == typeof(float) ? + (x.AsUInt32() - Vector512.Create((uint)bits)).As() : + (x.AsUInt64() - Vector512.Create(bits)).As(); } /// @@ -121,6 +198,16 @@ private static bool AggregateAnyAll(ReadOnlySpan x) { Debug.Assert(!x.IsEmpty); + if (TOperator.HasThresholdForm) + { + // The keys are folded as unsigned integers of the element size. Their 64-bit minimum and maximum are single instructions + // only with AVX-512; elsewhere the operator's own comparison is cheaper. + if (sizeof(T) == 1) return AggregateAnyAllThreshold(x); + if (sizeof(T) == 2) return AggregateAnyAllThreshold(x); + if (sizeof(T) == 4) return AggregateAnyAllThreshold(x); + if (sizeof(T) == 8 && Avx512F.VL.IsSupported) return AggregateAnyAllThreshold(x); + } + if (Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported && x.Length >= Vector512.Count) { return AggregateAnyAllVectorized512(x); @@ -381,6 +468,323 @@ private static bool AggregateAnyAllVectorized128(ReadOnly return TAnyAll.DefaultResult; } + /// + /// for an operator with a threshold form, whose keys are folded as + /// , the unsigned integer of the element size. + /// + private static bool AggregateAnyAllThreshold(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(TOperator.HasThresholdForm); + Debug.Assert(sizeof(TKey) == sizeof(T)); + Debug.Assert(!x.IsEmpty); + + // The fold is chosen by type rather than by a branch on the direction so that the loops contain a single use of each key, + // which the JIT then folds into the fold instruction's memory operand. + return ThresholdFoldsMax() ? + AggregateAnyAllThreshold, TOperator, TAnyAll>(x) : + AggregateAnyAllThreshold, TOperator, TAnyAll>(x); + } + + /// + /// Whether the threshold form of aggregated by folds the keys of a block + /// with their maximum rather than their minimum. Any looks for an element for which the operator holds and All for one for which it + /// does not: with the operator true below the threshold, those are a key below it (the minimum decides) and a key at or above it (the + /// maximum decides); with the operator true above the threshold, it is the other way round. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool ThresholdFoldsMax() + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator => + TOperator.TrueBelowThreshold == TAnyAll.DefaultResult; + + /// + /// with the fold of the keys, , chosen: + /// or over (see ). + /// + private static bool AggregateAnyAllThreshold(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TFold : struct, IBinaryOperator + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(typeof(TFold) == (ThresholdFoldsMax() ? typeof(MaxOperator) : typeof(MinOperator))); + + if (Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported && x.Length >= Vector512.Count) + { + return AggregateAnyAllThreshold512(x); + } + + if (Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported && x.Length >= Vector256.Count) + { + return AggregateAnyAllThreshold256(x); + } + + if (Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported && x.Length >= Vector128.Count) + { + return AggregateAnyAllThreshold128(x); + } + + return AggregateAnyAllScalar(x); + } + + /// The 512-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// The shape of , except that a block is folded with the unsigned + /// minimum or maximum of the elements' keys, one instruction per vector with no comparison, and the fold is compared against the + /// threshold once per block. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator inlined + private static bool AggregateAnyAllThreshold512(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TFold : struct, IBinaryOperator + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector512.IsHardwareAccelerated && TOperator.Vectorizable && Vector512.IsSupported); + Debug.Assert(x.Length >= Vector512.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector512.Count; + nuint i = 0; + + Vector512 identity = ThresholdFoldsMax() ? Vector512.Zero : Vector512.AllBitsSet; + Vector512 threshold = Vector512.Create(TKey.CreateTruncating(TOperator.ThresholdBits)); + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector512.Count); + if (length >= blockLength) + { + nuint oneBlockFromEnd = length - blockLength; + do + { + Vector512 accumulator0 = identity; + Vector512 accumulator1 = identity; + nuint blockEnd = i + blockLength; + do + { + accumulator0 = TFold.Invoke(accumulator0, TOperator.Key(Vector512.LoadUnsafe(ref xRef, i)).As()); + accumulator1 = TFold.Invoke(accumulator1, TOperator.Key(Vector512.LoadUnsafe(ref xRef, i + (uint)Vector512.Count)).As()); + i += (uint)(2 * Vector512.Count); + } + while (i < blockEnd); + + if (Settles(TFold.Invoke(accumulator0, accumulator1), threshold)) + { + return !TAnyAll.DefaultResult; + } + } + while (i <= oneBlockFromEnd); + } + + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector512 accumulator = identity; + do + { + accumulator = TFold.Invoke(accumulator, TOperator.Key(Vector512.LoadUnsafe(ref xRef, i)).As()); + i += (uint)Vector512.Count; + } + while (i <= oneVectorFromEnd); + + if (Settles(accumulator, threshold)) + { + return !TAnyAll.DefaultResult; + } + } + + // Handle any remaining elements with a final vector. + if (i != length && + Settles(TOperator.Key(Vector512.LoadUnsafe(ref xRef, oneVectorFromEnd)).As(), threshold)) + { + return !TAnyAll.DefaultResult; + } + + return TAnyAll.DefaultResult; + + // Whether the keys folded into the accumulator include one that settles the result: for Any a key on the operator's side of + // the threshold, for All a key on the other side (or on the threshold). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static bool Settles(Vector512 accumulator, Vector512 threshold) => + TAnyAll.DefaultResult ? + (TOperator.TrueBelowThreshold ? Vector512.GreaterThanOrEqualAny(accumulator, threshold) : Vector512.LessThanOrEqualAny(accumulator, threshold)) : + (TOperator.TrueBelowThreshold ? Vector512.LessThanAny(accumulator, threshold) : Vector512.GreaterThanAny(accumulator, threshold)); + } + + /// The 256-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// The shape of , except that a block is folded with the unsigned + /// minimum or maximum of the elements' keys, one instruction per vector with no comparison, and the fold is compared against the + /// threshold once per block. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator inlined + private static bool AggregateAnyAllThreshold256(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TFold : struct, IBinaryOperator + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector256.IsHardwareAccelerated && TOperator.Vectorizable && Vector256.IsSupported); + Debug.Assert(x.Length >= Vector256.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector256.Count; + nuint i = 0; + + Vector256 identity = ThresholdFoldsMax() ? Vector256.Zero : Vector256.AllBitsSet; + Vector256 threshold = Vector256.Create(TKey.CreateTruncating(TOperator.ThresholdBits)); + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector256.Count); + if (length >= blockLength) + { + nuint oneBlockFromEnd = length - blockLength; + do + { + Vector256 accumulator0 = identity; + Vector256 accumulator1 = identity; + nuint blockEnd = i + blockLength; + do + { + accumulator0 = TFold.Invoke(accumulator0, TOperator.Key(Vector256.LoadUnsafe(ref xRef, i)).As()); + accumulator1 = TFold.Invoke(accumulator1, TOperator.Key(Vector256.LoadUnsafe(ref xRef, i + (uint)Vector256.Count)).As()); + i += (uint)(2 * Vector256.Count); + } + while (i < blockEnd); + + if (Settles(TFold.Invoke(accumulator0, accumulator1), threshold)) + { + return !TAnyAll.DefaultResult; + } + } + while (i <= oneBlockFromEnd); + } + + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector256 accumulator = identity; + do + { + accumulator = TFold.Invoke(accumulator, TOperator.Key(Vector256.LoadUnsafe(ref xRef, i)).As()); + i += (uint)Vector256.Count; + } + while (i <= oneVectorFromEnd); + + if (Settles(accumulator, threshold)) + { + return !TAnyAll.DefaultResult; + } + } + + // Handle any remaining elements with a final vector. + if (i != length && + Settles(TOperator.Key(Vector256.LoadUnsafe(ref xRef, oneVectorFromEnd)).As(), threshold)) + { + return !TAnyAll.DefaultResult; + } + + return TAnyAll.DefaultResult; + + // Whether the keys folded into the accumulator include one that settles the result: for Any a key on the operator's side of + // the threshold, for All a key on the other side (or on the threshold). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static bool Settles(Vector256 accumulator, Vector256 threshold) => + TAnyAll.DefaultResult ? + (TOperator.TrueBelowThreshold ? Vector256.GreaterThanOrEqualAny(accumulator, threshold) : Vector256.LessThanOrEqualAny(accumulator, threshold)) : + (TOperator.TrueBelowThreshold ? Vector256.LessThanAny(accumulator, threshold) : Vector256.GreaterThanAny(accumulator, threshold)); + } + + /// The 128-bit path of : the whole vectors in blocks, then one final vector that overlaps the last whole one. + /// + /// The shape of , except that a block is folded with the unsigned + /// minimum or maximum of the elements' keys, one instruction per vector with no comparison, and the fold is compared against the + /// threshold once per block. + /// + [MethodImpl(MethodImplOptions.NoInlining)] // called once per aggregation; its own inlining budget keeps the operator inlined + private static bool AggregateAnyAllThreshold128(ReadOnlySpan x) + where TKey : unmanaged, IBinaryInteger + where TFold : struct, IBinaryOperator + where TOperator : struct, IBooleanUnaryOperator + where TAnyAll : struct, IAnyAllAggregator + { + Debug.Assert(Vector128.IsHardwareAccelerated && TOperator.Vectorizable && Vector128.IsSupported); + Debug.Assert(x.Length >= Vector128.Count); + + ref T xRef = ref MemoryMarshal.GetReference(x); + nuint length = (uint)x.Length; + nuint oneVectorFromEnd = length - (uint)Vector128.Count; + nuint i = 0; + + Vector128 identity = ThresholdFoldsMax() ? Vector128.Zero : Vector128.AllBitsSet; + Vector128 threshold = Vector128.Create(TKey.CreateTruncating(TOperator.ThresholdBits)); + + // Whole blocks: two accumulators, one decision per block. + nuint blockLength = (uint)(AnyAllBlockVectors * Vector128.Count); + if (length >= blockLength) + { + nuint oneBlockFromEnd = length - blockLength; + do + { + Vector128 accumulator0 = identity; + Vector128 accumulator1 = identity; + nuint blockEnd = i + blockLength; + do + { + accumulator0 = TFold.Invoke(accumulator0, TOperator.Key(Vector128.LoadUnsafe(ref xRef, i)).As()); + accumulator1 = TFold.Invoke(accumulator1, TOperator.Key(Vector128.LoadUnsafe(ref xRef, i + (uint)Vector128.Count)).As()); + i += (uint)(2 * Vector128.Count); + } + while (i < blockEnd); + + if (Settles(TFold.Invoke(accumulator0, accumulator1), threshold)) + { + return !TAnyAll.DefaultResult; + } + } + while (i <= oneBlockFromEnd); + } + + // The remaining whole vectors, fewer than a block. + if (i <= oneVectorFromEnd) + { + Vector128 accumulator = identity; + do + { + accumulator = TFold.Invoke(accumulator, TOperator.Key(Vector128.LoadUnsafe(ref xRef, i)).As()); + i += (uint)Vector128.Count; + } + while (i <= oneVectorFromEnd); + + if (Settles(accumulator, threshold)) + { + return !TAnyAll.DefaultResult; + } + } + + // Handle any remaining elements with a final vector. + if (i != length && + Settles(TOperator.Key(Vector128.LoadUnsafe(ref xRef, oneVectorFromEnd)).As(), threshold)) + { + return !TAnyAll.DefaultResult; + } + + return TAnyAll.DefaultResult; + + // Whether the keys folded into the accumulator include one that settles the result: for Any a key on the operator's side of + // the threshold, for All a key on the other side (or on the threshold). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static bool Settles(Vector128 accumulator, Vector128 threshold) => + TAnyAll.DefaultResult ? + (TOperator.TrueBelowThreshold ? Vector128.GreaterThanOrEqualAny(accumulator, threshold) : Vector128.LessThanOrEqualAny(accumulator, threshold)) : + (TOperator.TrueBelowThreshold ? Vector128.LessThanAny(accumulator, threshold) : Vector128.GreaterThanAny(accumulator, threshold)); + } + /// Performs an element-wise operation on and writes the results to . /// The element input type. /// Specifies the operation to perform on each element loaded from . diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsFinite.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsFinite.cs index 3c1e22bfb16097..52a374d6256bc2 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsFinite.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsFinite.cs @@ -70,6 +70,14 @@ private static bool AlwaysFinite() => public static Vector128 Invoke(Vector128 x) => Vector128.IsFinite(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsFinite(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsFinite(x); + + // A finite value is one whose absolute bit pattern lies below that of infinity. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => PositiveInfinityBits(); + public static Vector128 Key(Vector128 x) => Vector128.Abs(x); + public static Vector256 Key(Vector256 x) => Vector256.Abs(x); + public static Vector512 Key(Vector512 x) => Vector512.Abs(x); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNaN.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNaN.cs index 7e38263b25a347..2c5217ad4c69b1 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNaN.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNaN.cs @@ -68,6 +68,14 @@ private static bool MayBeNaN() => public static Vector128 Invoke(Vector128 x) => Vector128.IsNaN(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsNaN(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsNaN(x); + + // A NaN is a value whose absolute bit pattern lies above that of infinity. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => false; + public static ulong ThresholdBits => PositiveInfinityBits(); + public static Vector128 Key(Vector128 x) => Vector128.Abs(x); + public static Vector256 Key(Vector256 x) => Vector256.Abs(x); + public static Vector512 Key(Vector512 x) => Vector512.Abs(x); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNegative.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNegative.cs index 3ed5de79558d1a..fb4be418108b6e 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNegative.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNegative.cs @@ -72,6 +72,14 @@ private static bool MayBeNegative() => public static Vector128 Invoke(Vector128 x) => Vector128.IsNegative(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsNegative(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsNegative(x); + + // A negative value is one whose sign bit is set: whose bits lie above the values with the sign bit clear. + public static bool HasThresholdForm => MayBeNegative() && (IsPrimitiveBinaryInteger() || typeof(T) == typeof(float) || typeof(T) == typeof(double)); + public static bool TrueBelowThreshold => false; + public static ulong ThresholdBits => SignBit() - 1; + public static Vector128 Key(Vector128 x) => x; + public static Vector256 Key(Vector256 x) => x; + public static Vector512 Key(Vector512 x) => x; } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNormal.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNormal.cs index 1fc78f2bf87de4..be510e8491ebc2 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNormal.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsNormal.cs @@ -54,6 +54,15 @@ public static bool IsNormalAny(ReadOnlySpan x) public static Vector256 Invoke(Vector256 x) => Vector256.IsNormal(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsNormal(x); + + // A normal value is one whose absolute bit pattern lies between those of the smallest normal value (inclusive) and infinity (exclusive), + // the same range test as the operator's, which the subtraction turns into a single comparison: a smaller pattern wraps around. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => PositiveInfinityBits() - SmallestNormalBits(); + public static Vector128 Key(Vector128 x) => SubtractBits(Vector128.Abs(x), SmallestNormalBits()); + public static Vector256 Key(Vector256 x) => SubtractBits(Vector256.Abs(x), SmallestNormalBits()); + public static Vector512 Key(Vector512 x) => SubtractBits(Vector512.Abs(x), SmallestNormalBits()); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsPositive.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsPositive.cs index 187b57c1b37bc0..f03fd2b1dc0c8d 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsPositive.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsPositive.cs @@ -50,6 +50,14 @@ public static bool IsPositiveAny(ReadOnlySpan x) public static Vector128 Invoke(Vector128 x) => Vector128.IsPositive(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsPositive(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsPositive(x); + + // A positive value is one whose sign bit is clear: whose bits lie below the sign bit. + public static bool HasThresholdForm => MayBeNegative() && (IsPrimitiveBinaryInteger() || typeof(T) == typeof(float) || typeof(T) == typeof(double)); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => SignBit(); + public static Vector128 Key(Vector128 x) => x; + public static Vector256 Key(Vector256 x) => x; + public static Vector512 Key(Vector512 x) => x; } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsRealNumber.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsRealNumber.cs index daadd47a83ab3b..71d29578e05642 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsRealNumber.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsRealNumber.cs @@ -68,6 +68,14 @@ private static bool AlwaysReal() => public static Vector128 Invoke(Vector128 x) => ~Vector128.IsNaN(x); public static Vector256 Invoke(Vector256 x) => ~Vector256.IsNaN(x); public static Vector512 Invoke(Vector512 x) => ~Vector512.IsNaN(x); + + // A real number is any value but a NaN: one whose absolute bit pattern lies below that of the first NaN, the one after infinity. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => PositiveInfinityBits() + 1; + public static Vector128 Key(Vector128 x) => Vector128.Abs(x); + public static Vector256 Key(Vector256 x) => Vector256.Abs(x); + public static Vector512 Key(Vector512 x) => Vector512.Abs(x); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsSubnormal.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsSubnormal.cs index f821ac491ef922..0a799dc971953d 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsSubnormal.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsSubnormal.cs @@ -57,6 +57,15 @@ private static bool MayBeSubnormal() => public static Vector128 Invoke(Vector128 x) => Vector128.IsSubnormal(x); public static Vector256 Invoke(Vector256 x) => Vector256.IsSubnormal(x); public static Vector512 Invoke(Vector512 x) => Vector512.IsSubnormal(x); + + // A subnormal value is one whose absolute bit pattern lies between one (inclusive) and that of the smallest normal value (exclusive), + // the same range test as the operator's, which the subtraction turns into a single comparison: zero wraps around. + public static bool HasThresholdForm => typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => SmallestNormalBits() - 1; + public static Vector128 Key(Vector128 x) => SubtractBits(Vector128.Abs(x), 1); + public static Vector256 Key(Vector256 x) => SubtractBits(Vector256.Abs(x), 1); + public static Vector512 Key(Vector512 x) => SubtractBits(Vector512.Abs(x), 1); } } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsZero.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsZero.cs index eaca0c55dc73cf..00e7feb17fb0b9 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsZero.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IsZero.cs @@ -50,6 +50,14 @@ public static bool IsZeroAny(ReadOnlySpan x) public static Vector128 Invoke(Vector128 x) => Vector128.Equals(x, Vector128.Zero); public static Vector256 Invoke(Vector256 x) => Vector256.Equals(x, Vector256.Zero); public static Vector512 Invoke(Vector512 x) => Vector512.Equals(x, Vector512.Zero); + + // Zero is the value whose bits, for the floating-point types apart from the sign, are below one. + public static bool HasThresholdForm => IsPrimitiveBinaryInteger() || typeof(T) == typeof(float) || typeof(T) == typeof(double); + public static bool TrueBelowThreshold => true; + public static ulong ThresholdBits => 1; + public static Vector128 Key(Vector128 x) => typeof(T) == typeof(float) || typeof(T) == typeof(double) ? Vector128.Abs(x) : x; + public static Vector256 Key(Vector256 x) => typeof(T) == typeof(float) || typeof(T) == typeof(double) ? Vector256.Abs(x) : x; + public static Vector512 Key(Vector512 x) => typeof(T) == typeof(float) || typeof(T) == typeof(double) ? Vector512.Abs(x) : x; } } } diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs index f4f68ce4b4912e..97cdb73056deda 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs @@ -3406,6 +3406,256 @@ public void IsZeroAll_LongLengths() => [Fact] public void IsZeroAny_LongLengths() => AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAny, fill: ConvertFromSingle(1), hit: Zero, expectedWithoutHit: false); + + // The vectorized Any/All paths of the classifications below decide from the bits of the elements (an unsigned minimum or maximum + // of the block's bit patterns compared against a threshold) rather than from the operator, so these place the values whose + // bits lie next to the thresholds: the sign bit alone (-0), the smallest subnormal, the largest subnormal, the smallest normal, + // the largest finite value, infinity and the NaN whose bits follow it. + [Fact] + public void IsNegativeAll_LongLengths() + { + if (!HasNegativeValues) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAll, fill: ConvertFromSingle(-1), hit: ConvertFromSingle(1), expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAll, fill: MinValue, hit: Zero, expectedWithoutHit: true); + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAll, fill: NegativeZero, hit: Zero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAll, fill: -FirstNaN, hit: FirstNaN, expectedWithoutHit: true); + } + + [Fact] + public void IsNegativeAny_LongLengths_SignBitOnly() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAny, fill: Zero, hit: NegativeZero, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNegativeAny, fill: FirstNaN, hit: -FirstNaN, expectedWithoutHit: false); + } + + [Fact] + public void IsPositiveAll_LongLengths() + { + if (!HasNegativeValues) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAll, fill: ConvertFromSingle(1), hit: ConvertFromSingle(-1), expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAll, fill: Zero, hit: MinValue, expectedWithoutHit: true); + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAll, fill: Zero, hit: NegativeZero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAll, fill: FirstNaN, hit: -FirstNaN, expectedWithoutHit: true); + } + + [Fact] + public void IsPositiveAny_LongLengths_SignBitOnly() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAny, fill: NegativeZero, hit: Zero, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsPositiveAny, fill: -FirstNaN, hit: FirstNaN, expectedWithoutHit: false); + } + + [Fact] + public void IsZeroAll_LongLengths_SignBitAndEpsilon() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAll, fill: NegativeZero, hit: SmallestSubnormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAll, fill: Zero, hit: -SmallestSubnormal, expectedWithoutHit: true); + } + + [Fact] + public void IsZeroAny_LongLengths_SignBitAndEpsilon() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAny, fill: SmallestSubnormal, hit: NegativeZero, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsZeroAny, fill: NaN, hit: Zero, expectedWithoutHit: false); + } + + [Fact] + public void IsNaNAny_LongLengths_Boundaries() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAny, fill: PositiveInfinity, hit: FirstNaN, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAny, fill: -PositiveInfinity, hit: -FirstNaN, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAny, fill: T.MaxValue, hit: -NaN, expectedWithoutHit: false); + } + + [Fact] + public void IsNaNAll_LongLengths_Boundaries() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAll, fill: FirstNaN, hit: PositiveInfinity, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNaNAll, fill: -NaN, hit: -PositiveInfinity, expectedWithoutHit: true); + } + + [Fact] + public void IsFiniteAll_LongLengths_Boundaries() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: T.MaxValue, hit: -PositiveInfinity, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: MinValue, hit: FirstNaN, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAll, fill: SmallestSubnormal, hit: -NaN, expectedWithoutHit: true); + } + + [Fact] + public void IsFiniteAny_LongLengths_Boundaries() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAny, fill: NaN, hit: T.MaxValue, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAny, fill: PositiveInfinity, hit: MinValue, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsFiniteAny, fill: FirstNaN, hit: NegativeZero, expectedWithoutHit: false); + } + + [Fact] + public void IsRealNumberAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAll, fill: ConvertFromSingle(1), hit: FirstNaN, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAll, fill: PositiveInfinity, hit: NaN, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAll, fill: -PositiveInfinity, hit: -NaN, expectedWithoutHit: true); + } + + [Fact] + public void IsRealNumberAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAny, fill: NaN, hit: PositiveInfinity, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAny, fill: FirstNaN, hit: -PositiveInfinity, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsRealNumberAny, fill: -NaN, hit: ConvertFromSingle(1), expectedWithoutHit: false); + } + + [Fact] + public void IsNormalAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: ConvertFromSingle(1), hit: Zero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: SmallestNormal, hit: LargestSubnormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: -SmallestNormal, hit: -LargestSubnormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: T.MaxValue, hit: PositiveInfinity, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: MinValue, hit: NaN, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAll, fill: ConvertFromSingle(-1), hit: NegativeZero, expectedWithoutHit: true); + } + + [Fact] + public void IsNormalAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: Zero, hit: ConvertFromSingle(1), expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: LargestSubnormal, hit: SmallestNormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: -LargestSubnormal, hit: -SmallestNormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: PositiveInfinity, hit: T.MaxValue, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: NaN, hit: MinValue, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsNormalAny, fill: NegativeZero, hit: ConvertFromSingle(-1), expectedWithoutHit: false); + } + + [Fact] + public void IsSubnormalAll_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: SmallestSubnormal, hit: Zero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: -SmallestSubnormal, hit: NegativeZero, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: LargestSubnormal, hit: SmallestNormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: -LargestSubnormal, hit: -SmallestNormal, expectedWithoutHit: true); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAll, fill: SmallestSubnormal, hit: NaN, expectedWithoutHit: true); + } + + [Fact] + public void IsSubnormalAny_LongLengths() + { + if (!IsFloatingPoint) return; + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: Zero, hit: SmallestSubnormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: NegativeZero, hit: -SmallestSubnormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: SmallestNormal, hit: LargestSubnormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: -SmallestNormal, hit: -LargestSubnormal, expectedWithoutHit: false); + AssertIsAnyAllLongLengths(TensorPrimitives.IsSubnormalAny, fill: NaN, hit: SmallestSubnormal, expectedWithoutHit: false); + } + + /// The value of a floating-point with the given bits. + private static T FromBits(ulong bits) => + Unsafe.SizeOf() == 2 ? Unsafe.BitCast((ushort)bits) : + Unsafe.SizeOf() == 4 ? Unsafe.BitCast((uint)bits) : + Unsafe.BitCast(bits); + + private static (ulong PositiveInfinity, ulong SmallestNormal) FloatingPointBits => + Unsafe.SizeOf() == 2 ? (0x7C00ul, 0x0400ul) : + Unsafe.SizeOf() == 4 ? (0x7F80_0000ul, 0x0080_0000ul) : + (0x7FF0_0000_0000_0000ul, 0x0010_0000_0000_0000ul); + + private static T PositiveInfinity => FromBits(FloatingPointBits.PositiveInfinity); + /// The NaN whose bits follow those of positive infinity. + private static T FirstNaN => FromBits(FloatingPointBits.PositiveInfinity + 1); + private static T SmallestNormal => FromBits(FloatingPointBits.SmallestNormal); + private static T LargestSubnormal => FromBits(FloatingPointBits.SmallestNormal - 1); + private static T SmallestSubnormal => FromBits(1); + + public static IEnumerable IsAnyAllFunctionsToTest() + { + yield return Create(TensorPrimitives.IsCanonicalAny, TensorPrimitives.IsCanonicalAll, T.IsCanonical); + yield return Create(TensorPrimitives.IsComplexNumberAny, TensorPrimitives.IsComplexNumberAll, T.IsComplexNumber); + yield return Create(TensorPrimitives.IsEvenIntegerAny, TensorPrimitives.IsEvenIntegerAll, T.IsEvenInteger); + yield return Create(TensorPrimitives.IsFiniteAny, TensorPrimitives.IsFiniteAll, T.IsFinite); + yield return Create(TensorPrimitives.IsImaginaryNumberAny, TensorPrimitives.IsImaginaryNumberAll, T.IsImaginaryNumber); + yield return Create(TensorPrimitives.IsInfinityAny, TensorPrimitives.IsInfinityAll, T.IsInfinity); + yield return Create(TensorPrimitives.IsIntegerAny, TensorPrimitives.IsIntegerAll, T.IsInteger); + yield return Create(TensorPrimitives.IsNaNAny, TensorPrimitives.IsNaNAll, T.IsNaN); + yield return Create(TensorPrimitives.IsNegativeAny, TensorPrimitives.IsNegativeAll, T.IsNegative); + yield return Create(TensorPrimitives.IsNegativeInfinityAny, TensorPrimitives.IsNegativeInfinityAll, T.IsNegativeInfinity); + yield return Create(TensorPrimitives.IsNormalAny, TensorPrimitives.IsNormalAll, T.IsNormal); + yield return Create(TensorPrimitives.IsOddIntegerAny, TensorPrimitives.IsOddIntegerAll, T.IsOddInteger); + yield return Create(TensorPrimitives.IsPositiveAny, TensorPrimitives.IsPositiveAll, T.IsPositive); + yield return Create(TensorPrimitives.IsPositiveInfinityAny, TensorPrimitives.IsPositiveInfinityAll, T.IsPositiveInfinity); + yield return Create(TensorPrimitives.IsRealNumberAny, TensorPrimitives.IsRealNumberAll, T.IsRealNumber); + yield return Create(TensorPrimitives.IsSubnormalAny, TensorPrimitives.IsSubnormalAll, T.IsSubnormal); + yield return Create(TensorPrimitives.IsZeroAny, TensorPrimitives.IsZeroAll, T.IsZero); + + static object[] Create(SpanIsAllAnyDelegate anyMethod, SpanIsAllAnyDelegate allMethod, Func predicate) + => new object[] { anyMethod, allMethod, predicate }; + } + + /// + /// Checks Any and All against the scalar predicate for long spans filled with each special value, and for a span filled with one + /// special value in which another one is placed at every block, vector and ragged boundary. + /// + [Theory] + [MemberData(nameof(IsAnyAllFunctionsToTest))] + public void IsAnyAll_SpecialValues(SpanIsAllAnyDelegate anyMethod, SpanIsAllAnyDelegate allMethod, Func predicate) + { + List values = [Zero, One, NegativeOne, MinValue, T.MaxValue, ConvertFromSingle(2), ConvertFromSingle(3), .. GetSpecialValues()]; + if (IsFloatingPoint) + { + values.AddRange([NegativeZero, FirstNaN, -FirstNaN, SmallestNormal, -SmallestNormal, LargestSubnormal, -LargestSubnormal, SmallestSubnormal, -SmallestSubnormal]); + } + + Assert.All(s_minMaxLongLengths, tensorLength => + { + using BoundedMemory x = CreateTensor(tensorLength); + foreach (T value in values) + { + x.Span.Fill(value); + Assert.Equal(predicate(value), anyMethod(x)); + Assert.Equal(predicate(value), allMethod(x)); + } + }); + + // Two lengths that cover several blocks of every vector width, with a ragged end. + Assert.All(new[] { 1025, 4097 }, tensorLength => + { + using BoundedMemory x = CreateTensor(tensorLength); + foreach (T fill in values) + { + foreach (T hit in values) + { + if (predicate(hit) == predicate(fill)) + { + continue; + } + + x.Span.Fill(fill); + foreach (int position in IsAnyAllLongPositions(tensorLength)) + { + x[position] = hit; + Assert.True(anyMethod(x)); + Assert.False(allMethod(x)); + x[position] = fill; + } + } + } + }); + } #endregion #region HammingDistance