diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IIndexOfMinMaxOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IIndexOfMinMaxOperator.cs
index 12d290ab014f04..c1424050bb62a9 100644
--- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IIndexOfMinMaxOperator.cs
+++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IIndexOfMinMaxOperator.cs
@@ -1,25 +1,59 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
using System.Diagnostics;
using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
namespace System.Numerics.Tensors
{
public static unsafe partial class TensorPrimitives
{
+ /// The operations an IndexOfMin/Max-style search needs from its ordering.
+ ///
+ /// returns whether x should replace y: strictly better, or an equal value the
+ /// operator orders by sign (-0 before +0 and the like). and
+ /// must return an element that ranks no worse than any input, and for floating-point types must
+ /// propagate NaN, so that a block containing a NaN reduces to NaN.
+ ///
private interface IIndexOfMinMaxOperator
{
static abstract T Aggregate(Vector128 value);
static abstract T Aggregate(Vector256 value);
static abstract T Aggregate(Vector512 value);
+ static abstract T Reduce(T x, T y);
+ static abstract Vector128 Reduce(Vector128 x, Vector128 y);
+ static abstract Vector256 Reduce(Vector256 x, Vector256 y);
+ static abstract Vector512 Reduce(Vector512 x, Vector512 y);
static abstract bool Compare(T x, T y);
static abstract Vector128 Compare(Vector128 x, Vector128 y);
static abstract Vector256 Compare(Vector256 x, Vector256 y);
static abstract Vector512 Compare(Vector512 x, Vector512 y);
}
+ /// Number of vectors per block in the block-reduction search (256 ints per block at 256 bits).
+ private const int BlockVectors = 32;
+
+ ///
+ /// Finds the index of the best element of under , or -1 for an empty span,
+ /// with a two-pass block reduction: pass 1 reduces every block to its best element with a pure vector loop (one load and one
+ /// per vector, no index tracking and no blends) and
+ /// remembers the first block whose best beats the running result; pass 2 scans only that block for the first element the result
+ /// does not beat.
+ ///
+ ///
+ ///
+ /// The reduction is what makes this run at memory bandwidth regardless of the input pattern; an index-vector loop needs a compare
+ /// and two selects per element. Indices never live in vector lanes, so element sizes need no special handling.
+ ///
+ ///
+ /// Ties: within a block, the reduction picks some tied-best element and the scan then returns the earliest element it does not beat,
+ /// which is the earliest tied-best element; across blocks the strict keeps the
+ /// earliest block. For floating-point types a block containing a NaN reduces to NaN (the reductions propagate it), and the index of
+ /// the first NaN of that block is returned, which is the first NaN overall because earlier blocks contained none.
+ ///
+ ///
private static int IndexOfMinMaxCore(ReadOnlySpan x)
where T : INumber where TOperator : struct, IIndexOfMinMaxOperator
{
@@ -30,26 +64,17 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x)
if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count)
{
- return sizeof(T) == 8 ? IndexOfMinMaxVectorized512Size4Plus(x) :
- sizeof(T) == 4 ? IndexOfMinMaxVectorized512Size4Plus(x) :
- sizeof(T) == 2 ? IndexOfMinMaxVectorized512Size2(x) :
- IndexOfMinMaxVectorized512Size1(x);
+ return IndexOfMinMaxBlocks512(x);
}
if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count)
{
- return sizeof(T) == 8 ? IndexOfMinMaxVectorized256Size4Plus(x) :
- sizeof(T) == 4 ? IndexOfMinMaxVectorized256Size4Plus(x) :
- sizeof(T) == 2 ? IndexOfMinMaxVectorized256Size2(x) :
- IndexOfMinMaxVectorized256Size1(x);
+ return IndexOfMinMaxBlocks256(x);
}
if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count)
{
- return sizeof(T) == 8 ? IndexOfMinMaxVectorized128Size4Plus(x) :
- sizeof(T) == 4 ? IndexOfMinMaxVectorized128Size4Plus(x) :
- sizeof(T) == 2 ? IndexOfMinMaxVectorized128Size2(x) :
- IndexOfMinMaxVectorized128Size1(x);
+ return IndexOfMinMaxBlocks128(x);
}
return IndexOfMinMaxFallback(x);
@@ -82,697 +107,499 @@ private static int IndexOfMinMaxFallback(ReadOnlySpan x)
return resultIndex;
}
- private static int IndexOfMinMaxVectorized128Size4Plus(ReadOnlySpan x)
- where T : INumber where TOperator : struct, IIndexOfMinMaxOperator where TInt : IBinaryInteger
+ ///
+ /// Debug-only contract check for : no element among the first
+ /// elements at beats under , and none is NaN
+ /// (a NaN would have had to propagate into ).
+ ///
+ private static bool NoElementBeats(ref T xRef, int length, T value)
+ where T : INumber where TOperator : struct, IIndexOfMinMaxOperator
{
- Debug.Assert(sizeof(T) == 4 || sizeof(T) == 8);
- Debug.Assert(typeof(TInt) == typeof(uint) || typeof(TInt) == typeof(ulong));
- Debug.Assert(sizeof(TInt) == sizeof(T));
-
- // Initialize result by reading first vector and quick return if possible.
- Vector128 result = Vector128.Create(x);
- if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
+ for (int i = 0; i < length; i++)
{
- Vector128 nanMask = Vector128.IsNaN(result);
- if (nanMask != Vector128.Zero)
+ T element = Unsafe.Add(ref xRef, i);
+ if (T.IsNaN(element) || TOperator.Compare(element, value))
{
- return IndexOfFirstMatch(nanMask);
+ return false;
}
}
- // Initialize indices.
- Vector128 indexIncrement = Vector128.Create(TInt.CreateTruncating(Vector128.Count));
- Vector128 resultIndex = Vector128.Indices;
- Vector128 currentIndex = resultIndex + indexIncrement;
- ReadOnlySpan span = x.Slice(Vector128.Count);
+ return true;
+ }
- while (!span.IsEmpty)
+ /// See .
+ private static int IndexOfMinMaxBlocks128(ReadOnlySpan x)
+ where T : INumber where TOperator : struct, IIndexOfMinMaxOperator
+ {
+ Debug.Assert(Vector128.IsHardwareAccelerated && Vector128.IsSupported);
+ Debug.Assert(x.Length >= Vector128.Count);
+ Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8);
+
+ int blockSize = BlockVectors * Vector128.Count;
+ int length = x.Length;
+ ref T xRef = ref MemoryMarshal.GetReference(x);
+
+ // Pass 1: reduce every block to its best element; the first block whose best beats the running result wins ties.
+ T result = xRef;
+ int resultBlock = -1;
+ for (int i = 0; i < length; i += blockSize)
{
- Vector128 current;
- if (span.Length >= Vector128.Count)
- {
- current = Vector128.Create(span);
- span = span.Slice(Vector128.Count);
- }
- else
- {
- // Process a final back-shifted to cover remaining elements in x in one vector.
- int start = x.Length - Vector128.Count;
- current = Vector128.Create(x.Slice(start));
- currentIndex = Vector128.Create(TInt.CreateTruncating(start)) + Vector128.Indices;
- span = ReadOnlySpan.Empty;
- }
+ int blockLength = Math.Min(blockSize, length - i);
+ T blockResult = BlockReduce128(ref Unsafe.Add(ref xRef, i), blockLength);
- // Quick return if possible.
if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
{
- Vector128 nanMask = Vector128.IsNaN(current);
- if (nanMask != Vector128.Zero)
+ if (T.IsNaN(blockResult))
{
- return int.CreateTruncating(currentIndex.ToScalar()) + IndexOfFirstMatch(nanMask);
+ return i + IndexOfFirstNaN128(ref Unsafe.Add(ref xRef, i), blockLength);
}
}
- // Get mask for which lanes that should have result updated.
- Vector128 mask = TOperator.Compare(current, result);
+ Debug.Assert(NoElementBeats(ref Unsafe.Add(ref xRef, i), blockLength, blockResult),
+ "Reduce/Aggregate must return an element no other element of the block beats under Compare, and must propagate NaN.");
- // Update result and indices.
- result = ElementWiseSelect(mask, current, result);
- resultIndex = ElementWiseSelect(mask.As(), currentIndex, resultIndex);
- currentIndex += indexIncrement;
+ if (resultBlock < 0 || TOperator.Compare(blockResult, result))
+ {
+ result = blockResult;
+ resultBlock = i;
+ }
}
- {
- // Where result does not bitwise-equal the aggregate min/max value; replace indices with uint.MaxValue. Then find the min index.
- T aggResult = TOperator.Aggregate(result);
- Vector128 aggMask = ~Vector128.Equals(result.As(), Vector128.Create(aggResult).As());
- Vector128 aggIndex = resultIndex | aggMask;
- return int.CreateTruncating(HorizontalAggregate>(aggIndex));
- }
+ Debug.Assert(resultBlock >= 0);
+
+ // Pass 2: the first element of the winning block that the result does not beat, i.e. the first tied-best element.
+ return resultBlock + IndexOfFirstNotBeaten128(ref Unsafe.Add(ref xRef, resultBlock), Math.Min(blockSize, length - resultBlock), result);
}
- private static int IndexOfMinMaxVectorized128Size2(ReadOnlySpan x)
+ /// Reduces elements starting at to their best element (no bounds checks).
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static T BlockReduce128(ref T xRef, int length)
where T : INumber where TOperator : struct, IIndexOfMinMaxOperator
{
- Debug.Assert(sizeof(T) == 2);
+ Debug.Assert(length >= 1);
- // Initialize result by reading first vector and quick return if possible.
- Vector128 result = Vector128.Create(x);
- if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
- {
- Vector128 nanMask = Vector128.IsNaN(result);
- if (nanMask != Vector128.Zero)
- {
- return IndexOfFirstMatch(nanMask);
- }
- }
+ nuint count = (nuint)Vector128.Count;
+ nuint end = (nuint)length;
+ T result;
+ nuint i;
- // Initialize indices.
- Vector128 indexIncrement = Vector128.Create((uint)Vector128.Count);
- Vector128 resultIndex1 = Vector128.Indices;
- Vector128 resultIndex2 = resultIndex1 + indexIncrement;
- Vector128 currentIndex = resultIndex2 + indexIncrement;
- ReadOnlySpan span = x.Slice(Vector128.Count);
-
- while (!span.IsEmpty)
+ if (end >= 2 * count)
{
- Vector128 current;
- if (span.Length >= Vector128.Count)
- {
- current = Vector128.Create(span);
- span = span.Slice(Vector128.Count);
- }
- else
+ // Two independent accumulators so that consecutive reductions do not serialize on one register.
+ Vector128 acc1 = Vector128.LoadUnsafe(ref xRef);
+ Vector128 acc2 = Vector128.LoadUnsafe(ref xRef, count);
+ nuint last = end - 2 * count;
+ for (i = 2 * count; i <= last; i += 2 * count)
{
- // Process a final back-shifted to cover remaining elements in x in one vector.
- int start = x.Length - Vector128.Count;
- current = Vector128.Create(x.Slice(start));
- currentIndex = Vector128.Create((uint)start) + Vector128.Indices;
- span = ReadOnlySpan.Empty;
+ acc1 = TOperator.Reduce(acc1, Vector128.LoadUnsafe(ref xRef, i));
+ acc2 = TOperator.Reduce(acc2, Vector128.LoadUnsafe(ref xRef, i + count));
}
- // Quick return if possible.
- if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
+ if (i + count <= end)
{
- Vector128 nanMask = Vector128.IsNaN(current);
- if (nanMask != Vector128.Zero)
- {
- return (int)currentIndex.ToScalar() + IndexOfFirstMatch(nanMask);
- }
+ acc1 = TOperator.Reduce(acc1, Vector128.LoadUnsafe(ref xRef, i));
+ i += count;
}
- // Get mask for which lanes that should have result updated, also widen it for updating the indices.
- Vector128 mask = TOperator.Compare(current, result);
- (Vector128 mask1, Vector128 mask2) = Vector128.Widen(mask.AsInt16());
-
- // Update result and indices.
- result = ElementWiseSelect(mask, current, result);
- resultIndex1 = ElementWiseSelect(mask1.AsUInt32(), currentIndex, resultIndex1);
- currentIndex += indexIncrement;
- resultIndex2 = ElementWiseSelect(mask2.AsUInt32(), currentIndex, resultIndex2);
- currentIndex += indexIncrement;
+ result = TOperator.Aggregate(TOperator.Reduce(acc1, acc2));
}
-
+ else if (end >= count)
{
- // Where result does not bitwise-equal the aggregate min/max value; replace indices with uint.MaxValue. Then find the min index.
- T aggResult = TOperator.Aggregate(result);
- Vector128 aggMask = ~Vector128.Equals(result.AsInt16(), Vector128.Create(aggResult).AsInt16());
-
- (Vector128 mask1, Vector128 mask2) = Vector128.Widen(aggMask);
- Vector128 aggIndex = resultIndex1 | mask1.AsUInt32();
- aggIndex = MinOperator.Invoke(aggIndex, resultIndex2 | mask2.AsUInt32());
+ result = TOperator.Aggregate(Vector128.LoadUnsafe(ref xRef));
+ i = count;
+ }
+ else
+ {
+ result = xRef;
+ i = 1;
+ }
- return (int)HorizontalAggregate>(aggIndex);
+ for (; i < end; i++)
+ {
+ result = TOperator.Reduce(result, Unsafe.Add(ref xRef, i));
}
+
+ return result;
}
- private static int IndexOfMinMaxVectorized128Size1(ReadOnlySpan x)
- where T : INumber where TOperator : struct, IIndexOfMinMaxOperator
+ /// Index of the first NaN among elements starting at ; there must be one.
+ [MethodImpl(MethodImplOptions.NoInlining)] // cold: called at most once per search; keeps the caller within the inlining budget
+ private static int IndexOfFirstNaN128(ref T xRef, int length)
+ where T : INumber
{
- Debug.Assert(sizeof(T) == 1);
+ int count = Vector128.Count;
+ int i = 0;
- // Initialize result by reading first vector and quick return if possible.
- Vector128 result = Vector128.Create(x);
- if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
+ for (; i + count <= length; i += count)
{
- Vector128 nanMask = Vector128.IsNaN(result);
+ Vector128 nanMask = Vector128.IsNaN(Vector128.LoadUnsafe(ref xRef, (nuint)i));
if (nanMask != Vector128.Zero)
{
- return IndexOfFirstMatch(nanMask);
+ return i + IndexOfFirstMatch(nanMask);
}
}
- // Initialize indices.
- Vector128 indexIncrement = Vector128.Create((uint)Vector128.Count);
- Vector128 resultIndex1 = Vector128.Indices;
- Vector128 resultIndex2 = resultIndex1 + indexIncrement;
- Vector128 resultIndex3 = resultIndex2 + indexIncrement;
- Vector128 resultIndex4 = resultIndex3 + indexIncrement;
- Vector128 currentIndex = resultIndex4 + indexIncrement;
- ReadOnlySpan span = x.Slice(Vector128.Count);
-
- while (!span.IsEmpty)
+ for (; i < length; i++)
{
- Vector128 current;
- if (span.Length >= Vector128.Count)
- {
- current = Vector128.Create(span);
- span = span.Slice(Vector128.Count);
- }
- else
- {
- // Process a final back-shifted to cover remaining elements in x in one vector.
- int start = x.Length - Vector128.Count;
- current = Vector128.Create(x.Slice(start));
- currentIndex = Vector128.Create((uint)start) + Vector128.Indices;
- span = ReadOnlySpan.Empty;
- }
-
- // Quick return if possible.
- if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
+ if (T.IsNaN(Unsafe.Add(ref xRef, i)))
{
- Vector128 nanMask = Vector128.IsNaN(current);
- if (nanMask != Vector128.Zero)
- {
- return (int)currentIndex.ToScalar() + IndexOfFirstMatch(nanMask);
- }
+ return i;
}
-
- // Get mask for which lanes that should have result updated, also widen it for updating the indices.
- Vector128 mask = TOperator.Compare(current, result);
- (Vector128 lowerMask, Vector128 upperMask) = Vector128.Widen(mask.AsSByte());
- (Vector128 mask1, Vector128 mask2) = Vector128.Widen(lowerMask);
- (Vector128 mask3, Vector128 mask4) = Vector128.Widen(upperMask);
-
- // Update result and indices.
- result = ElementWiseSelect(mask, current, result);
- resultIndex1 = ElementWiseSelect(mask1.AsUInt32(), currentIndex, resultIndex1);
- currentIndex += indexIncrement;
- resultIndex2 = ElementWiseSelect(mask2.AsUInt32(), currentIndex, resultIndex2);
- currentIndex += indexIncrement;
- resultIndex3 = ElementWiseSelect(mask3.AsUInt32(), currentIndex, resultIndex3);
- currentIndex += indexIncrement;
- resultIndex4 = ElementWiseSelect(mask4.AsUInt32(), currentIndex, resultIndex4);
- currentIndex += indexIncrement;
}
- {
- // Where result does not bitwise-equal the aggregate min/max value; replace indices with uint.MaxValue. Then find the min index.
- T aggResult = TOperator.Aggregate(result);
- Vector128 aggMask = ~Vector128.Equals(result.AsSByte(), Vector128.Create(aggResult).AsSByte());
-
- (Vector128 lowerMask, Vector128 upperMask) = Vector128.Widen(aggMask);
- (Vector128 mask1, Vector128 mask2) = Vector128.Widen(lowerMask);
- (Vector128 mask3, Vector128 mask4) = Vector128.Widen(upperMask);
- Vector128 aggIndex = resultIndex1 | mask1.AsUInt32();
- aggIndex = MinOperator.Invoke(aggIndex, resultIndex2 | mask2.AsUInt32());
- aggIndex = MinOperator.Invoke(aggIndex, resultIndex3 | mask3.AsUInt32());
- aggIndex = MinOperator.Invoke(aggIndex, resultIndex4 | mask4.AsUInt32());
-
- return (int)HorizontalAggregate>(aggIndex);
- }
+ Debug.Fail("A NaN was expected in the block.");
+ return -1;
}
- private static int IndexOfMinMaxVectorized256Size4Plus(ReadOnlySpan x)
- where T : INumber where TOperator : struct, IIndexOfMinMaxOperator where TInt : IBinaryInteger
+ ///
+ /// Index of the first element among elements starting at that
+ /// does not beat under ; must be the
+ /// block's best element, so this is the first element tied with it (equal, or an equal-magnitude tie the operator does not order).
+ ///
+ [MethodImpl(MethodImplOptions.NoInlining)] // called once per search; keeps the caller within the inlining budget
+ private static int IndexOfFirstNotBeaten128(ref T xRef, int length, T value)
+ where T : INumber where TOperator : struct, IIndexOfMinMaxOperator
{
- Debug.Assert(sizeof(T) == 4 || sizeof(T) == 8);
- Debug.Assert(typeof(TInt) == typeof(uint) || typeof(TInt) == typeof(ulong));
- Debug.Assert(sizeof(TInt) == sizeof(T));
+ int count = Vector128.Count;
+ Vector128 best = Vector128.Create(value);
+ int i = 0;
- // Initialize result by reading first vector and quick return if possible.
- Vector256 result = Vector256.Create(x);
- if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
+ for (; i + count <= length; i += count)
{
- Vector256 nanMask = Vector256.IsNaN(result);
- if (nanMask != Vector256.Zero)
+ var bits = (~TOperator.Compare(best, Vector128.LoadUnsafe(ref xRef, (nuint)i))).ExtractMostSignificantBits();
+ if (bits != 0)
{
- return IndexOfFirstMatch(nanMask);
+ return i + BitOperations.TrailingZeroCount(bits);
}
}
- // Initialize indices.
- Vector256 indexIncrement = Vector256.Create(TInt.CreateTruncating(Vector256.Count));
- Vector256 resultIndex = Vector256.Indices;
- Vector256 currentIndex = resultIndex + indexIncrement;
- ReadOnlySpan span = x.Slice(Vector256.Count);
-
- while (!span.IsEmpty)
+ for (; i < length; i++)
{
- Vector256 current;
- if (span.Length >= Vector256.Count)
- {
- current = Vector256.Create(span);
- span = span.Slice(Vector256.Count);
- }
- else
+ if (!TOperator.Compare(value, Unsafe.Add(ref xRef, i)))
{
- // Process a final back-shifted to cover remaining elements in x in one vector.
- int start = x.Length - Vector256.Count;
- current = Vector256.Create(x.Slice(start));
- currentIndex = Vector256.Create(TInt.CreateTruncating(start)) + Vector256.Indices;
- span = ReadOnlySpan.Empty;
+ return i;
}
+ }
+
+ Debug.Fail("The block's best element was expected in the block.");
+ return -1;
+ }
+
+ /// See .
+ private static int IndexOfMinMaxBlocks256(ReadOnlySpan x)
+ where T : INumber where TOperator : struct, IIndexOfMinMaxOperator
+ {
+ Debug.Assert(Vector256.IsHardwareAccelerated && Vector256.IsSupported);
+ Debug.Assert(x.Length >= Vector256.Count);
+ Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8);
+
+ int blockSize = BlockVectors * Vector256.Count;
+ int length = x.Length;
+ ref T xRef = ref MemoryMarshal.GetReference(x);
+
+ // Pass 1: reduce every block to its best element; the first block whose best beats the running result wins ties.
+ T result = xRef;
+ int resultBlock = -1;
+ for (int i = 0; i < length; i += blockSize)
+ {
+ int blockLength = Math.Min(blockSize, length - i);
+ T blockResult = BlockReduce256(ref Unsafe.Add(ref xRef, i), blockLength);
- // Quick return if possible.
if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
{
- Vector256 nanMask = Vector256.IsNaN(current);
- if (nanMask != Vector256.Zero)
+ if (T.IsNaN(blockResult))
{
- return int.CreateTruncating(currentIndex.ToScalar()) + IndexOfFirstMatch(nanMask);
+ return i + IndexOfFirstNaN256(ref Unsafe.Add(ref xRef, i), blockLength);
}
}
- // Get mask for which lanes that should have result updated.
- Vector256 mask = TOperator.Compare(current, result);
+ Debug.Assert(NoElementBeats(ref Unsafe.Add(ref xRef, i), blockLength, blockResult),
+ "Reduce/Aggregate must return an element no other element of the block beats under Compare, and must propagate NaN.");
- // Update result and indices.
- result = ElementWiseSelect(mask, current, result);
- resultIndex = ElementWiseSelect(mask.As(), currentIndex, resultIndex);
- currentIndex += indexIncrement;
+ if (resultBlock < 0 || TOperator.Compare(blockResult, result))
+ {
+ result = blockResult;
+ resultBlock = i;
+ }
}
- {
- // Where result does not bitwise-equal the aggregate min/max value; replace indices with uint.MaxValue. Then find the min index.
- T aggResult = TOperator.Aggregate(result);
- Vector256 aggMask = ~Vector256.Equals(result.As(), Vector256.Create(aggResult).As());
- Vector256 aggIndex = resultIndex | aggMask;
- return int.CreateTruncating(HorizontalAggregate>(aggIndex));
- }
+ Debug.Assert(resultBlock >= 0);
+
+ // Pass 2: the first element of the winning block that the result does not beat, i.e. the first tied-best element.
+ return resultBlock + IndexOfFirstNotBeaten256(ref Unsafe.Add(ref xRef, resultBlock), Math.Min(blockSize, length - resultBlock), result);
}
- private static int IndexOfMinMaxVectorized256Size2(ReadOnlySpan x)
+ /// Reduces elements starting at to their best element (no bounds checks).
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ private static T BlockReduce256(ref T xRef, int length)
where T : INumber where TOperator : struct, IIndexOfMinMaxOperator
{
- Debug.Assert(sizeof(T) == 2);
+ Debug.Assert(length >= 1);
- // Initialize result by reading first vector and quick return if possible.
- Vector256 result = Vector256.Create(x);
- if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
- {
- Vector256 nanMask = Vector256.IsNaN(result);
- if (nanMask != Vector256.Zero)
- {
- return IndexOfFirstMatch(nanMask);
- }
- }
-
- // Initialize indices.
- Vector256 indexIncrement = Vector256.Create((uint)Vector256.Count);
- Vector256 resultIndex1 = Vector256.Indices;
- Vector256 resultIndex2 = resultIndex1 + indexIncrement;
- Vector256 currentIndex = resultIndex2 + indexIncrement;
- ReadOnlySpan span = x.Slice(Vector256.Count);
+ nuint count = (nuint)Vector256.Count;
+ nuint end = (nuint)length;
+ T result;
+ nuint i;
- while (!span.IsEmpty)
+ if (end >= 2 * count)
{
- Vector256 current;
- if (span.Length >= Vector256.Count)
- {
- current = Vector256.Create(span);
- span = span.Slice(Vector256.Count);
- }
- else
+ // Two independent accumulators so that consecutive reductions do not serialize on one register.
+ Vector256 acc1 = Vector256.LoadUnsafe(ref xRef);
+ Vector256 acc2 = Vector256.LoadUnsafe(ref xRef, count);
+ nuint last = end - 2 * count;
+ for (i = 2 * count; i <= last; i += 2 * count)
{
- // Process a final back-shifted to cover remaining elements in x in one vector.
- int start = x.Length - Vector256.Count;
- current = Vector256.Create(x.Slice(start));
- currentIndex = Vector256.Create((uint)start) + Vector256.Indices;
- span = ReadOnlySpan.Empty;
+ acc1 = TOperator.Reduce(acc1, Vector256.LoadUnsafe(ref xRef, i));
+ acc2 = TOperator.Reduce(acc2, Vector256.LoadUnsafe(ref xRef, i + count));
}
- // Quick return if possible.
- if (typeof(T) == typeof(float) || typeof(T) == typeof(double))
+ if (i + count <= end)
{
- Vector256 nanMask = Vector256.IsNaN(current);
- if (nanMask != Vector256.Zero)
- {
- return (int)currentIndex.ToScalar() + IndexOfFirstMatch(nanMask);
- }
+ acc1 = TOperator.Reduce(acc1, Vector256.LoadUnsafe(ref xRef, i));
+ i += count;
}
- // Get mask for which lanes that should have result updated, also widen it for updating the indices.
- Vector256 mask = TOperator.Compare(current, result);
- (Vector256 mask1, Vector256 mask2) = Vector256.Widen(mask.AsInt16());
-
- // Update result and indices.
- result = ElementWiseSelect(mask, current, result);
- resultIndex1 = ElementWiseSelect(mask1.AsUInt32(), currentIndex, resultIndex1);
- currentIndex += indexIncrement;
- resultIndex2 = ElementWiseSelect(mask2.AsUInt32(), currentIndex, resultIndex2);
- currentIndex += indexIncrement;
+ result = TOperator.Aggregate(TOperator.Reduce(acc1, acc2));
}
-
+ else if (end >= count)
{
- // Where result does not bitwise-equal the aggregate min/max value; replace indices with uint.MaxValue. Then find the min index.
- T aggResult = TOperator.Aggregate(result);
- Vector256