Skip to content

Commit 5898e80

Browse files
SwapnilGaikwadSwapnilGaikwad
committed
Add intrinsic for IndexOfAny on AArch64
1 parent ebdb045 commit 5898e80

File tree

1 file changed

+43
-12
lines changed

1 file changed

+43
-12
lines changed

‎src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
using System.Numerics;
77
using System.Runtime.CompilerServices;
88
using System.Runtime.Intrinsics;
9+
using System.Runtime.Intrinsics.Arm;
910

1011
namespace System
1112
{
@@ -1543,7 +1544,7 @@ private static int IndexOfValueType<TValue, TNegator>(ref TValue searchSpace, TV
15431544
do
15441545
{
15451546
equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref currentSearchSpace)));
1546-
if (equals == Vector128<TValue>.Zero)
1547+
if (!VectorContainsMatch(equals))
15471548
{
15481549
currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<TValue>.Count);
15491550
continue;
@@ -1557,7 +1558,7 @@ private static int IndexOfValueType<TValue, TNegator>(ref TValue searchSpace, TV
15571558
if ((uint)length % Vector128<TValue>.Count != 0)
15581559
{
15591560
equals = TNegator.NegateIfNeeded(Vector128.Equals(values, Vector128.LoadUnsafe(ref oneVectorAwayFromEnd)));
1560-
if (equals != Vector128<TValue>.Zero)
1561+
if (VectorContainsMatch(equals))
15611562
{
15621563
return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
15631564
}
@@ -1691,7 +1692,7 @@ private static int IndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace,
16911692
{
16921693
current = Vector128.LoadUnsafe(ref currentSearchSpace);
16931694
equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
1694-
if (equals == Vector128<TValue>.Zero)
1695+
if (!VectorContainsMatch(equals))
16951696
{
16961697
currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<TValue>.Count);
16971698
continue;
@@ -1706,7 +1707,7 @@ private static int IndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace,
17061707
{
17071708
current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
17081709
equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current));
1709-
if (equals != Vector128<TValue>.Zero)
1710+
if (VectorContainsMatch(equals))
17101711
{
17111712
return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
17121713
}
@@ -1835,7 +1836,7 @@ private static int IndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace,
18351836
{
18361837
current = Vector128.LoadUnsafe(ref currentSearchSpace);
18371838
equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
1838-
if (equals == Vector128<TValue>.Zero)
1839+
if (!VectorContainsMatch(equals))
18391840
{
18401841
currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<TValue>.Count);
18411842
continue;
@@ -1850,7 +1851,7 @@ private static int IndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace,
18501851
{
18511852
current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
18521853
equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current));
1853-
if (equals != Vector128<TValue>.Zero)
1854+
if (VectorContainsMatch(equals))
18541855
{
18551856
return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
18561857
}
@@ -1954,7 +1955,7 @@ private static int IndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace,
19541955
current = Vector128.LoadUnsafe(ref currentSearchSpace);
19551956
equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
19561957
| Vector128.Equals(values2, current) | Vector128.Equals(values3, current));
1957-
if (equals == Vector128<TValue>.Zero)
1958+
if (!VectorContainsMatch(equals))
19581959
{
19591960
currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<TValue>.Count);
19601961
continue;
@@ -1970,7 +1971,7 @@ private static int IndexOfAnyValueType<TValue, TNegator>(ref TValue searchSpace,
19701971
current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
19711972
equals = TNegator.NegateIfNeeded(Vector128.Equals(values0, current) | Vector128.Equals(values1, current)
19721973
| Vector128.Equals(values2, current) | Vector128.Equals(values3, current));
1973-
if (equals != Vector128<TValue>.Zero)
1974+
if (VectorContainsMatch(equals))
19741975
{
19751976
return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
19761977
}
@@ -2067,7 +2068,7 @@ internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1
20672068
current = Vector128.LoadUnsafe(ref currentSearchSpace);
20682069
equals = Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current)
20692070
| Vector128.Equals(values3, current) | Vector128.Equals(values4, current);
2070-
if (equals == Vector128<T>.Zero)
2071+
if (!VectorContainsMatch(equals))
20712072
{
20722073
currentSearchSpace = ref Unsafe.Add(ref currentSearchSpace, Vector128<T>.Count);
20732074
continue;
@@ -2083,7 +2084,7 @@ internal static int IndexOfAnyValueType<T>(ref T searchSpace, T value0, T value1
20832084
current = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd);
20842085
equals = Vector128.Equals(values0, current) | Vector128.Equals(values1, current) | Vector128.Equals(values2, current)
20852086
| Vector128.Equals(values3, current) | Vector128.Equals(values4, current);
2086-
if (equals != Vector128<T>.Zero)
2087+
if (VectorContainsMatch(equals))
20872088
{
20882089
return ComputeFirstIndex(ref searchSpace, ref oneVectorAwayFromEnd, equals);
20892090
}
@@ -2614,11 +2615,41 @@ private static int LastIndexOfAnyValueType<TValue, TNegator>(ref TValue searchSp
26142615
return -1;
26152616
}
26162617

2618+
[MethodImpl(MethodImplOptions.AggressiveOptimization)]
2619+
private static bool VectorContainsMatch<TValue>(Vector128<TValue> inputVector) where TValue : struct, INumber<TValue>
2620+
{
2621+
// The inputVector has all the bits of a matching element set.
2622+
if (AdvSimd.Arm64.IsSupported)
2623+
{
2624+
// We select a larger byte from the adjacent pair of bytes. Thus, if any of the 16 bytes of inputVector are set,
2625+
// representing a matched element, they get reflected in the bottom half (8 bytes) of the register.
2626+
// We then detect a match when any of the bottom 8 bytes are non-zero.
2627+
return AdvSimd.Arm64.MaxPairwise(inputVector.AsByte(), inputVector.AsByte()).AsUInt64().ToScalar() != 0;
2628+
}
2629+
else
2630+
{
2631+
return (inputVector != Vector128<TValue>.Zero);
2632+
}
2633+
}
2634+
26172635
[MethodImpl(MethodImplOptions.AggressiveInlining)]
26182636
private static int ComputeFirstIndex<T>(ref T searchSpace, ref T current, Vector128<T> equals) where T : struct
26192637
{
2620-
uint notEqualsElements = equals.ExtractMostSignificantBits();
2621-
int index = BitOperations.TrailingZeroCount(notEqualsElements);
2638+
int index;
2639+
if (AdvSimd.Arm64.IsSupported && Unsafe.SizeOf<T>() > 1)
2640+
{
2641+
// PairwiseMax selects a larger of the adjacent elements. Now the bottom half of
2642+
// the target register (64 bits) represents elements of half the original size.
2643+
// Dividing the number of trailing zeros by 4 gives the byte offset position of
2644+
// the first matching element, and dividing it by the element size gives its index.
2645+
index = (BitOperations.TrailingZeroCount(
2646+
AdvSimd.Arm64.MaxPairwise(equals.AsByte(), equals.AsByte()).AsUInt64().ToScalar()) >> 2) / Unsafe.SizeOf<T>();
2647+
}
2648+
else
2649+
{
2650+
uint notEqualsElements = equals.ExtractMostSignificantBits();
2651+
index = BitOperations.TrailingZeroCount(notEqualsElements);
2652+
}
26222653
return index + (int)(Unsafe.ByteOffset(ref searchSpace, ref current) / Unsafe.SizeOf<T>());
26232654
}
26242655

0 commit comments

Comments
 (0)