Skip to content

Commit

Permalink
port SpanHelpers.IndexOf(ref char, char, int) (#73368)
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamsitnik
adamsitnik committed Aug 5, 2022
1 parent a0a03e4 commit 66c93ca
Showing 1 changed file with 16 additions and 48 deletions.
64 changes: 16 additions & 48 deletions src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Char.cs
View file
Expand Up @@ -526,7 +526,7 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
{
// Input isn't char aligned, we won't be able to align it to a Vector
}
else if (Sse2.IsSupported || AdvSimd.Arm64.IsSupported)
else if (Vector128.IsHardwareAccelerated)
{
// Avx2 branch also operates on Sse2 sizes, so check is combined.
// Needs to be double length to allow us to align the data first.
Expand Down Expand Up @@ -575,13 +575,14 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
lengthToExamine--;
}

// We get past SequentialScan only if IsHardwareAccelerated or intrinsic .IsSupported is true. However, we still have the redundant check to allow
// We get past SequentialScan only if IsHardwareAccelerated is true. However, we still have the redundant check to allow
// the JIT to see that the code is unreachable and eliminate it when the platform does not have hardware accelerated.
if (Avx2.IsSupported)
if (Vector256.IsHardwareAccelerated)
{
if (offset < length)
{
Debug.Assert(length - offset >= Vector128<ushort>.Count);
ref ushort ushortSearchSpace = ref Unsafe.As<char, ushort>(ref searchSpace);
if (((nint)Unsafe.AsPointer(ref Unsafe.Add(ref searchSpace, (nint)offset)) & (nint)(Vector256<byte>.Count - 1)) != 0)
{
// Not currently aligned to Vector256 (is aligned to Vector128); this can cause a problem for searches
Expand All @@ -600,10 +601,10 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
// the misalignment that may occur after; to we default to giving the GC a free hand to relocate and
// its up to the caller whether they are operating over fixed data.
Vector128<ushort> values = Vector128.Create((ushort)value);
Vector128<ushort> search = LoadVector128(ref searchSpace, offset);
Vector128<ushort> search = Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset);

// Same method as below
int matches = Sse2.MoveMask(Sse2.CompareEqual(values, search).AsByte());
uint matches = Vector128.Equals(values, search).AsByte().ExtractMostSignificantBits();
if (matches == 0)
{
// Zero flags set so no matches
Expand All @@ -624,8 +625,8 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
{
Debug.Assert(lengthToExamine >= Vector256<ushort>.Count);

Vector256<ushort> search = LoadVector256(ref searchSpace, offset);
int matches = Avx2.MoveMask(Avx2.CompareEqual(values, search).AsByte());
Vector256<ushort> search = Vector256.LoadUnsafe(ref ushortSearchSpace, (nuint)offset);
uint matches = Vector256.Equals(values, search).AsByte().ExtractMostSignificantBits();
// Note that MoveMask has converted the equal vector elements into a set of bit flags,
// So the bit position in 'matches' corresponds to the element offset.
if (matches == 0)
Expand All @@ -648,10 +649,10 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
Debug.Assert(lengthToExamine >= Vector128<ushort>.Count);

Vector128<ushort> values = Vector128.Create((ushort)value);
Vector128<ushort> search = LoadVector128(ref searchSpace, offset);
Vector128<ushort> search = Vector128.LoadUnsafe(ref ushortSearchSpace, (nuint)offset);

// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values, search).AsByte());
uint matches = Vector128.Equals(values, search).AsByte().ExtractMostSignificantBits();
if (matches == 0)
{
// Zero flags set so no matches
Expand All @@ -673,11 +674,12 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
}
}
}
else if (Sse2.IsSupported)
else if (Vector128.IsHardwareAccelerated)
{
if (offset < length)
{
Debug.Assert(length - offset >= Vector128<ushort>.Count);
ref ushort ushortSearchSpace = ref Unsafe.As<char, ushort>(ref searchSpace);

lengthToExamine = GetCharVector128SpanLength(offset, length);
if (lengthToExamine > 0)
Expand All @@ -687,11 +689,11 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
{
Debug.Assert(lengthToExamine >= Vector128<ushort>.Count);

Vector128<ushort> search = LoadVector128(ref searchSpace, offset);
Vector128<ushort> search = Vector128.LoadUnsafe(ref ushortSearchSpace, (uint)offset);

// Same method as above
int matches = Sse2.MoveMask(Sse2.CompareEqual(values, search).AsByte());
if (matches == 0)
Vector128<ushort> compareResult = Vector128.Equals(values, search);
if (compareResult == Vector128<ushort>.Zero)
{
// Zero flags set so no matches
offset += Vector128<ushort>.Count;
Expand All @@ -701,6 +703,7 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)

// Find bitflag offset of first match and add to current offset,
// flags are in bytes so divide for chars
uint matches = compareResult.AsByte().ExtractMostSignificantBits();
return (int)(offset + ((uint)BitOperations.TrailingZeroCount(matches) / sizeof(char)));
} while (lengthToExamine > 0);
}
Expand All @@ -712,41 +715,6 @@ public static unsafe int IndexOf(ref char searchSpace, char value, int length)
}
}
}
else if (AdvSimd.Arm64.IsSupported)
{
if (offset < length)
{
Debug.Assert(length - offset >= Vector128<ushort>.Count);

lengthToExamine = GetCharVector128SpanLength(offset, length);
if (lengthToExamine > 0)
{
Vector128<ushort> values = Vector128.Create((ushort)value);
do
{
Debug.Assert(lengthToExamine >= Vector128<ushort>.Count);

Vector128<ushort> search = LoadVector128(ref searchSpace, offset);
Vector128<ushort> compareResult = AdvSimd.CompareEqual(values, search);

if (compareResult == Vector128<ushort>.Zero)
{
offset += Vector128<ushort>.Count;
lengthToExamine -= Vector128<ushort>.Count;
continue;
}

return (int)(offset + FindFirstMatchedLane(compareResult));
} while (lengthToExamine > 0);
}

if (offset < length)
{
lengthToExamine = length - offset;
goto SequentialScan;
}
}
}
else if (Vector.IsHardwareAccelerated)
{
if (offset < length)
Expand Down

0 comments on commit 66c93ca

Please sign in to comment.