Skip to content

Commit

Permalink
[NativeAOT] Unify common cases of Array.Copy with CoreCLR (#83351)
Browse files Browse the repository at this point in the history 
* Unify common cases of Array.Copy

* ContainsGCPointers

* removed EETypePtr.FastEquals

* AreSameType

* fix the build after rebase
  • Loading branch information
@VSadov
VSadov committed Mar 14, 2023
1 parent 2b185ac commit 648d477
Show file tree
Hide file tree
Showing 15 changed files with 115 additions and 165 deletions.
69 changes: 2 additions & 67 deletions src/coreclr/System.Private.CoreLib/src/System/Array.CoreCLR.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,73 +18,8 @@ public abstract partial class Array : ICloneable, IList, IStructuralComparable,
[MethodImpl(MethodImplOptions.InternalCall)]
private static extern unsafe Array InternalCreate(RuntimeType elementType, int rank, int* pLengths, int* pLowerBounds);

// Copies length elements from sourceArray, starting at index 0, to
// destinationArray, starting at index 0.
//
public static unsafe void Copy(Array sourceArray, Array destinationArray, int length)
{
if (sourceArray == null)
ThrowHelper.ThrowArgumentNullException(ExceptionArgument.sourceArray);
if (destinationArray == null)
ThrowHelper.ThrowArgumentNullException(ExceptionArgument.destinationArray);

MethodTable* pMT = RuntimeHelpers.GetMethodTable(sourceArray);
if (pMT == RuntimeHelpers.GetMethodTable(destinationArray) &&
!pMT->IsMultiDimensionalArray &&
(uint)length <= sourceArray.NativeLength &&
(uint)length <= destinationArray.NativeLength)
{
nuint byteCount = (uint)length * (nuint)pMT->ComponentSize;
ref byte src = ref Unsafe.As<RawArrayData>(sourceArray).Data;
ref byte dst = ref Unsafe.As<RawArrayData>(destinationArray).Data;

if (pMT->ContainsGCPointers)
Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount);
else
Buffer.Memmove(ref dst, ref src, byteCount);

// GC.KeepAlive(sourceArray) not required. pMT kept alive via sourceArray
return;
}

// Less common
Copy(sourceArray, sourceArray.GetLowerBound(0), destinationArray, destinationArray.GetLowerBound(0), length, reliable: false);
}

// Copies length elements from sourceArray, starting at sourceIndex, to
// destinationArray, starting at destinationIndex.
//
public static unsafe void Copy(Array sourceArray, int sourceIndex, Array destinationArray, int destinationIndex, int length)
{
if (sourceArray != null && destinationArray != null)
{
MethodTable* pMT = RuntimeHelpers.GetMethodTable(sourceArray);
if (pMT == RuntimeHelpers.GetMethodTable(destinationArray) &&
!pMT->IsMultiDimensionalArray &&
length >= 0 && sourceIndex >= 0 && destinationIndex >= 0 &&
(uint)(sourceIndex + length) <= sourceArray.NativeLength &&
(uint)(destinationIndex + length) <= destinationArray.NativeLength)
{
nuint elementSize = (nuint)pMT->ComponentSize;
nuint byteCount = (uint)length * elementSize;
ref byte src = ref Unsafe.AddByteOffset(ref Unsafe.As<RawArrayData>(sourceArray).Data, (uint)sourceIndex * elementSize);
ref byte dst = ref Unsafe.AddByteOffset(ref Unsafe.As<RawArrayData>(destinationArray).Data, (uint)destinationIndex * elementSize);

if (pMT->ContainsGCPointers)
Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount);
else
Buffer.Memmove(ref dst, ref src, byteCount);

// GC.KeepAlive(sourceArray) not required. pMT kept alive via sourceArray
return;
}
}

// Less common
Copy(sourceArray!, sourceIndex, destinationArray!, destinationIndex, length, reliable: false);
}

private static unsafe void Copy(Array sourceArray, int sourceIndex, Array destinationArray, int destinationIndex, int length, bool reliable)
private static unsafe void CopyImpl(Array sourceArray, int sourceIndex, Array destinationArray, int destinationIndex, int length, bool reliable)
{
if (sourceArray == null)
ThrowHelper.ThrowArgumentNullException(ExceptionArgument.sourceArray);
Expand Down Expand Up @@ -155,7 +90,7 @@ private static unsafe void Copy(Array sourceArray, int sourceIndex, Array destin
// It will up-cast, assuming the array types are correct.
public static void ConstrainedCopy(Array sourceArray, int sourceIndex, Array destinationArray, int destinationIndex, int length)
{
Copy(sourceArray, sourceIndex, destinationArray, destinationIndex, length, reliable: true);
CopyImpl(sourceArray, sourceIndex, destinationArray, destinationIndex, length, reliable: true);
}

/// <summary>
Expand Down
2 changes: 2 additions & 0 deletions ...eclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,8 @@ internal unsafe struct MethodTable

public bool HasTypeEquivalence => (Flags & enum_flag_HasTypeEquivalence) != 0;

internal static bool AreSameType(MethodTable* mt1, MethodTable* mt2) => mt1 == mt2;

public bool HasDefaultConstructor => (Flags & (enum_flag_HasComponentSize | enum_flag_HasDefaultCtor)) == enum_flag_HasDefaultCtor;

public bool IsMultiDimensionalArray
Expand Down
12 changes: 11 additions & 1 deletion src/coreclr/nativeaot/Common/src/Internal/Runtime/MethodTable.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,16 @@ internal bool IsSzArray
}
}

internal bool IsMultiDimensionalArray
{
get
{
Debug.Assert(HasComponentSize);
// See comment on RawArrayData for details
return BaseSize > (uint)(3 * sizeof(IntPtr));
}
}

internal bool IsGeneric
{
get
Expand Down Expand Up @@ -762,7 +772,7 @@ internal bool IsPrimitive
}
}

internal bool HasGCPointers
internal bool ContainsGCPointers
{
get
{
Expand Down
8 changes: 8 additions & 0 deletions src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/MethodTable.Runtime.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,14 @@ internal static bool BothSimpleCasting(MethodTable* pThis, MethodTable* pOther)
return ((pThis->_uFlags | pOther->_uFlags) & (uint)EETypeFlags.ComplexCastingMask) == 0;
}

internal static bool AreSameType(MethodTable* mt1, MethodTable* mt2)
{
if (mt1 == mt2)
return true;

return mt1->IsEquivalentTo(mt2);
}

internal bool IsEquivalentTo(MethodTable* pOtherEEType)
{
fixed (MethodTable* pThis = &this)
Expand Down
4 changes: 2 additions & 2 deletions src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeExports.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ public static unsafe object RhBox(MethodTable* pEEType, ref byte data)

// Copy the unboxed value type data into the new object.
// Perform any write barriers necessary for embedded reference fields.
if (pEEType->HasGCPointers)
if (pEEType->ContainsGCPointers)
{
InternalCalls.RhBulkMoveWithWriteBarrier(ref result.GetRawData(), ref dataAdjustedForNullable, pEEType->ValueTypeSize);
}
Expand Down Expand Up @@ -263,7 +263,7 @@ public static unsafe void RhUnbox(object? obj, ref byte data, MethodTable* pUnbo

ref byte fields = ref obj.GetRawData();

if (pEEType->HasGCPointers)
if (pEEType->ContainsGCPointers)
{
// Copy the boxed fields into the new location in a GC safe manner
InternalCalls.RhBulkMoveWithWriteBarrier(ref data, ref fields, pEEType->ValueTypeSize);
Expand Down
69 changes: 5 additions & 64 deletions src/coreclr/nativeaot/System.Private.CoreLib/src/System/Array.NativeAot.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,65 +148,6 @@ public static void ConstrainedCopy(Array sourceArray, int sourceIndex, Array des
CopyImpl(sourceArray, sourceIndex, destinationArray, destinationIndex, length, reliable: true);
}

public static void Copy(Array sourceArray, Array destinationArray, int length)
{
if (sourceArray is null)
ThrowHelper.ThrowArgumentNullException(ExceptionArgument.sourceArray);
if (destinationArray is null)
ThrowHelper.ThrowArgumentNullException(ExceptionArgument.destinationArray);

EETypePtr eeType = sourceArray.GetEETypePtr();
if (eeType.FastEquals(destinationArray.GetEETypePtr()) &&
eeType.IsSzArray &&
(uint)length <= sourceArray.NativeLength &&
(uint)length <= destinationArray.NativeLength)
{
nuint byteCount = (uint)length * (nuint)eeType.ComponentSize;
ref byte src = ref Unsafe.As<RawArrayData>(sourceArray).Data;
ref byte dst = ref Unsafe.As<RawArrayData>(destinationArray).Data;

if (eeType.HasPointers)
Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount);
else
Buffer.Memmove(ref dst, ref src, byteCount);

// GC.KeepAlive(sourceArray) not required. pMT kept alive via sourceArray
return;
}

// Less common
CopyImpl(sourceArray, sourceArray.GetLowerBound(0), destinationArray, destinationArray.GetLowerBound(0), length, reliable: false);
}

public static unsafe void Copy(Array sourceArray, int sourceIndex, Array destinationArray, int destinationIndex, int length)
{
if (sourceArray != null && destinationArray != null)
{
EETypePtr eeType = sourceArray.GetEETypePtr();
if (eeType.FastEquals(destinationArray.GetEETypePtr()) &&
eeType.IsSzArray &&
length >= 0 && sourceIndex >= 0 && destinationIndex >= 0 &&
(uint)(sourceIndex + length) <= sourceArray.NativeLength &&
(uint)(destinationIndex + length) <= destinationArray.NativeLength)
{
nuint elementSize = (nuint)eeType.ComponentSize;
nuint byteCount = (uint)length * elementSize;
ref byte src = ref Unsafe.AddByteOffset(ref Unsafe.As<RawArrayData>(sourceArray).Data, (uint)sourceIndex * elementSize);
ref byte dst = ref Unsafe.AddByteOffset(ref Unsafe.As<RawArrayData>(destinationArray).Data, (uint)destinationIndex * elementSize);

if (eeType.HasPointers)
Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount);
else
Buffer.Memmove(ref dst, ref src, byteCount);

// GC.KeepAlive(sourceArray) not required. pMT kept alive via sourceArray
return;
}
}

// Less common
CopyImpl(sourceArray!, sourceIndex, destinationArray!, destinationIndex, length, reliable: false);
}

//
// Funnel for all the Array.Copy() overloads. The "reliable" parameter indicates whether the caller for ConstrainedCopy()
Expand Down Expand Up @@ -261,7 +202,7 @@ private static unsafe void CopyImpl(Array sourceArray, int sourceIndex, Array de
{
if (RuntimeImports.AreTypesEquivalent(sourceElementEEType, destinationElementEEType))
{
if (sourceElementEEType.HasPointers)
if (sourceElementEEType.ContainsGCPointers)
{
CopyImplValueTypeArrayWithInnerGcRefs(sourceArray, sourceIndex, destinationArray, destinationIndex, length, reliable);
}
Expand Down Expand Up @@ -521,9 +462,9 @@ private static unsafe void CopyImplValueTypeArrayWithInnerGcRefs(Array sourceArr
//
private static unsafe void CopyImplValueTypeArrayNoInnerGcRefs(Array sourceArray, int sourceIndex, Array destinationArray, int destinationIndex, int length)
{
Debug.Assert((sourceArray.ElementEEType.IsValueType && !sourceArray.ElementEEType.HasPointers) ||
Debug.Assert((sourceArray.ElementEEType.IsValueType && !sourceArray.ElementEEType.ContainsGCPointers) ||
sourceArray.ElementEEType.IsPointer);
Debug.Assert((destinationArray.ElementEEType.IsValueType && !destinationArray.ElementEEType.HasPointers) ||
Debug.Assert((destinationArray.ElementEEType.IsValueType && !destinationArray.ElementEEType.ContainsGCPointers) ||
destinationArray.ElementEEType.IsPointer);

// Copy scenario: ValueType-array to value-type array with no embedded gc-refs.
Expand Down Expand Up @@ -822,7 +763,7 @@ public static unsafe void Clear(Array array)
nuint totalByteLength = eeType.ComponentSize * array.NativeLength;
ref byte pStart = ref MemoryMarshal.GetArrayDataReference(array);

if (!eeType.HasPointers)
if (!eeType.ContainsGCPointers)
{
SpanHelpers.ClearWithoutReferences(ref pStart, totalByteLength);
}
Expand Down Expand Up @@ -859,7 +800,7 @@ public static unsafe void Clear(Array array, int index, int length)
ref byte ptr = ref Unsafe.AddByteOffset(ref p, (uint)offset * elementSize);
nuint byteLength = (uint)length * elementSize;

if (eeType.HasPointers)
if (eeType.ContainsGCPointers)
{
Debug.Assert(byteLength % (nuint)sizeof(IntPtr) == 0);
SpanHelpers.ClearWithReferences(ref Unsafe.As<byte, IntPtr>(ref ptr), byteLength / (uint)sizeof(IntPtr));
Expand Down
20 changes: 3 additions & 17 deletions src/coreclr/nativeaot/System.Private.CoreLib/src/System/EETypePtr.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,20 +79,6 @@ public override int GetHashCode()
return (int)_value->HashCode;
}

//
// Faster version of Equals for use on EETypes that are known not to be null and where the "match" case is the hot path.
//
public bool FastEquals(EETypePtr other)
{
Debug.Assert(!this.IsNull);
Debug.Assert(!other.IsNull);

// Fast check for raw equality before making call to helper.
if (this.RawValue == other.RawValue)
return true;
return RuntimeImports.AreTypesEquivalent(this, other);
}

// Caution: You cannot safely compare RawValue's as RH does NOT unify EETypes. Use the == or Equals() methods exposed by EETypePtr itself.
internal IntPtr RawValue
{
Expand Down Expand Up @@ -356,12 +342,12 @@ internal IntPtr DispatchMap
}
}

// Has internal gc pointers.
internal bool HasPointers
// Instance contains pointers to managed objects.
internal bool ContainsGCPointers
{
get
{
return _value->HasGCPointers;
return _value->ContainsGCPointers;
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/nativeaot/System.Private.CoreLib/src/System/Object.NativeAot.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ protected internal object MemberwiseClone()
ref byte src = ref this.GetRawData();
ref byte dst = ref clone.GetRawData();

if (this.GetEETypePtr().HasPointers)
if (this.GetEETypePtr().ContainsGCPointers)
Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount);
else
Buffer.Memmove(ref dst, ref src, byteCount);
Expand Down
2 changes: 1 addition & 1 deletion ...ot/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.NativeAot.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ public static unsafe bool TryEnsureSufficientExecutionStack()
public static bool IsReferenceOrContainsReferences<T>()
{
var pEEType = EETypePtr.EETypePtrOf<T>();
return !pEEType.IsValueType || pEEType.HasPointers;
return !pEEType.IsValueType || pEEType.ContainsGCPointers;
}

[Intrinsic]
Expand Down
2 changes: 1 addition & 1 deletion .../nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/InteropExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ internal static bool MightBeBlittable(this EETypePtr eeType)
// This is used as the approximate implementation of MethodTable::IsBlittable(). It will err in the direction of declaring
// things blittable since it is used for argument validation only.
//
return !eeType.HasPointers;
return !eeType.ContainsGCPointers;
}

public static bool IsBlittable(this RuntimeTypeHandle handle)
Expand Down
2 changes: 1 addition & 1 deletion .../nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.NativeAot.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ public static void SetLastPInvokeError(int error)

internal static bool IsPinnable(object o)
{
return (o == null) || !o.GetEETypePtr().HasPointers;
return (o == null) || !o.GetEETypePtr().ContainsGCPointers;
}

[EditorBrowsable(EditorBrowsableState.Never)]
Expand Down
4 changes: 2 additions & 2 deletions src/coreclr/nativeaot/System.Private.CoreLib/src/System/ValueType.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ public override unsafe bool Equals([NotNullWhen(true)] object? obj)
if (numFields == UseFastHelper)
{
// Sanity check - if there are GC references, we should not be comparing bytes
Debug.Assert(!this.GetEETypePtr().HasPointers);
Debug.Assert(!this.GetEETypePtr().ContainsGCPointers);

// Compare the memory
int valueTypeSize = (int)this.GetEETypePtr().ValueTypeSize;
Expand Down Expand Up @@ -115,7 +115,7 @@ private unsafe int GetHashCodeImpl()
private static unsafe int FastGetValueTypeHashCodeHelper(MethodTable* type, ref byte data)
{
// Sanity check - if there are GC references, we should not be hashing bytes
Debug.Assert(!type->HasGCPointers);
Debug.Assert(!type->ContainsGCPointers);

int size = (int)type->ValueTypeSize;
int hashCode = 0;
Expand Down

0 comments on commit 648d477

Please sign in to comment.