Skip to content

Commit

Permalink
JIT: Convert Interlocked intrinsics to NamedIntrinsics (#55170)
Browse files Browse the repository at this point in the history
  • Loading branch information
@EgorBo
EgorBo committed Sep 25, 2021
1 parent 234b789 commit b9ef92f
Show file tree
Hide file tree
Showing 8 changed files with 128 additions and 161 deletions.
3 changes: 3 additions & 0 deletions src/coreclr/System.Private.CoreLib/src/System/Threading/Interlocked.CoreCLR.cs
View file
Expand Up @@ -57,6 +57,7 @@ public static partial class Interlocked
/// <param name="value">The value to which the <paramref name="location1"/> parameter is set.</param>
/// <returns>The original value of <paramref name="location1"/>.</returns>
/// <exception cref="NullReferenceException">The address of location1 is a null pointer.</exception>
[Intrinsic]
[MethodImpl(MethodImplOptions.InternalCall)]
public static extern long Exchange(ref long location1, long value);

Expand Down Expand Up @@ -117,6 +118,7 @@ public static partial class Interlocked
/// <param name="comparand">The value that is compared to the value at <paramref name="location1"/>.</param>
/// <returns>The original value in <paramref name="location1"/>.</returns>
/// <exception cref="NullReferenceException">The address of <paramref name="location1"/> is a null pointer.</exception>
[Intrinsic]
[MethodImpl(MethodImplOptions.InternalCall)]
public static extern long CompareExchange(ref long location1, long value, long comparand);

Expand Down Expand Up @@ -192,6 +194,7 @@ public static partial class Interlocked
[MethodImpl(MethodImplOptions.InternalCall)]
private static extern int ExchangeAdd(ref int location1, int value);

[Intrinsic]
[MethodImpl(MethodImplOptions.InternalCall)]
private static extern long ExchangeAdd(ref long location1, long value);
#endregion
Expand Down
10 changes: 0 additions & 10 deletions src/coreclr/inc/corinfo.h
View file
Expand Up @@ -892,16 +892,6 @@ enum CorInfoIntrinsics
CORINFO_INTRINSIC_StubHelpers_GetStubContextAddr,
CORINFO_INTRINSIC_StubHelpers_NextCallReturnAddress,

CORINFO_INTRINSIC_InterlockedAdd32,
CORINFO_INTRINSIC_InterlockedAdd64,
CORINFO_INTRINSIC_InterlockedXAdd32,
CORINFO_INTRINSIC_InterlockedXAdd64,
CORINFO_INTRINSIC_InterlockedXchg32,
CORINFO_INTRINSIC_InterlockedXchg64,
CORINFO_INTRINSIC_InterlockedCmpXchg32,
CORINFO_INTRINSIC_InterlockedCmpXchg64,
CORINFO_INTRINSIC_MemoryBarrier,
CORINFO_INTRINSIC_MemoryBarrierLoad,
CORINFO_INTRINSIC_ByReference_Ctor,
CORINFO_INTRINSIC_ByReference_Value,
CORINFO_INTRINSIC_GetRawHandle,
Expand Down
10 changes: 5 additions & 5 deletions src/coreclr/inc/jiteeversionguid.h
View file
Expand Up @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
#define GUID_DEFINED
#endif // !GUID_DEFINED

constexpr GUID JITEEVersionIdentifier = { /* 4ef06a0e-e58d-4796-abb7-0cda6460610c */
0x4ef06a0e,
0xe58d,
0x4796,
{0xab, 0xb7, 0xc, 0xda, 0x64, 0x60, 0x61, 0xc}
constexpr GUID JITEEVersionIdentifier = { /* 802cceb2-2ebd-4ff9-ac31-4c3546a02aa5 */
0x802cceb2,
0x2ebd,
0x4ff9,
{0xac, 0x31, 0x4c, 0x35, 0x46, 0xa0, 0x2a, 0xa5}
};

//////////////////////////////////////////////////////////////////////////////////////////////////////////
Expand Down
204 changes: 107 additions & 97 deletions src/coreclr/jit/importer.cpp
View file
Expand Up @@ -3734,10 +3734,6 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis,

*pIntrinsicID = intrinsicID;

#ifndef TARGET_ARM
genTreeOps interlockedOperator;
#endif

if (intrinsicID == CORINFO_INTRINSIC_StubHelpers_GetStubContext)
{
// must be done regardless of DbgCode and MinOpts
Expand Down Expand Up @@ -3785,96 +3781,6 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis,
{
GenTree* op1;

#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
// TODO-ARM-CQ: reenable treating Interlocked operation as intrinsic

// Note that CORINFO_INTRINSIC_InterlockedAdd32/64 are not actually used.
// Anyway, we can import them as XADD and leave it to lowering/codegen to perform
// whatever optimizations may arise from the fact that result value is not used.
case CORINFO_INTRINSIC_InterlockedAdd32:
case CORINFO_INTRINSIC_InterlockedXAdd32:
interlockedOperator = GT_XADD;
goto InterlockedBinOpCommon;
case CORINFO_INTRINSIC_InterlockedXchg32:
interlockedOperator = GT_XCHG;
goto InterlockedBinOpCommon;

#ifdef TARGET_64BIT
case CORINFO_INTRINSIC_InterlockedAdd64:
case CORINFO_INTRINSIC_InterlockedXAdd64:
interlockedOperator = GT_XADD;
goto InterlockedBinOpCommon;
case CORINFO_INTRINSIC_InterlockedXchg64:
interlockedOperator = GT_XCHG;
goto InterlockedBinOpCommon;
#endif // TARGET_AMD64

InterlockedBinOpCommon:
assert(callType != TYP_STRUCT);
assert(sig->numArgs == 2);

GenTree* op2;
op2 = impPopStack().val;
op1 = impPopStack().val;

// This creates:
// val
// XAdd
// addr
// field (for example)
//
// In the case where the first argument is the address of a local, we might
// want to make this *not* make the var address-taken -- but atomic instructions
// on a local are probably pretty useless anyway, so we probably don't care.

op1 = gtNewOperNode(interlockedOperator, genActualType(callType), op1, op2);
op1->gtFlags |= GTF_GLOB_REF | GTF_ASG;
retNode = op1;
break;
#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64)

case CORINFO_INTRINSIC_MemoryBarrier:
case CORINFO_INTRINSIC_MemoryBarrierLoad:

assert(sig->numArgs == 0);

op1 = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID);
op1->gtFlags |= GTF_GLOB_REF | GTF_ASG;

// On XARCH `CORINFO_INTRINSIC_MemoryBarrierLoad` fences need not be emitted.
// However, we still need to capture the effect on reordering.
if (intrinsicID == CORINFO_INTRINSIC_MemoryBarrierLoad)
{
op1->gtFlags |= GTF_MEMORYBARRIER_LOAD;
}

retNode = op1;
break;

#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
// TODO-ARM-CQ: reenable treating InterlockedCmpXchg32 operation as intrinsic
case CORINFO_INTRINSIC_InterlockedCmpXchg32:
#ifdef TARGET_64BIT
case CORINFO_INTRINSIC_InterlockedCmpXchg64:
#endif
{
assert(callType != TYP_STRUCT);
assert(sig->numArgs == 3);
GenTree* op2;
GenTree* op3;

op3 = impPopStack().val; // comparand
op2 = impPopStack().val; // value
op1 = impPopStack().val; // location

GenTree* node = new (this, GT_CMPXCHG) GenTreeCmpXchg(genActualType(callType), op1, op2, op3);

node->AsCmpXchg()->gtOpLocation->gtFlags |= GTF_DONT_CSE;
retNode = node;
break;
}
#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64)

case CORINFO_INTRINSIC_InitializeArray:
retNode = impInitializeArrayIntrinsic(sig);
break;
Expand Down Expand Up @@ -4372,6 +4278,90 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis,
}
#endif // TARGET_ARM64

#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
// TODO-ARM-CQ: reenable treating InterlockedCmpXchg32 operation as intrinsic
case NI_System_Threading_Interlocked_CompareExchange:
{
var_types retType = JITtype2varType(sig->retType);
if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4))
{
break;
}
if ((retType != TYP_INT) && (retType != TYP_LONG))
{
break;
}

assert(callType != TYP_STRUCT);
assert(sig->numArgs == 3);

GenTree* op3 = impPopStack().val; // comparand
GenTree* op2 = impPopStack().val; // value
GenTree* op1 = impPopStack().val; // location

GenTree* node = new (this, GT_CMPXCHG) GenTreeCmpXchg(genActualType(callType), op1, op2, op3);

node->AsCmpXchg()->gtOpLocation->gtFlags |= GTF_DONT_CSE;
retNode = node;
break;
}

case NI_System_Threading_Interlocked_Exchange:
case NI_System_Threading_Interlocked_ExchangeAdd:
{
assert(callType != TYP_STRUCT);
assert(sig->numArgs == 2);

var_types retType = JITtype2varType(sig->retType);
if ((retType == TYP_LONG) && (TARGET_POINTER_SIZE == 4))
{
break;
}
if ((retType != TYP_INT) && (retType != TYP_LONG))
{
break;
}

GenTree* op2 = impPopStack().val;
GenTree* op1 = impPopStack().val;

// This creates:
// val
// XAdd
// addr
// field (for example)
//
// In the case where the first argument is the address of a local, we might
// want to make this *not* make the var address-taken -- but atomic instructions
// on a local are probably pretty useless anyway, so we probably don't care.

op1 = gtNewOperNode(ni == NI_System_Threading_Interlocked_ExchangeAdd ? GT_XADD : GT_XCHG,
genActualType(callType), op1, op2);
op1->gtFlags |= GTF_GLOB_REF | GTF_ASG;
retNode = op1;
break;
}
#endif // defined(TARGET_XARCH) || defined(TARGET_ARM64)

case NI_System_Threading_Interlocked_MemoryBarrier:
case NI_System_Threading_Interlocked_ReadMemoryBarrier:
{
assert(sig->numArgs == 0);

GenTree* op1 = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID);
op1->gtFlags |= GTF_GLOB_REF | GTF_ASG;

// On XARCH `NI_System_Threading_Interlocked_ReadMemoryBarrier` fences need not be emitted.
// However, we still need to capture the effect on reordering.
if (ni == NI_System_Threading_Interlocked_ReadMemoryBarrier)
{
op1->gtFlags |= GTF_MEMORYBARRIER_LOAD;
}

retNode = op1;
break;
}

#ifdef FEATURE_HW_INTRINSICS
case NI_System_Math_FusedMultiplyAdd:
{
Expand Down Expand Up @@ -4962,10 +4952,10 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
result = NI_System_Threading_Thread_get_ManagedThreadId;
}
}
#ifndef TARGET_ARM64
// TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239).
else if (strcmp(className, "Interlocked") == 0)
{
#ifndef TARGET_ARM64
// TODO-CQ: Implement for XArch (https://github.com/dotnet/runtime/issues/32239).
if (strcmp(methodName, "And") == 0)
{
result = NI_System_Threading_Interlocked_And;
Expand All @@ -4974,8 +4964,28 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method)
{
result = NI_System_Threading_Interlocked_Or;
}
}
#endif
if (strcmp(methodName, "CompareExchange") == 0)
{
result = NI_System_Threading_Interlocked_CompareExchange;
}
else if (strcmp(methodName, "Exchange") == 0)
{
result = NI_System_Threading_Interlocked_Exchange;
}
else if (strcmp(methodName, "ExchangeAdd") == 0)
{
result = NI_System_Threading_Interlocked_ExchangeAdd;
}
else if (strcmp(methodName, "MemoryBarrier") == 0)
{
result = NI_System_Threading_Interlocked_MemoryBarrier;
}
else if (strcmp(methodName, "ReadMemoryBarrier") == 0)
{
result = NI_System_Threading_Interlocked_ReadMemoryBarrier;
}
}
}
#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
else if (strcmp(namespaceName, "System.Buffers.Binary") == 0)
Expand Down
5 changes: 5 additions & 0 deletions src/coreclr/jit/namedintrinsiclist.h
View file
Expand Up @@ -73,6 +73,11 @@ enum NamedIntrinsic : unsigned short

NI_System_Threading_Interlocked_And,
NI_System_Threading_Interlocked_Or,
NI_System_Threading_Interlocked_CompareExchange,
NI_System_Threading_Interlocked_Exchange,
NI_System_Threading_Interlocked_ExchangeAdd,
NI_System_Threading_Interlocked_MemoryBarrier,
NI_System_Threading_Interlocked_ReadMemoryBarrier,

#ifdef FEATURE_HW_INTRINSICS
NI_HW_INTRINSIC_START,
Expand Down
30 changes: 0 additions & 30 deletions src/coreclr/tools/Common/JitInterface/CorInfoImpl.Intrinsics.cs
View file
Expand Up @@ -85,16 +85,6 @@ static IntrinsicHashtable InitializeIntrinsicHashtable()
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_StubHelpers_GetStubContext, "GetStubContext", "System.StubHelpers", "StubHelpers"); // interop-specific
// table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_StubHelpers_GetStubContextAddr, "GetStubContextAddr", "System.StubHelpers", "StubHelpers"); // interop-specific
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_StubHelpers_NextCallReturnAddress, "NextCallReturnAddress", "System.StubHelpers", "StubHelpers");
// table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedAdd32, "Add", System.Threading", "Interlocked"); // unused
// table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedAdd64, "Add", System.Threading", "Interlocked"); // unused
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXAdd32, "ExchangeAdd", "System.Threading", "Interlocked");
// table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXAdd64, "ExchangeAdd", "System.Threading", "Interlocked"); // ambiguous match
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXchg32, "Exchange", "System.Threading", "Interlocked");
// table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXchg64, "Exchange", "System.Threading", "Interlocked"); // ambiguous match
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedCmpXchg32, "CompareExchange", "System.Threading", "Interlocked");
// table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedCmpXchg64, "CompareExchange", "System.Threading", "Interlocked"); // ambiguous match
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_MemoryBarrier, "MemoryBarrier", "System.Threading", "Interlocked");
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_MemoryBarrierLoad, "LoadBarrier", "System.Threading", "Interlocked");
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_ByReference_Ctor, ".ctor", "System", "ByReference`1");
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_ByReference_Value, "get_Value", "System", "ByReference`1");
table.Add(CorInfoIntrinsics.CORINFO_INTRINSIC_GetRawHandle, "EETypePtrOf", "System", "EETypePtr");
Expand Down Expand Up @@ -145,26 +135,6 @@ private CorInfoIntrinsics getIntrinsicID(MethodDesc method, byte* pMustExpand)
return CorInfoIntrinsics.CORINFO_INTRINSIC_Illegal;
break;

case CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXAdd32:
case CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXchg32:
case CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedCmpXchg32:
{
// RyuJIT handles int32 and int64 overloads only
var returnTypeCategory = method.Signature.ReturnType.Category;
if (returnTypeCategory != TypeFlags.Int32 && returnTypeCategory != TypeFlags.Int64 && returnTypeCategory != TypeFlags.IntPtr)
return CorInfoIntrinsics.CORINFO_INTRINSIC_Illegal;

// int64 overloads have different ids
if (returnTypeCategory == TypeFlags.Int64)
{
Debug.Assert((int)CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXAdd32 + 1 == (int)CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXAdd64);
Debug.Assert((int)CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXchg32 + 1 == (int)CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedXchg64);
Debug.Assert((int)CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedCmpXchg32 + 1 == (int)CorInfoIntrinsics.CORINFO_INTRINSIC_InterlockedCmpXchg64);
id = (CorInfoIntrinsics)((int)id + 1);
}
}
break;

case CorInfoIntrinsics.CORINFO_INTRINSIC_RTH_GetValueInternal:
#if !READYTORUN
case CorInfoIntrinsics.CORINFO_INTRINSIC_InitializeArray:
Expand Down
10 changes: 0 additions & 10 deletions src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs
View file
Expand Up @@ -448,16 +448,6 @@ public enum CorInfoIntrinsics
CORINFO_INTRINSIC_StubHelpers_GetStubContext,
CORINFO_INTRINSIC_StubHelpers_GetStubContextAddr,
CORINFO_INTRINSIC_StubHelpers_NextCallReturnAddress,
CORINFO_INTRINSIC_InterlockedAdd32,
CORINFO_INTRINSIC_InterlockedAdd64,
CORINFO_INTRINSIC_InterlockedXAdd32,
CORINFO_INTRINSIC_InterlockedXAdd64,
CORINFO_INTRINSIC_InterlockedXchg32,
CORINFO_INTRINSIC_InterlockedXchg64,
CORINFO_INTRINSIC_InterlockedCmpXchg32,
CORINFO_INTRINSIC_InterlockedCmpXchg64,
CORINFO_INTRINSIC_MemoryBarrier,
CORINFO_INTRINSIC_MemoryBarrierLoad,
CORINFO_INTRINSIC_ByReference_Ctor,
CORINFO_INTRINSIC_ByReference_Value,
CORINFO_INTRINSIC_GetRawHandle,
Expand Down

0 comments on commit b9ef92f

Please sign in to comment.