diff --git a/src/coreclr/utilcode/util.cpp b/src/coreclr/utilcode/util.cpp index 58423a704e970a..20a25dc4492073 100644 --- a/src/coreclr/utilcode/util.cpp +++ b/src/coreclr/utilcode/util.cpp @@ -23,6 +23,13 @@ #ifndef DACCESS_COMPILE UINT32 g_nClrInstanceId = 0; + +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) +// Flag to check if atomics feature is available on +// the machine +bool g_arm64_atomics_present = false; +#endif + #endif //!DACCESS_COMPILE //***************************************************************************** diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index 37502f21ef1d3b..2710d8c5c9ca25 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -1566,6 +1566,7 @@ void EEJitManager::SetCpuInfo() if (IsProcessorFeaturePresent(PF_ARM_V81_ATOMIC_INSTRUCTIONS_AVAILABLE)) { CPUCompileFlags.Set(InstructionSet_Atomics); + g_arm64_atomics_present = true; } // PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE (43) diff --git a/src/coreclr/vm/syncblk.cpp b/src/coreclr/vm/syncblk.cpp index 4665c62e23a1a0..5b0a5ed6647c7c 100644 --- a/src/coreclr/vm/syncblk.cpp +++ b/src/coreclr/vm/syncblk.cpp @@ -1652,7 +1652,11 @@ AwareLock::EnterHelperResult ObjHeader::EnterObjMonitorHelperSpin(Thread* pCurTh } LONG newValue = oldValue | tid; +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) + if (FastInterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) +#else if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) +#endif { return AwareLock::EnterHelperResult_Entered; } diff --git a/src/coreclr/vm/syncblk.h b/src/coreclr/vm/syncblk.h index 8e83a29cbd4f6f..da480700daa7cb 100644 --- a/src/coreclr/vm/syncblk.h +++ b/src/coreclr/vm/syncblk.h @@ -382,13 +382,21 @@ class AwareLock LockState CompareExchange(LockState toState, LockState fromState) { LIMITED_METHOD_CONTRACT; +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) + return (UINT32)FastInterlockedCompareExchange((LONG *)&m_state, (LONG)toState, (LONG)fromState); +#else return (UINT32)InterlockedCompareExchange((LONG *)&m_state, (LONG)toState, (LONG)fromState); +#endif } LockState CompareExchangeAcquire(LockState toState, LockState fromState) { LIMITED_METHOD_CONTRACT; +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) + return (UINT32)FastInterlockedCompareExchangeAcquire((LONG *)&m_state, (LONG)toState, (LONG)fromState); +#else return (UINT32)InterlockedCompareExchangeAcquire((LONG *)&m_state, (LONG)toState, (LONG)fromState); +#endif } public: diff --git a/src/coreclr/vm/syncblk.inl b/src/coreclr/vm/syncblk.inl index 6b6f5da8cef543..05753058577227 100644 --- a/src/coreclr/vm/syncblk.inl +++ b/src/coreclr/vm/syncblk.inl @@ -602,7 +602,11 @@ FORCEINLINE AwareLock::EnterHelperResult ObjHeader::EnterObjMonitorHelper(Thread } LONG newValue = oldValue | tid; +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) + if (FastInterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) +#else if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) +#endif { return AwareLock::EnterHelperResult_Entered; } @@ -650,7 +654,11 @@ FORCEINLINE AwareLock::EnterHelperResult ObjHeader::EnterObjMonitorHelper(Thread return AwareLock::EnterHelperResult_UseSlowPath; } +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) + if (FastInterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) +#else if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) +#endif { return AwareLock::EnterHelperResult_Entered; } @@ -723,7 +731,12 @@ FORCEINLINE AwareLock::LeaveHelperAction ObjHeader::LeaveObjMonitorHelper(Thread { // We are leaving the lock DWORD newValue = (syncBlockValue & (~SBLK_MASK_LOCK_THREADID)); + +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) + if (FastInterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue) +#else if (InterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue) +#endif { return AwareLock::LeaveHelperAction_Yield; } @@ -732,7 +745,11 @@ FORCEINLINE AwareLock::LeaveHelperAction ObjHeader::LeaveObjMonitorHelper(Thread { // recursion and ThinLock DWORD newValue = syncBlockValue - SBLK_LOCK_RECLEVEL_INC; +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) + if (FastInterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue) +#else if (InterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue) +#endif { return AwareLock::LeaveHelperAction_Yield; } diff --git a/src/coreclr/vm/util.hpp b/src/coreclr/vm/util.hpp index f5dc51daf7ee9e..3004cbcfcf2e9d 100644 --- a/src/coreclr/vm/util.hpp +++ b/src/coreclr/vm/util.hpp @@ -25,6 +25,14 @@ #define MAX_CACHE_LINE_SIZE 64 #endif +#ifndef DACCESS_COMPILE +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) +// Flag to check if atomics feature is available on +// the machine +extern bool g_arm64_atomics_present; +#endif +#endif + #ifndef TARGET_UNIX // Copied from malloc.h: don't want to bring in the whole header file. void * __cdecl _alloca(size_t); @@ -71,6 +79,64 @@ BOOL inline FitsInU4(unsigned __int64 val) return val == (unsigned __int64)(unsigned __int32)val; } +#if defined(DACCESS_COMPILE) +#define FastInterlockedCompareExchange InterlockedCompareExchange +#define FastInterlockedCompareExchangeAcquire InterlockedCompareExchangeAcquire +#define FastInterlockedCompareExchangeRelease InterlockedCompareExchangeRelease +#else + +#if defined(TARGET_WINDOWS) && defined(TARGET_ARM64) + +FORCEINLINE LONG FastInterlockedCompareExchange( + LONG volatile *Destination, + LONG Exchange, + LONG Comperand) +{ + if (g_arm64_atomics_present) + { + return (LONG) __casal32((unsigned __int32*) Destination, (unsigned __int32)Comperand, (unsigned __int32)Exchange); + } + else + { + return InterlockedCompareExchange(Destination, Exchange, Comperand); + } +} + +FORCEINLINE LONG FastInterlockedCompareExchangeAcquire( + IN OUT LONG volatile *Destination, + IN LONG Exchange, + IN LONG Comperand +) +{ + if (g_arm64_atomics_present) + { + return (LONG) __casa32((unsigned __int32*) Destination, (unsigned __int32)Comperand, (unsigned __int32)Exchange); + } + else + { + return InterlockedCompareExchangeAcquire(Destination, Exchange, Comperand); + } +} + +FORCEINLINE LONG FastInterlockedCompareExchangeRelease( + IN OUT LONG volatile *Destination, + IN LONG Exchange, + IN LONG Comperand +) +{ + if (g_arm64_atomics_present) + { + return (LONG) __casl32((unsigned __int32*) Destination, (unsigned __int32)Comperand, (unsigned __int32)Exchange); + } + else + { + return InterlockedCompareExchangeRelease(Destination, Exchange, Comperand); + } +} + +#endif // defined(TARGET_WINDOWS) && defined(TARGET_ARM64) + +#endif //defined(DACCESS_COMPILE) //************************************************************************