Fix named mutexes on OSX to work between arm64 and emulated x64 processes (#62765)

- The page size is different between arm64 processes and emulated x64 processes
- The shared memory file size is set to the page size and there was a strict check on the file size, leading to an exception from the second process of a different arch that tries to share the same mutex
- Made the file size check less strict, and allowed an arch to increase but not decrease the file size such that it can be mapped at page size granularity
- Fix for #62140 in main

Author: kouvel

src/coreclr/pal/src/include/pal/mutex.hpp

@@ -120,9 +120,16 @@ Miscellaneous
   existing shared memory, naming, and waiting infrastructure is not suitable for this purpose, and is not used.
 */
 
-// Temporarily disabling usage of pthread process-shared mutexes on ARM/ARM64 due to functional issues that cannot easily be
-// detected with code due to hangs. See https://github.com/dotnet/runtime/issues/6014.
-#if HAVE_FULLY_FEATURED_PTHREAD_MUTEXES && HAVE_FUNCTIONAL_PTHREAD_ROBUST_MUTEXES && !(defined(HOST_ARM) || defined(HOST_ARM64) || defined(__FreeBSD__))
+// - Temporarily disabling usage of pthread process-shared mutexes on ARM/ARM64 due to functional issues that cannot easily be
+//   detected with code due to hangs. See https://github.com/dotnet/runtime/issues/6014.
+// - On FreeBSD, pthread process-shared robust mutexes cannot be placed in shared memory mapped independently by the processes
+//   involved. See https://github.com/dotnet/runtime/issues/10519.
+// - On OSX, pthread robust mutexes were/are not available at the time of this writing. In case they are made available in the
+//   future, their use is disabled for compatibility.
+#if HAVE_FULLY_FEATURED_PTHREAD_MUTEXES && \
+    HAVE_FUNCTIONAL_PTHREAD_ROBUST_MUTEXES && \
+    !(defined(HOST_ARM) || defined(HOST_ARM64) || defined(__FreeBSD__) || defined(TARGET_OSX))
+
     #define NAMED_MUTEX_USE_PTHREAD_MUTEX 1
 #else
     #define NAMED_MUTEX_USE_PTHREAD_MUTEX 0

src/coreclr/pal/src/include/pal/sharedmemory.h

@@ -173,7 +173,8 @@ class SharedMemorySharedDataHeader
     };
 
 public:
-    static SIZE_T DetermineTotalByteCount(SIZE_T dataByteCount);
+    static SIZE_T GetUsedByteCount(SIZE_T dataByteCount);
+    static SIZE_T GetTotalByteCount(SIZE_T dataByteCount);
 
 public:
     SharedMemorySharedDataHeader(SharedMemoryType type, UINT8 version);

src/coreclr/pal/src/sharedmemory/sharedmemory.cpp

@@ -519,9 +519,14 @@ bool SharedMemoryId::AppendSessionDirectoryName(PathCharString& path) const
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // SharedMemorySharedDataHeader
 
-SIZE_T SharedMemorySharedDataHeader::DetermineTotalByteCount(SIZE_T dataByteCount)
+SIZE_T SharedMemorySharedDataHeader::GetUsedByteCount(SIZE_T dataByteCount)
 {
-    return SharedMemoryHelpers::AlignUp(sizeof(SharedMemorySharedDataHeader) + dataByteCount, GetVirtualPageSize());
+    return sizeof(SharedMemorySharedDataHeader) + dataByteCount;
+}
+
+SIZE_T SharedMemorySharedDataHeader::GetTotalByteCount(SIZE_T dataByteCount)
+{
+    return SharedMemoryHelpers::AlignUp(GetUsedByteCount(dataByteCount), GetVirtualPageSize());
 }
 
 SharedMemorySharedDataHeader::SharedMemorySharedDataHeader(SharedMemoryType type, UINT8 version)
@@ -642,7 +647,7 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen(
     {
         _ASSERTE(
             processDataHeader->GetSharedDataTotalByteCount() ==
-            SharedMemorySharedDataHeader::DetermineTotalByteCount(sharedDataByteCount));
+            SharedMemorySharedDataHeader::GetTotalByteCount(sharedDataByteCount));
         processDataHeader->IncRefCount();
         return processDataHeader;
     }
@@ -697,14 +702,23 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen(
     }
 
     // Set or validate the file length
-    SIZE_T sharedDataTotalByteCount = SharedMemorySharedDataHeader::DetermineTotalByteCount(sharedDataByteCount);
+    SIZE_T sharedDataUsedByteCount = SharedMemorySharedDataHeader::GetUsedByteCount(sharedDataByteCount);
+    SIZE_T sharedDataTotalByteCount = SharedMemorySharedDataHeader::GetTotalByteCount(sharedDataByteCount);
     if (createdFile)
     {
         SharedMemoryHelpers::SetFileSize(fileDescriptor, sharedDataTotalByteCount);
     }
-    else if (SharedMemoryHelpers::GetFileSize(fileDescriptor) != sharedDataTotalByteCount)
+    else
     {
-        throw SharedMemoryException(static_cast<DWORD>(SharedMemoryError::HeaderMismatch));
+        SIZE_T currentFileSize = SharedMemoryHelpers::GetFileSize(fileDescriptor);
+        if (currentFileSize < sharedDataUsedByteCount)
+        {
+            throw SharedMemoryException(static_cast<DWORD>(SharedMemoryError::HeaderMismatch));
+        }
+        if (currentFileSize < sharedDataTotalByteCount)
+        {
+            SharedMemoryHelpers::SetFileSize(fileDescriptor, sharedDataTotalByteCount);
+        }
     }
 
     // Acquire and hold a shared file lock on the shared memory file as long as it is open, to indicate that this process is
@@ -726,7 +740,7 @@ SharedMemoryProcessDataHeader *SharedMemoryProcessDataHeader::CreateOrOpen(
     {
         if (clearContents)
         {
-            memset(mappedBuffer, 0, sharedDataTotalByteCount);
+            memset(mappedBuffer, 0, sharedDataUsedByteCount);
         }
         sharedDataHeader = new(mappedBuffer) SharedMemorySharedDataHeader(requiredSharedDataHeader);
     }