Skip to content

Return NULL for mmap fail case on Unix #78069

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
janvorli merged 7 commits into from
Nov 10, 2022
Merged

Return NULL for mmap fail case on Unix #78069

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9a8a047
Return NULL for mmap fail case on Unix
wscho77 Nov 8, 2022
8eb5724
call memset when mmap succeeds
wscho77 Nov 8, 2022
3b75352
check MAP_FAILED instead of NULL
wscho77 Nov 8, 2022
c50ee0e
return false for failing mmap
wscho77 Nov 9, 2022
d9d1f0a
Call g_fatalErrorHandler if releasing failed
wscho77 Nov 9, 2022
936acd6
Update src/coreclr/utilcode/executableallocator.cpp
wscho77 Nov 9, 2022
3e1850c
Fix wrong condition check
wscho77 Nov 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions src/coreclr/minipal/Unix/doublemapping.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,10 @@ bool VMToOSInterface::ReleaseDoubleMappedMemory(void *mapperHandle, void* pStart
{
#ifndef TARGET_OSX
int fd = (int)(size_t)mapperHandle;
mmap(pStart, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, offset);
if (mmap(pStart, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, offset) == MAP_FAILED)
{
return false;
}
memset(pStart, 0, size);
#endif // TARGET_OSX
return munmap(pStart, size) != -1;
Expand All @@ -208,7 +211,12 @@ void* VMToOSInterface::GetRWMapping(void *mapperHandle, void* pStart, size_t off
{
#ifndef TARGET_OSX
int fd = (int)(size_t)mapperHandle;
return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset);
void* result = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset);
if (result == MAP_FAILED)
{
result = NULL;
}
return result;
#else // TARGET_OSX
#ifdef TARGET_AMD64
vm_address_t startRW;
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/nativeaot/Runtime/unix/PalRedhawkUnix.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -743,7 +743,7 @@ REDHAWK_PALEXPORT _Ret_maybenull_ _Post_writable_byte_size_(size) void* REDHAWK_

void * pRetVal = mmap(pAddress, alignedSize, unixProtect, flags, -1, 0);

if (pRetVal != NULL)
if (pRetVal != MAP_FAILED)
{
void * pAlignedRetVal = (void *)(((size_t)pRetVal + (Alignment - 1)) & ~(Alignment - 1));
size_t startPadding = (size_t)pAlignedRetVal - (size_t)pRetVal;
Expand Down
5 changes: 4 additions & 1 deletion src/coreclr/utilcode/executableallocator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,10 @@ void ExecutableAllocator::Release(void* pRX)

if (pBlock != NULL)
{
VMToOSInterface::ReleaseDoubleMappedMemory(m_doubleMemoryMapperHandle, pRX, pBlock->offset, pBlock->size);
if (!VMToOSInterface::ReleaseDoubleMappedMemory(m_doubleMemoryMapperHandle, pRX, pBlock->offset, pBlock->size))
{
g_fatalErrorHandler(COR_E_EXECUTIONENGINE, W("Releasing the double mapped memory failed"));
}
// Put the released block into the free block list
pBlock->baseRX = NULL;
pBlock->next = m_pFirstFreeBlockRX;
Expand Down