Skip to content

Fix VS4Mac crash report and core dump generation perf problems #60205

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mikem8361 merged 5 commits into from
Oct 12, 2021
Merged

Fix VS4Mac crash report and core dump generation perf problems #60205

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a14b23c
Refactor the DAC enumerate memory region phase out of gather crash info
Oct 8, 2021
e8bd4c0
Since on both Linux and MacOS all the RW regions have been already added
Oct 9, 2021
8eea698
Move current_c_gc_state out of !MULTIPLE_HEAPS
cshung Oct 10, 2021
96ee054
Code review feedback
Oct 11, 2021
f93a60e
Opt into fast heap dumps with COMPlus_DbgEnableFastHeapDumps=1 env va…
Oct 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 119 additions & 88 deletions src/coreclr/debug/createdump/crashinfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ CrashInfo::CrashInfo(pid_t pid, bool gatherFrames, pid_t crashThread, uint32_t s
m_pid(pid),
m_ppid(-1),
m_hdac(nullptr),
m_pClrDataEnumRegions(nullptr),
m_pClrDataProcess(nullptr),
m_gatherFrames(gatherFrames),
m_crashThread(crashThread),
m_signal(signal),
Expand Down Expand Up @@ -44,6 +46,15 @@ CrashInfo::~CrashInfo()
}
m_moduleInfos.clear();

// Clean up DAC interfaces
if (m_pClrDataEnumRegions != nullptr)
{
m_pClrDataEnumRegions->Release();
}
if (m_pClrDataProcess != nullptr)
{
m_pClrDataProcess->Release();
}
// Unload DAC module
if (m_hdac != nullptr)
{
Expand Down Expand Up @@ -190,8 +201,16 @@ CrashInfo::GatherCrashInfo(MINIDUMP_TYPE minidumpType)
thread->GetThreadStack();
}
}
// Gather all the useful memory regions from the DAC
if (!EnumerateMemoryRegionsWithDAC(minidumpType))
// Load and initialize DAC interfaces
if (!InitializeDAC())
{
return false;
}
if (!EnumerateManagedModules())
{
return false;
}
if (!UnwindAllThreads())
{
return false;
}
Expand All @@ -204,19 +223,15 @@ CrashInfo::GatherCrashInfo(MINIDUMP_TYPE minidumpType)
// Enumerate all the memory regions using the DAC memory region support given a minidump type
//
bool
CrashInfo::EnumerateMemoryRegionsWithDAC(MINIDUMP_TYPE minidumpType)
CrashInfo::InitializeDAC()
{
ReleaseHolder<DumpDataTarget> dataTarget = new DumpDataTarget(*this);
PFN_CLRDataCreateInstance pfnCLRDataCreateInstance = nullptr;
ICLRDataEnumMemoryRegions* pClrDataEnumRegions = nullptr;
IXCLRDataProcess* pClrDataProcess = nullptr;
HRESULT hr = S_OK;
bool result = false;
HRESULT hr = S_OK;

if (!m_coreclrPath.empty())
{
TRACE("EnumerateMemoryRegionsWithDAC: Memory enumeration STARTED\n");

// We assume that the DAC is in the same location as the libcoreclr.so module
std::string dacPath;
dacPath.append(m_coreclrPath);
Expand All @@ -235,140 +250,156 @@ CrashInfo::EnumerateMemoryRegionsWithDAC(MINIDUMP_TYPE minidumpType)
fprintf(stderr, "GetProcAddress(CLRDataCreateInstance) FAILED %d\n", GetLastError());
goto exit;
}
if ((minidumpType & MiniDumpWithFullMemory) == 0)
{
hr = pfnCLRDataCreateInstance(__uuidof(ICLRDataEnumMemoryRegions), dataTarget, (void**)&pClrDataEnumRegions);
if (FAILED(hr))
{
fprintf(stderr, "CLRDataCreateInstance(ICLRDataEnumMemoryRegions) FAILED %08x\n", hr);
goto exit;
}
// Calls CrashInfo::EnumMemoryRegion for each memory region found by the DAC
hr = pClrDataEnumRegions->EnumMemoryRegions(this, minidumpType, CLRDATA_ENUM_MEM_DEFAULT);
if (FAILED(hr))
{
fprintf(stderr, "EnumMemoryRegions FAILED %08x\n", hr);
goto exit;
}
}
hr = pfnCLRDataCreateInstance(__uuidof(IXCLRDataProcess), dataTarget, (void**)&pClrDataProcess);
hr = pfnCLRDataCreateInstance(__uuidof(ICLRDataEnumMemoryRegions), dataTarget, (void**)&m_pClrDataEnumRegions);
if (FAILED(hr))
{
fprintf(stderr, "CLRDataCreateInstance(IXCLRDataProcess) FAILED %08x\n", hr);
fprintf(stderr, "CLRDataCreateInstance(ICLRDataEnumMemoryRegions) FAILED %08x\n", hr);
goto exit;
}
TRACE("EnumerateMemoryRegionsWithDAC: Memory enumeration FINISHED\n");
if (!EnumerateManagedModules(pClrDataProcess))
hr = pfnCLRDataCreateInstance(__uuidof(IXCLRDataProcess), dataTarget, (void**)&m_pClrDataProcess);
if (FAILED(hr))
{
fprintf(stderr, "CLRDataCreateInstance(IXCLRDataProcess) FAILED %08x\n", hr);
goto exit;
}
}
else {
TRACE("EnumerateMemoryRegionsWithDAC: coreclr not found; not using DAC\n");
}
if (!UnwindAllThreads(pClrDataProcess))
else
{
goto exit;
TRACE("InitializeDAC: coreclr not found; not using DAC\n");
}
result = true;
exit:
if (pClrDataEnumRegions != nullptr)
{
pClrDataEnumRegions->Release();
}
if (pClrDataProcess != nullptr)
return result;
}

//
// Enumerate all the memory regions using the DAC memory region support given a minidump type
//
bool
CrashInfo::EnumerateMemoryRegionsWithDAC(MINIDUMP_TYPE minidumpType)
{
if (m_pClrDataEnumRegions != nullptr && (minidumpType & MiniDumpWithFullMemory) == 0)
{
pClrDataProcess->Release();
TRACE("EnumerateMemoryRegionsWithDAC: Memory enumeration STARTED\n");

// Since on both Linux and MacOS all the RW regions will be added for heap
// dumps by createdump, the only thing differentiating a MiniDumpNormal and
// a MiniDumpWithPrivateReadWriteMemory is that the later uses the EnumMemory
// APIs. This is kind of expensive on larger applications (4 minutes, or even
// more), and this should already be in RW pages. Change the dump type to the
// faster normal one. This one already ensures necessary DAC globals, etc.
// without the costly assembly, module, class, type runtime data structures
// enumeration.
if (minidumpType & MiniDumpWithPrivateReadWriteMemory)
{
char* fastHeapDumps = getenv("COMPlus_DbgEnableFastHeapDumps");
Copy link
Member

@hoyosjs hoyosjs Oct 12, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The only part I don't like about this is we are back to adding debt around the COMPlus_ vs DOTNET_. It's fixed in main - and I don't want to regress this. Granted - this is in 7.0 and I don't know if it's better in a follow up PR or on this one and just ignore that commit on the cherrypick. Basically #include <clrconfignocache.h> and CLRConfigNoCache fastHeapDumpCfg = CLRConfigNoCache::Get("DbgEnableFastHeapDumps", /*noprefix*/ false, &getenv); and then if (fastHeapDumpCfg.IsSet() && fastHeapDumpCfg.TryAsInteger(10, x) && x == 1)

Copy link
Contributor Author

@mikem8361 mikem8361 Oct 12, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I don't like that either. After this gets backported to 6.0 I'm going to either move it to a command option and do the env var in the pal launch code or use the clrconfigNoCache directly.

if (fastHeapDumps != nullptr && strcmp(fastHeapDumps, "1") == 0)
{
minidumpType = MiniDumpNormal;
}
}
// Calls CrashInfo::EnumMemoryRegion for each memory region found by the DAC
HRESULT hr = m_pClrDataEnumRegions->EnumMemoryRegions(this, minidumpType, CLRDATA_ENUM_MEM_DEFAULT);
if (FAILED(hr))
{
fprintf(stderr, "EnumMemoryRegions FAILED %08x\n", hr);
return false;
}
TRACE("EnumerateMemoryRegionsWithDAC: Memory enumeration FINISHED\n");
}
return result;
return true;
}

//
// Enumerate all the managed modules and replace the module mapping with the module name found.
//
bool
CrashInfo::EnumerateManagedModules(IXCLRDataProcess* pClrDataProcess)
CrashInfo::EnumerateManagedModules()
{
CLRDATA_ENUM enumModules = 0;
bool result = true;
HRESULT hr = S_OK;

if (FAILED(hr = pClrDataProcess->StartEnumModules(&enumModules))) {
fprintf(stderr, "StartEnumModules FAILED %08x\n", hr);
return false;
}

while (true)
if (m_pClrDataProcess != nullptr)
{
ReleaseHolder<IXCLRDataModule> pClrDataModule;
if ((hr = pClrDataProcess->EnumModule(&enumModules, &pClrDataModule)) != S_OK) {
break;
}
TRACE("EnumerateManagedModules: Module enumeration STARTED\n");

// Skip any dynamic modules. The Request call below on some DACs crashes on dynamic modules.
ULONG32 flags;
if ((hr = pClrDataModule->GetFlags(&flags)) != S_OK) {
TRACE("MODULE: GetFlags FAILED %08x\n", hr);
continue;
}
if (flags & CLRDATA_MODULE_IS_DYNAMIC) {
TRACE("MODULE: Skipping dynamic module\n");
continue;
if (FAILED(hr = m_pClrDataProcess->StartEnumModules(&enumModules))) {
fprintf(stderr, "StartEnumModules FAILED %08x\n", hr);
return false;
}

DacpGetModuleData moduleData;
if (SUCCEEDED(hr = moduleData.Request(pClrDataModule.GetPtr())))
while (true)
{
TRACE("MODULE: %" PRIA PRIx64 " dyn %d inmem %d file %d pe %" PRIA PRIx64 " pdb %" PRIA PRIx64, (uint64_t)moduleData.LoadedPEAddress, moduleData.IsDynamic,
moduleData.IsInMemory, moduleData.IsFileLayout, (uint64_t)moduleData.PEAssembly, (uint64_t)moduleData.InMemoryPdbAddress);
ReleaseHolder<IXCLRDataModule> pClrDataModule;
if ((hr = m_pClrDataProcess->EnumModule(&enumModules, &pClrDataModule)) != S_OK) {
break;
}

if (!moduleData.IsDynamic && moduleData.LoadedPEAddress != 0)
// Skip any dynamic modules. The Request call below on some DACs crashes on dynamic modules.
ULONG32 flags;
if ((hr = pClrDataModule->GetFlags(&flags)) != S_OK) {
TRACE("MODULE: GetFlags FAILED %08x\n", hr);
continue;
}
if (flags & CLRDATA_MODULE_IS_DYNAMIC) {
TRACE("MODULE: Skipping dynamic module\n");
continue;
}

DacpGetModuleData moduleData;
if (SUCCEEDED(hr = moduleData.Request(pClrDataModule.GetPtr())))
{
ArrayHolder<WCHAR> wszUnicodeName = new WCHAR[MAX_LONGPATH + 1];
if (SUCCEEDED(hr = pClrDataModule->GetFileName(MAX_LONGPATH, nullptr, wszUnicodeName)))
TRACE("MODULE: %" PRIA PRIx64 " dyn %d inmem %d file %d pe %" PRIA PRIx64 " pdb %" PRIA PRIx64, (uint64_t)moduleData.LoadedPEAddress, moduleData.IsDynamic,
moduleData.IsInMemory, moduleData.IsFileLayout, (uint64_t)moduleData.PEAssembly, (uint64_t)moduleData.InMemoryPdbAddress);

if (!moduleData.IsDynamic && moduleData.LoadedPEAddress != 0)
{
std::string moduleName = FormatString("%S", wszUnicodeName.GetPtr());
ArrayHolder<WCHAR> wszUnicodeName = new WCHAR[MAX_LONGPATH + 1];
if (SUCCEEDED(hr = pClrDataModule->GetFileName(MAX_LONGPATH, nullptr, wszUnicodeName)))
{
std::string moduleName = FormatString("%S", wszUnicodeName.GetPtr());

// Change the module mapping name
ReplaceModuleMapping(moduleData.LoadedPEAddress, moduleData.LoadedPESize, moduleName);
// Change the module mapping name
ReplaceModuleMapping(moduleData.LoadedPEAddress, moduleData.LoadedPESize, moduleName);

// Add managed module info
AddModuleInfo(true, moduleData.LoadedPEAddress, pClrDataModule, moduleName);
// Add managed module info
AddModuleInfo(true, moduleData.LoadedPEAddress, pClrDataModule, moduleName);
}
else {
TRACE("\nModule.GetFileName FAILED %08x\n", hr);
}
}
else {
TRACE("\nModule.GetFileName FAILED %08x\n", hr);
TRACE("\n");
}
}
else {
TRACE("\n");
TRACE("moduleData.Request FAILED %08x\n", hr);
}
}
else {
TRACE("moduleData.Request FAILED %08x\n", hr);
}
}

if (enumModules != 0) {
pClrDataProcess->EndEnumModules(enumModules);
if (enumModules != 0) {
m_pClrDataProcess->EndEnumModules(enumModules);
}
TRACE("EnumerateManagedModules: Module enumeration FINISHED\n");
}

return result;
return true;
}

//
// Unwind all the native threads to ensure that the dwarf unwind info is added to the core dump.
//
bool
CrashInfo::UnwindAllThreads(IXCLRDataProcess* pClrDataProcess)
CrashInfo::UnwindAllThreads()
{
ReleaseHolder<ISOSDacInterface> pSos = nullptr;
if (pClrDataProcess != nullptr) {
pClrDataProcess->QueryInterface(__uuidof(ISOSDacInterface), (void**)&pSos);
if (m_pClrDataProcess != nullptr) {
m_pClrDataProcess->QueryInterface(__uuidof(ISOSDacInterface), (void**)&pSos);
}
// For each native and managed thread
for (ThreadInfo* thread : m_threads)
{
if (!thread->UnwindThread(pClrDataProcess, pSos)) {
if (!thread->UnwindThread(m_pClrDataProcess, pSos)) {
return false;
}
}
Expand Down Expand Up @@ -827,5 +858,5 @@ FormatGuid(const GUID* guid)
{
uint8_t* bytes = (uint8_t*)guid;
return FormatString("%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x",
bytes[3], bytes[2], bytes[1], bytes[0], bytes[5], bytes[4], bytes[7], bytes[6], bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]);
bytes[3], bytes[2], bytes[1], bytes[0], bytes[5], bytes[4], bytes[7], bytes[6], bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]);
}
9 changes: 6 additions & 3 deletions src/coreclr/debug/createdump/crashinfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ class CrashInfo : public ICLRDataEnumMemoryRegionsCallback,
pid_t m_ppid; // parent pid
pid_t m_tgid; // process group
HMODULE m_hdac; // dac module handle when loaded
ICLRDataEnumMemoryRegions* m_pClrDataEnumRegions; // dac enumerate memory interface instance
IXCLRDataProcess* m_pClrDataProcess; // dac process interface instance
bool m_gatherFrames; // if true, add the native and managed stack frames to the thread info
pid_t m_crashThread; // crashing thread id or 0 if none
uint32_t m_signal; // crash signal code or 0 if none
Expand Down Expand Up @@ -84,6 +86,7 @@ class CrashInfo : public ICLRDataEnumMemoryRegionsCallback,
void CleanupAndResumeProcess();
bool EnumerateAndSuspendThreads();
bool GatherCrashInfo(MINIDUMP_TYPE minidumpType);
bool EnumerateMemoryRegionsWithDAC(MINIDUMP_TYPE minidumpType);
bool ReadMemory(void* address, void* buffer, size_t size); // read memory and add to dump
bool ReadProcessMemory(void* address, void* buffer, size_t size, size_t* read); // read raw memory
uint64_t GetBaseAddressFromAddress(uint64_t address);
Expand Down Expand Up @@ -137,9 +140,9 @@ class CrashInfo : public ICLRDataEnumMemoryRegionsCallback,
void VisitProgramHeader(uint64_t loadbias, uint64_t baseAddress, ElfW(Phdr)* phdr);
bool EnumerateModuleMappings();
#endif
bool EnumerateMemoryRegionsWithDAC(MINIDUMP_TYPE minidumpType);
bool EnumerateManagedModules(IXCLRDataProcess* pClrDataProcess);
bool UnwindAllThreads(IXCLRDataProcess* pClrDataProcess);
bool InitializeDAC();
bool EnumerateManagedModules();
bool UnwindAllThreads();
void ReplaceModuleMapping(CLRDATA_ADDRESS baseAddress, ULONG64 size, const std::string& pszName);
void InsertMemoryBackedRegion(const MemoryRegion& region);
void InsertMemoryRegion(const MemoryRegion& region);
Expand Down
7 changes: 6 additions & 1 deletion src/coreclr/debug/createdump/createdumpunix.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,12 @@ CreateDump(const char* dumpPathTemplate, int pid, const char* dumpType, MINIDUMP
CrashReportWriter crashReportWriter(*crashInfo);
crashReportWriter.WriteCrashReport(dumpPath);
}
printf("Writing %s to file %s\n", dumpType, dumpPath.c_str());
// Gather all the useful memory regions from the DAC
if (!crashInfo->EnumerateMemoryRegionsWithDAC(minidumpType))
{
goto exit;
}
fprintf(stdout, "Writing %s to file %s\n", dumpType, dumpPath.c_str());

// Write the actual dump file
if (!dumpWriter.OpenDump(dumpPath.c_str()))
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/debug/daccess/enummem.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1597,6 +1597,7 @@ HRESULT ClrDataAccess::EnumMemoryRegionsWorkerSkinny(IN CLRDataEnumMemoryFlags f
//
// collect CLR static
CATCH_ALL_EXCEPT_RETHROW_COR_E_OPERATIONCANCELLED( status = EnumMemCLRStatic(flags); )
CATCH_ALL_EXCEPT_RETHROW_COR_E_OPERATIONCANCELLED( status = EnumMemCLRHeapCrticalStatic(flags); );

// Dump AppDomain-specific info needed for MiniDumpNormal.
CATCH_ALL_EXCEPT_RETHROW_COR_E_OPERATIONCANCELLED( status = EnumMemDumpAppDomainInfo(flags); )
Expand Down
7 changes: 5 additions & 2 deletions src/coreclr/debug/daccess/request_svr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,11 @@ ClrDataAccess::ServerGCHeapDetails(CLRDATA_ADDRESS heapAddr, DacpGcHeapDetails *

detailsData->lowest_address = PTR_CDADDR(g_lowest_address);
detailsData->highest_address = PTR_CDADDR(g_highest_address);
detailsData->current_c_gc_state = (CLRDATA_ADDRESS)*g_gcDacGlobals->current_c_gc_state;

detailsData->current_c_gc_state = c_gc_state_free;
if (g_gcDacGlobals->current_c_gc_state != NULL)
{
detailsData->current_c_gc_state = (CLRDATA_ADDRESS)*g_gcDacGlobals->current_c_gc_state;
Copy link
Member

@hoyosjs hoyosjs Oct 10, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we try to poison this in the case the pointer is null? And did you find a case where this was true? It should only be true if BACKGROUND_GC is not true.

Copy link
Contributor Author

@mikem8361 mikem8361 Oct 10, 2021 •
edited by cshung
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I attached to the WebApp debuggee process from the SOS tests (they were segfault'ing on this line) and the g_gcDacGlobals pointer was null. I'm not sure how this is happening unless BACKGROUND_GC is true like you said. The actual GC variable (gc_heap::current_c_gc_state) it is suppose to be pointing to was 2 (c_gc_state_free). I probably should set it to c_gc_state_free if it is null.

/cc: @Maoni0 @cshung do you have any clue why g_dacDacGlobals->current_c_gc_state isn't initialized?

Copy link
Contributor

@cshung cshung Oct 10, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like I accidentally moved that into !MULTIPLE_HEAP so it fails for server GC. It should work now after I pushed the fix.

Copy link
Member

@janvorli janvorli Oct 11, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the detailsData->current_c_gc_state initialized to NULL somewhere upstream so that it doesn't end up having a random valueif the condition is false?

Copy link
Contributor Author

@mikem8361 mikem8361 Oct 11, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I'm going to push a change that initializes it to c_gc_state_free.

}
// now get information specific to this heap (server mode gives us several heaps; we're getting
// information about only one of them.
detailsData->alloc_allocated = (CLRDATA_ADDRESS)pHeap->alloc_allocated;
Expand Down
7 changes: 5 additions & 2 deletions src/coreclr/gc/gc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46708,11 +46708,15 @@ void PopulateDacVars(GcDacVars *gcDacVars)
gcDacVars->generation_size = sizeof(generation);
gcDacVars->total_generation_count = total_generation_count;
gcDacVars->max_gen = &g_max_generation;
#ifdef BACKGROUND_GC
gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
#else //BACKGROUND_GC
gcDacVars->current_c_gc_state = 0;
#endif //BACKGROUND_GC
#ifndef MULTIPLE_HEAPS
gcDacVars->ephemeral_heap_segment = reinterpret_cast<dac_heap_segment**>(&gc_heap::ephemeral_heap_segment);
#ifdef BACKGROUND_GC
gcDacVars->mark_array = &gc_heap::mark_array;
gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
gcDacVars->background_saved_lowest_address = &gc_heap::background_saved_lowest_address;
gcDacVars->background_saved_highest_address = &gc_heap::background_saved_highest_address;
gcDacVars->next_sweep_obj = &gc_heap::next_sweep_obj;
Expand All @@ -46725,7 +46729,6 @@ void PopulateDacVars(GcDacVars *gcDacVars)
#endif //USE_REGIONS
#else //BACKGROUND_GC
gcDacVars->mark_array = 0;
gcDacVars->current_c_gc_state = 0;
gcDacVars->background_saved_lowest_address = 0;
gcDacVars->background_saved_highest_address = 0;
gcDacVars->next_sweep_obj = 0;
Expand Down