Merge branch 'main' into wasm_load_crypto_asset

# Conflicts:
#	src/mono/wasm/runtime/crypto-worker.ts
#	src/mono/wasm/runtime/workers/dotnet-crypto-worker.js

docs/design/coreclr/botr/xplat-minidump-generation.md

@@ -56,13 +56,18 @@ NOTE: Core dump generation in docker containers require the ptrace capability (-
 
 Any configuration or policy is set with environment variables which are passed as options to the _createdump_ utility.
 
+Starting with .NET 7.0 or greater the below environment variables can be prefixed with DOTNET_ or COMPlus_.
+
 Environment variables supported:
 
 - `COMPlus_DbgEnableMiniDump`: if set to "1", enables this core dump generation. The default is NOT to generate a dump.
 - `COMPlus_DbgMiniDumpType`: See below. Default is "2" MiniDumpWithPrivateReadWriteMemory.
 - `COMPlus_DbgMiniDumpName`: if set, use as the template to create the dump path and file name. See "Dump name formatting" for how the dump name can be formatted. The default is _/tmp/coredump.%p_.
 - `COMPlus_CreateDumpDiagnostics`: if set to "1", enables the _createdump_ utilities diagnostic messages (TRACE macro).
+- `COMPlus_CreateDumpVerboseDiagnostics`: if set to "1", enables the _createdump_ utilities verbose diagnostic messages (TRACE_VERBOSE macro).
+- `COMPlus_CreateDumpLogToFile`: if set, it is the path of the file to write the _createdump_ diagnostic messages.
 - `COMPlus_EnableCrashReport`: In .NET 6.0 or greater, if set to "1", createdump also generates a json formatted crash report which includes information about the threads and stack frames of the crashing application. The crash report name is the dump path/name with _.crashreport.json_ appended.
+- `COMPlus_EnableCrashReportOnly`: In .NET 7.0 or greater, same as COMPlus_EnableCrashReport except the core dump is not generated.
 
 COMPlus_DbgMiniDumpType values:
 
@@ -93,9 +98,12 @@ createdump [options] pid
 -u, --full - create full core dump.
 -d, --diag - enable diagnostic messages.
 -v, --verbose - enable verbose diagnostic messages.
---crashreport - write crash report file.
+-l, --logtofile - file path and name to log diagnostic messages.
+--crashreport - write crash report file (dump file path + .crashreport.json).
+--crashreportonly - write crash report file only (no dump).
 --crashthread <id> - the thread id of the crashing thread.
 --signal <code> - the signal code of the crash.
+--singlefile - enable single-file app check.
 ```
 
 **Dump name formatting**

eng/Versions.props

@@ -100,6 +100,7 @@
     <SystemMemoryVersion>4.5.4</SystemMemoryVersion>
     <SystemNumericsVectorsVersion>4.5.0</SystemNumericsVectorsVersion>
     <SystemReflectionMetadataVersion>6.0.0</SystemReflectionMetadataVersion>
+    <SystemReflectionMetadataLoadContextVersion>4.7.1</SystemReflectionMetadataLoadContextVersion>
     <SystemReflectionEmitVersion>4.7.0</SystemReflectionEmitVersion>
     <SystemReflectionEmitILGenerationVersion>4.7.0</SystemReflectionEmitILGenerationVersion>
     <SystemReflectionEmitLightweightVersion>4.7.0</SystemReflectionEmitLightweightVersion>

eng/pipelines/coreclr/templates/perf-send-to-helix.yml

@@ -21,7 +21,7 @@ parameters:
   condition: succeeded()                 # optional -- condition for step to execute; defaults to succeeded()
   continueOnError: false                 # optional -- determines whether to continue the build if the step errors; defaults to false
   osGroup: ''                            # required -- operating system for the job
-            
+  HelixPrereqCommands: ''
 
 steps:
 - template: /eng/pipelines/common/templates/runtimes/send-to-helix-inner-step.yml
@@ -40,6 +40,7 @@ steps:
       HelixAccessToken: ${{ parameters.HelixAccessToken }}
       HelixPreCommands: ${{ parameters.HelixPreCommands }}
       HelixPostCommands: ${{ parameters.HelixPostCommands }}
+      HelixPrereqCommands: ${{ parameters.HelixPrereqCommands }}
       WorkItemDirectory: ${{ parameters.WorkItemDirectory }}
       WorkItemTimeout: ${{ parameters.WorkItemTimeout }}
       CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }}

eng/pipelines/coreclr/templates/run-performance-job.yml

@@ -58,9 +58,35 @@ jobs:
 
     - IsInternal: ''
     - HelixApiAccessToken: ''
-    - HelixPreCommandStemWindows: 'set ORIGPYPATH=%PYTHONPATH%;py -m pip install -U pip;py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install -U pip;py -3 -m pip install azure.storage.blob==12.0.0;py -3 -m pip install azure.storage.queue==12.0.0;set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"'
-    - HelixPreCommandStemLinux: 'export ORIGPYPATH=$PYTHONPATH;export CRYPTOGRAPHY_ALLOW_OPENSSL_102=true;python3 -m pip install -U pip;sudo apt-get -y install python3-venv;python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.0.0;pip3 install azure.storage.queue==12.0.0;sudo apt-get update;sudo apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates;curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -;sudo apt-get -y install nodejs;sudo apt-get -y install npm;npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g;$HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 --engines=v8,javascriptcore;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
-    - HelixPreCommandStemMusl: 'export ORIGPYPATH=$PYTHONPATH;sudo apk add icu-libs krb5-libs libgcc libintl libssl1.1 libstdc++ zlib cargo;sudo apk add libgdiplus --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing; python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.7.1;pip3 install azure.storage.queue==12.1.5;export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
+    - ${{ if and(eq(parameters.runtimeType, 'wasm'), in(variables['Build.Reason'], 'PullRequest')) }}:
+      - HelixPerfUploadTokenValue: ''
+    - ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), ne(parameters.osGroup, 'windows')) }}:
+      - HelixPerfUploadTokenValue: '$(PerfCommandUploadTokenLinux)'
+    - ${{ if and(notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.osGroup, 'windows')) }}:
+      - HelixPerfUploadTokenValue: '$(PerfCommandUploadToken)'
+    - HelixPreCommandStemWindows: 'set ORIGPYPATH=%PYTHONPATH%;py -m pip install -U pip;py -3 -m venv %HELIX_WORKITEM_PAYLOAD%\.venv;call %HELIX_WORKITEM_PAYLOAD%\.venv\Scripts\activate.bat;set PYTHONPATH=;py -3 -m pip install -U pip;py -3 -m pip install azure.storage.blob==12.0.0;py -3 -m pip install azure.storage.queue==12.0.0;set "PERFLAB_UPLOAD_TOKEN=$(HelixPerfUploadTokenValue)"'
+    - HelixPreCommandStemLinux: >-
+        export ORIGPYPATH=$PYTHONPATH
+        export CRYPTOGRAPHY_ALLOW_OPENSSL_102=true;
+        echo "** Installing prerequistes **";
+        python3 -m pip install -U pip &&
+        sudo apt-get -y install python3-venv &&
+        python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv &&
+        ls -l $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate &&
+        export PYTHONPATH= &&
+        python3 -m pip install -U pip &&
+        pip3 install azure.storage.blob==12.0.0 &&
+        pip3 install azure.storage.queue==12.0.0 &&
+        sudo apt-get update &&
+        sudo apt -y install curl dirmngr apt-transport-https lsb-release ca-certificates &&
+        curl -fsSL https://deb.nodesource.com/setup_18.x | sudo -E bash - &&
+        sudo apt-get -y install nodejs &&
+        npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g &&
+        $HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 --engines=v8,javascriptcore &&
+        export PERFLAB_UPLOAD_TOKEN="$(HelixPerfUploadTokenValue)" &&
+        export PERF_PREREQS_INSTALLED=1;
+        test "x$PERF_PREREQS_INSTALLED" = "x1" || echo "** Error: Failed to install prerequites **"
+    - HelixPreCommandStemMusl: 'export ORIGPYPATH=$PYTHONPATH;sudo apk add icu-libs krb5-libs libgcc libintl libssl1.1 libstdc++ zlib cargo;sudo apk add libgdiplus --repository http://dl-cdn.alpinelinux.org/alpine/edge/testing; python3 -m venv $HELIX_WORKITEM_PAYLOAD/.venv;source $HELIX_WORKITEM_PAYLOAD/.venv/bin/activate;export PYTHONPATH=;python3 -m pip install -U pip;pip3 install azure.storage.blob==12.7.1;pip3 install azure.storage.queue==12.1.5;export PERFLAB_UPLOAD_TOKEN="$(HelixPerfUploadTokenValue)"'
     - ExtraMSBuildLogsWindows: 'set MSBUILDDEBUGCOMM=1;set "MSBUILDDEBUGPATH=%HELIX_WORKITEM_UPLOAD_ROOT%"'
     - ExtraMSBuildLogsLinux: 'export MSBUILDDEBUGCOMM=1;export "MSBUILDDEBUGPATH=$HELIX_WORKITEM_UPLOAD_ROOT"'
     - HelixPreCommand: ''
@@ -89,7 +115,7 @@ jobs:
         - HelixPreCommand: $(HelixPreCommandStemWindows);$(ExtraMSBuildLogsWindows)
         - HelixPostCommand: 'export PYTHONPATH=$ORIGPYPATH;${{ parameters.collectHelixLogsScript }}'
       - ${{ if ne(parameters.osGroup, 'windows') }}:
-        - HelixPreCommand: $(HelixPreCommandStemLinux);$(ExtraMSBuildLogsLinux);npm install --prefix $HELIX_WORKITEM_PAYLOAD jsvu -g;$HELIX_WORKITEM_PAYLOAD/bin/jsvu --os=linux64 --engines=v8,javascriptcore
+        - HelixPreCommand: $(HelixPreCommandStemLinux);$(ExtraMSBuildLogsLinux);
         - HelixPostCommand: 'export PYTHONPATH=$ORIGPYPATH;${{ parameters.collectHelixLogsScript }}'
 
     - ${{ if and(eq(parameters.codeGenType, 'Interpreter'), eq(parameters.runtimeType, 'mono')) }}:

eng/pipelines/coreclr/templates/run-scenarios-job.yml

@@ -110,7 +110,7 @@ jobs:
     - ${{ if ne(parameters.runtimeType, 'wasm') }}:
       - ExtraSetupArguments: --install-dir $(PayloadDirectory)/dotnet
     - ${{ if and(eq(parameters.runtimeType, 'wasm'), in(variables['Build.Reason'], 'PullRequest')) }}:
-      - ExtraSetupArguments: --not-in-lab
+      - ExtraSetupArguments: ''
 
     workspace:
       clean: all

eng/testing/performance/microbenchmarks.proj

@@ -131,7 +131,12 @@
     <HelixWorkItem Include="@(Partition)">
       <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
       <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</PreCommands>
-      <Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
+      <Command>
+        if [ "x$PERF_PREREQS_INSTALLED" = "x1" ]; then
+          $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)";
+        else
+          echo "\n\n** Error: Failed to install prerequisites **\n\n"; export _commandExitCode=1;
+        fi</Command>
       <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(PERFLAB_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand)</PostCommands>
       <Timeout>$(WorkItemTimeout)</Timeout>
     </HelixWorkItem>
@@ -141,7 +146,12 @@
     <HelixWorkItem Include="$(BuildConfig).WorkItem">
       <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
       <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)"</PreCommands>
-      <Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)"</Command>
+      <Command>
+        if [ "x$PERF_PREREQS_INSTALLED" = "x1" ]; then
+          $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)";
+        else
+          echo "\n\n** Error: Failed to install prerequisites **\n\n"; export _commandExitCode=1;
+        fi</Command>
       <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(PERFLAB_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults)</PostCommands>
       <Timeout>4:00</Timeout>
     </HelixWorkItem>

eng/testing/performance/performance-setup.sh

@@ -310,6 +310,10 @@ fi
 common_setup_arguments="--channel $cleaned_branch_name --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture"
 setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"
 
+if [[ "$internal" != true ]]; then
+    setup_arguments="$setup_arguments --not-in-lab"
+fi
+
 if [[ "$run_from_perf_repo" == true ]]; then
     payload_directory=
     workitem_directory=$source_directory

src/coreclr/debug/createdump/crashinfomac.cpp

@@ -463,6 +463,10 @@ ModuleInfo::LoadModule()
                 {
                     g_image_infos = (const struct dyld_all_image_infos*)dyld_info.all_image_info_addr;
                 }
+                else
+                {
+                    TRACE("LoadModule: task_info(self) FAILED %x %s\n", result, mach_error_string(result));
+                }
             }
             if (g_image_infos != nullptr)
             {
@@ -475,7 +479,15 @@ ModuleInfo::LoadModule()
                         break;
                     }
                 }
+                if (m_localBaseAddress == 0)
+                {
+                    TRACE("LoadModule: local base address not found for %s\n", m_moduleName.c_str());
+                }
             }
         }
+        else
+        {
+            TRACE("LoadModule: dlopen(%s) FAILED %d %s\n", m_moduleName.c_str(), errno, strerror(errno));
+        }
     }
 }

src/coreclr/debug/createdump/createdump.h

@@ -107,7 +107,7 @@ typedef int T_CONTEXT;
 #endif
 
 extern bool FormatDumpName(std::string& name, const char* pattern, const char* exename, int pid);
-extern bool CreateDump(const char* dumpPathTemplate, int pid, const char* dumpType, MINIDUMP_TYPE minidumpType, bool crashReport, int crashThread, int signal);
+extern bool CreateDump(const char* dumpPathTemplate, int pid, const char* dumpType, MINIDUMP_TYPE minidumpType, bool createDump, bool crashReport, int crashThread, int signal);
 
 extern std::string GetLastErrorString();
 extern void printf_status(const char* format, ...);

src/coreclr/debug/createdump/createdumpunix.cpp

@@ -11,7 +11,7 @@ long g_pageSize = 0;
 // The Linux/MacOS create dump code
 //
 bool
-CreateDump(const char* dumpPathTemplate, int pid, const char* dumpType, MINIDUMP_TYPE minidumpType, bool crashReport, int crashThread, int signal)
+CreateDump(const char* dumpPathTemplate, int pid, const char* dumpType, MINIDUMP_TYPE minidumpType, bool createDump, bool crashReport, int crashThread, int signal)
 {
     ReleaseHolder<CrashInfo> crashInfo = new CrashInfo(pid, crashReport, crashThread, signal);
     DumpWriter dumpWriter(*crashInfo);
@@ -57,28 +57,31 @@ CreateDump(const char* dumpPathTemplate, int pid, const char* dumpType, MINIDUMP
         CrashReportWriter crashReportWriter(*crashInfo);
         crashReportWriter.WriteCrashReport(dumpPath);
     }
-    // Gather all the useful memory regions from the DAC
-    if (!crashInfo->EnumerateMemoryRegionsWithDAC(minidumpType))
+    if (createDump)
     {
-        goto exit;
-    }
-    // Join all adjacent memory regions
-    crashInfo->CombineMemoryRegions();
-
-    printf_status("Writing %s to file %s\n", dumpType, dumpPath.c_str());
+        // Gather all the useful memory regions from the DAC
+        if (!crashInfo->EnumerateMemoryRegionsWithDAC(minidumpType))
+        {
+            goto exit;
+        }
+        // Join all adjacent memory regions
+        crashInfo->CombineMemoryRegions();
+
+        printf_status("Writing %s to file %s\n", dumpType, dumpPath.c_str());
 
-    // Write the actual dump file
-    if (!dumpWriter.OpenDump(dumpPath.c_str()))
-    {
-        goto exit;
-    }
-    if (!dumpWriter.WriteDump())
-    {
-        printf_error("Writing dump FAILED\n");
+        // Write the actual dump file
+        if (!dumpWriter.OpenDump(dumpPath.c_str()))
+        {
+            goto exit;
+        }
+        if (!dumpWriter.WriteDump())
+        {
+            printf_error("Writing dump FAILED\n");
 
-        // Delete the partial dump file on error
-        remove(dumpPath.c_str());
-        goto exit;
+            // Delete the partial dump file on error
+            remove(dumpPath.c_str());
+            goto exit;
+        }
     }
     result = true;
 exit:

src/coreclr/debug/createdump/createdumpwindows.cpp

@@ -18,12 +18,15 @@ typedef struct _PROCESS_BASIC_INFORMATION_ {
 // The Windows create dump code
 //
 bool
-CreateDump(const char* dumpPathTemplate, int pid, const char* dumpType, MINIDUMP_TYPE minidumpType, bool crashReport, int crashThread, int signal)
+CreateDump(const char* dumpPathTemplate, int pid, const char* dumpType, MINIDUMP_TYPE minidumpType, bool createDump, bool crashReport, int crashThread, int signal)
 {
     HANDLE hFile = INVALID_HANDLE_VALUE;
     HANDLE hProcess = NULL;
     bool result = false;
 
+    _ASSERTE(createDump);
+    _ASSERTE(!crashReport);
+
     ArrayHolder<char> pszName = new char[MAX_LONGPATH + 1];
     std::string dumpPath;
 

src/coreclr/debug/createdump/main.cpp

@@ -30,6 +30,7 @@ const char* g_help = "createdump [options]\n"
 "-l, --logtofile - file path and name to log diagnostic messages.\n"
 #ifdef HOST_UNIX
 "--crashreport - write crash report file (dump file path + .crashreport.json).\n"
+"--crashreportonly - write crash report file only (no dump).\n"
 "--crashthread <id> - the thread id of the crashing thread.\n"
 "--signal <code> - the signal code of the crash.\n"
 "--singlefile - enable single-file app check.\n"
@@ -64,6 +65,7 @@ int __cdecl main(const int argc, const char* argv[])
     const char* dumpType = "minidump with heap";
     const char* dumpPathTemplate = nullptr;
     bool crashReport = false;
+    bool createDump = true;
     bool help = false;
     int signal = 0;
     int crashThread = 0;
@@ -136,6 +138,11 @@ int __cdecl main(const int argc, const char* argv[])
             {
                 crashReport = true;
             }
+            else if (strcmp(*argv, "--crashreportonly") == 0)
+            {
+                crashReport = true;
+                createDump = false;
+            }
             else if (strcmp(*argv, "--crashthread") == 0)
             {
                 crashThread = atoi(*++argv);
@@ -221,7 +228,7 @@ int __cdecl main(const int argc, const char* argv[])
         dumpPathTemplate = tmpPath;
     }
 
-    if (CreateDump(dumpPathTemplate, pid, dumpType, minidumpType, crashReport, crashThread, signal))
+    if (CreateDump(dumpPathTemplate, pid, dumpType, minidumpType, createDump, crashReport, crashThread, signal))
     {
         printf_status("Dump successfully written in %llums\n", GetTimeStamp() - g_startTime);
     }