[release/6.0] Build improvements based on feedback

eng/Versions.props

@@ -169,7 +169,7 @@
     <SQLitePCLRawbundle_greenVersion>2.0.4</SQLitePCLRawbundle_greenVersion>
     <MoqVersion>4.12.0</MoqVersion>
     <FsCheckVersion>2.14.3</FsCheckVersion>
-    <SdkVersionForWorkloadTesting>6.0.100-rc.2.21463.12</SdkVersionForWorkloadTesting>
+    <SdkVersionForWorkloadTesting>6.0.100-rtm.21480.21</SdkVersionForWorkloadTesting>
     <!-- Docs -->
     <MicrosoftPrivateIntellisenseVersion>6.0.0-preview-20210916.1</MicrosoftPrivateIntellisenseVersion>
     <!-- ILLink -->

eng/testing/scenarios/BuildWasmAppsJobsList.txt

@@ -11,6 +11,7 @@ Wasm.Build.Tests.LocalEMSDKTests
 Wasm.Build.Tests.MainWithArgsTests
 Wasm.Build.Tests.NativeBuildTests
 Wasm.Build.Tests.NativeLibraryTests
+Wasm.Build.Tests.PInvokeTableGeneratorTests
 Wasm.Build.Tests.RebuildTests
 Wasm.Build.Tests.SatelliteAssembliesTests
 Wasm.Build.Tests.WasmBuildAppTest

src/libraries/workloads-testing.targets

@@ -16,7 +16,6 @@
     </ItemGroup>
 
     <Copy SourceFiles="@(_SourceFiles)" DestinationFolder="$(SdkWithWorkloadForTestingPath)\%(_SourceFiles.RecursiveDir)" />
-    <Copy SourceFiles="$(MonoProjectRoot)\wasm\BlazorOverwrite.targets" DestinationFiles="$(SdkWithWorkloadForTestingPath)\sdk\$(SdkVersionForWorkloadTesting)\Sdks\Microsoft.NET.Sdk.BlazorWebAssembly\targets\Microsoft.NET.Sdk.BlazorWebAssembly.6_0.targets" />
 
     <WriteLinesToFile File="$(SdkWithWorkloadStampPath)" Lines="" Overwrite="true" />
   </Target>
@@ -41,7 +40,6 @@
     <Exec Condition="$([MSBuild]::IsOSPlatform('windows'))"
           Command='powershell -ExecutionPolicy ByPass -NoProfile -command "&amp; $(_DotNetInstallScriptPath) -InstallDir $(SdkWithNoWorkloadForTestingPath) -Version $(SdkVersionForWorkloadTesting)"' />
 
-    <Copy SourceFiles="$(MonoProjectRoot)\wasm\BlazorOverwrite.targets" DestinationFiles="$(SdkWithNoWorkloadForTestingPath)\sdk\$(SdkVersionForWorkloadTesting)\Sdks\Microsoft.NET.Sdk.BlazorWebAssembly\targets\Microsoft.NET.Sdk.BlazorWebAssembly.6_0.targets" />
     <WriteLinesToFile File="$(SdkWithNoWorkloadStampPath)" Lines="" Overwrite="true" />
   </Target>
 

src/mono/wasm/build/WasmApp.Native.targets

@@ -143,6 +143,7 @@
     <PropertyGroup>
       <_MonoAotCrossCompilerPath>@(MonoAotCrossCompiler->WithMetadataValue('RuntimeIdentifier','browser-wasm'))</_MonoAotCrossCompilerPath>
       <_EmccDefaultFlagsRsp>$([MSBuild]::NormalizePath($(_WasmRuntimePackSrcDir), 'emcc-default.rsp'))</_EmccDefaultFlagsRsp>
+      <WasmNativeStrip Condition="'$(WasmNativeStrip)' == '' and '$(Configuration)' == 'Debug' and '$(WasmBuildingForNestedPublish)' != 'true'">false</WasmNativeStrip>
       <WasmNativeStrip Condition="'$(WasmNativeStrip)' == ''">true</WasmNativeStrip>
       <WasmNativeDebugSymbols Condition="'$(WasmNativeDebugSymbols)' == ''">true</WasmNativeDebugSymbols>
       <WasmLinkIcalls Condition="'$(WasmLinkIcalls)' == ''">$(WasmBuildNative)</WasmLinkIcalls>
@@ -156,8 +157,7 @@
 
       <_EmccAssertionLevelDefault>0</_EmccAssertionLevelDefault>
       <_EmccOptimizationFlagDefault Condition="'$(_WasmDevel)' == 'true'">-O0 -s ASSERTIONS=$(_EmccAssertionLevelDefault)</_EmccOptimizationFlagDefault>
-      <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(OS)' != 'Windows_NT' and '$(Configuration)' == 'Debug'">-Os</_EmccOptimizationFlagDefault>
-      <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' != 'Debug'">-Oz</_EmccOptimizationFlagDefault>
+      <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' == 'Debug' and '$(WasmBuildingForNestedPublish)' != 'true'">-O1</_EmccOptimizationFlagDefault>
       <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == ''">-Oz</_EmccOptimizationFlagDefault>
 
       <EmccCompileOptimizationFlag Condition="'$(EmccCompileOptimizationFlag)' == ''">$(_EmccOptimizationFlagDefault)</EmccCompileOptimizationFlag>
@@ -206,6 +206,9 @@
       <_EmccLDFlags Include="@(_EmccCommonFlags)" />
       <_EmccLDFlags Include="-s TOTAL_MEMORY=$(EmccTotalMemory)" />
 
+      <!-- ILLinker should have removed unused imports, so error for Publish -->
+      <_EmccLDFlags Include="-s ERROR_ON_UNDEFINED_SYMBOLS=0" Condition="'$(WasmBuildingForNestedPublish)' != 'true'" />
+
       <_DriverCDependencies Include="$(_WasmPInvokeHPath);$(_WasmICallTablePath)" />
       <_DriverCDependencies Include="$(_DriverGenCPath)" Condition="'$(_DriverGenCNeeded)' == 'true'" />
 
@@ -296,7 +299,8 @@
           Inputs="@(_BitcodeFile);$(_EmccDefaultFlagsRsp);$(_EmccCompileBitcodeRsp)"
           Outputs="@(_BitcodeFile->'%(ObjectFile)')"
           Condition="'$(_WasmShouldAOT)' == 'true' and @(_BitcodeFile->Count()) > 0"
-          DependsOnTargets="_WasmWriteRspForCompilingBitcode">
+          DependsOnTargets="_WasmWriteRspForCompilingBitcode"
+          Returns="@(FileWrites)">
 
     <ItemGroup>
       <_BitCodeFile Dependencies="%(_BitCodeFile.Dependencies);$(_EmccDefaultFlagsRsp);$(_EmccCompileBitcodeRsp)" />
@@ -362,7 +366,8 @@
   <Target Name="_WasmLinkDotNet"
           Inputs="@(_WasmLinkDependencies);$(_EmccDefaultFlagsRsp);$(_EmccLinkRsp)"
           Outputs="$(_WasmIntermediateOutputPath)dotnet.js;$(_WasmIntermediateOutputPath)dotnet.wasm"
-          DependsOnTargets="_WasmSelectRuntimeComponentsForLinking;_WasmCompileAssemblyBitCodeFilesForAOT;_WasmWriteRspFilesForLinking">
+          DependsOnTargets="_WasmSelectRuntimeComponentsForLinking;_WasmCompileAssemblyBitCodeFilesForAOT;_WasmWriteRspFilesForLinking"
+          Returns="@(FileWrites)" >
 
     <Message Text="Linking with emcc. This may take a while ..." Importance="High" />
     <Message Text="Running emcc with @(_EmccLinkStepArgs->'%(Identity)', ' ')" Importance="Low" />

src/tasks/AotCompilerTask/MonoAOTCompiler.cs

@@ -325,6 +325,13 @@ private bool ProcessAndValidateArguments()
             throw new LogAsErrorException($"'{nameof(OutputType)}=Library' can not be used with '{nameof(UseStaticLinking)}=true'.");
         }
 
+        foreach (var asmItem in Assemblies)
+        {
+            string? fullPath = asmItem.GetMetadata("FullPath");
+            if (!File.Exists(fullPath))
+                throw new LogAsErrorException($"Could not find {fullPath} to AOT");
+        }
+
         return !Log.HasLoggedErrors;
     }
 
@@ -339,6 +346,12 @@ public override bool Execute()
             Log.LogError(laee.Message);
             return false;
         }
+        finally
+        {
+            if (_cache != null && _cache.Save(CacheFilePath!))
+                _fileWrites.Add(CacheFilePath!);
+            FileWrites = _fileWrites.ToArray();
+        }
     }
 
     private bool ExecuteInternal()
@@ -347,6 +360,7 @@ private bool ExecuteInternal()
             return false;
 
         _assembliesToCompile = EnsureAndGetAssembliesInTheSameDir(Assemblies);
+        _assembliesToCompile = FilterAssemblies(_assembliesToCompile);
 
         if (!string.IsNullOrEmpty(AotModulesTablePath) && !GenerateAotModulesTable(_assembliesToCompile, Profilers, AotModulesTablePath))
             return false;
@@ -372,42 +386,28 @@ private bool ExecuteInternal()
         }
         else
         {
-            int allowedParallelism = Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount);
+            int allowedParallelism = DisableParallelAot ? 1 : Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount);
             if (BuildEngine is IBuildEngine9 be9)
                 allowedParallelism = be9.RequestCores(allowedParallelism);
 
-            if (DisableParallelAot || allowedParallelism == 1)
+            ParallelLoopResult result = Parallel.ForEach(
+                                            argsList,
+                                            new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism },
+                                            (args, state) => PrecompileLibraryParallel(args, state));
+
+            if (result.IsCompleted)
             {
-                foreach (var args in argsList)
-                {
-                    if (!PrecompileLibrarySerial(args))
-                        return !Log.HasLoggedErrors;
-                }
+                int numUnchanged = _totalNumAssemblies - _numCompiled;
+                if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies)
+                    Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies.");
             }
-            else
+            else if (!Log.HasLoggedErrors)
             {
-                ParallelLoopResult result = Parallel.ForEach(
-                                                        argsList,
-                                                        new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism },
-                                                        (args, state) => PrecompileLibraryParallel(args, state));
-
-                if (!result.IsCompleted)
-                {
-                    return false;
-                }
+                Log.LogError($"Precompiling failed due to unknown reasons. Check log for more info");
             }
-
-            int numUnchanged = _totalNumAssemblies - _numCompiled;
-            if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies)
-                Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies.");
         }
 
         CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, _assembliesToCompile).ToArray();
-
-        if (_cache.Save(CacheFilePath!))
-            _fileWrites.Add(CacheFilePath!);
-        FileWrites = _fileWrites.ToArray();
-
         return !Log.HasLoggedErrors;
     }
 
@@ -428,60 +428,68 @@ static bool IsNewerThanOutput(string inFile, string outFile)
                     (File.GetLastWriteTimeUtc(inFile) > File.GetLastWriteTimeUtc(outFile));
     }
 
-    private IList<ITaskItem> EnsureAndGetAssembliesInTheSameDir(ITaskItem[] originalAssemblies)
+    private IList<ITaskItem> FilterAssemblies(IEnumerable<ITaskItem> assemblies)
     {
         List<ITaskItem> filteredAssemblies = new();
-        string firstAsmDir = Path.GetDirectoryName(originalAssemblies[0].GetMetadata("FullPath")) ?? string.Empty;
-        bool allInSameDir = true;
-
-        foreach (var origAsm in originalAssemblies)
+        foreach (var asmItem in assemblies)
         {
-            if (allInSameDir && Path.GetDirectoryName(origAsm.GetMetadata("FullPath")) != firstAsmDir)
-                allInSameDir = false;
-
-            if (ShouldSkip(origAsm))
+            if (ShouldSkip(asmItem))
             {
                 if (parsedAotMode == MonoAotMode.LLVMOnly)
-                    throw new LogAsErrorException($"Building in AOTMode=LLVMonly is not compatible with excluding any assemblies for AOT. Excluded assembly: {origAsm.ItemSpec}");
+                    throw new LogAsErrorException($"Building in AOTMode=LLVMonly is not compatible with excluding any assemblies for AOT. Excluded assembly: {asmItem.ItemSpec}");
 
-                Log.LogMessage(MessageImportance.Low, $"Skipping {origAsm.ItemSpec} because it has %(AOT_InternalForceToInterpret)=true");
+                Log.LogMessage(MessageImportance.Low, $"Skipping {asmItem.ItemSpec} because it has %(AOT_InternalForceToInterpret)=true");
                 continue;
             }
 
-            filteredAssemblies.Add(origAsm);
+            string assemblyPath = asmItem.GetMetadata("FullPath");
+            using var assemblyFile = File.OpenRead(assemblyPath);
+            using PEReader reader = new(assemblyFile, PEStreamOptions.Default);
+            if (!reader.HasMetadata)
+            {
+                Log.LogWarning($"Skipping unmanaged {assemblyPath} for AOT");
+                continue;
+            }
+
+            filteredAssemblies.Add(asmItem);
         }
 
+        return filteredAssemblies;
+
+        static bool ShouldSkip(ITaskItem asmItem)
+            => bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip;
+    }
+
+    private IList<ITaskItem> EnsureAndGetAssembliesInTheSameDir(IList<ITaskItem> assemblies)
+    {
+        string firstAsmDir = Path.GetDirectoryName(assemblies.First().GetMetadata("FullPath")) ?? string.Empty;
+        bool allInSameDir = assemblies.All(asm => Path.GetDirectoryName(asm.GetMetadata("FullPath")) == firstAsmDir);
         if (allInSameDir)
-            return filteredAssemblies;
+            return assemblies;
 
         // Copy to aot-in
 
         string aotInPath = Path.Combine(IntermediateOutputPath, "aot-in");
         Directory.CreateDirectory(aotInPath);
 
         List<ITaskItem> newAssemblies = new();
-        foreach (var origAsm in originalAssemblies)
+        foreach (var asmItem in assemblies)
         {
-            string asmPath = origAsm.GetMetadata("FullPath");
+            string asmPath = asmItem.GetMetadata("FullPath");
             string newPath = Path.Combine(aotInPath, Path.GetFileName(asmPath));
 
             // FIXME: delete files not in originalAssemblies though
             // FIXME: or .. just delete the whole dir?
             if (Utils.CopyIfDifferent(asmPath, newPath, useHash: true))
                 Log.LogMessage(MessageImportance.Low, $"Copying {asmPath} to {newPath}");
+            _fileWrites.Add(newPath);
 
-            if (!ShouldSkip(origAsm))
-            {
-                ITaskItem newAsm = new TaskItem(newPath);
-                origAsm.CopyMetadataTo(newAsm);
-                newAssemblies.Add(newAsm);
-            }
+            ITaskItem newAsm = new TaskItem(newPath);
+            asmItem.CopyMetadataTo(newAsm);
+            newAssemblies.Add(newAsm);
         }
 
         return newAssemblies;
-
-        static bool ShouldSkip(ITaskItem asmItem)
-            => bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip;
     }
 
     private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, string? monoPaths)
@@ -773,28 +781,6 @@ private bool PrecompileLibrary(PrecompileArguments args)
         return true;
     }
 
-    private bool PrecompileLibrarySerial(PrecompileArguments args)
-    {
-        try
-        {
-            if (PrecompileLibrary(args))
-                return true;
-        }
-        catch (LogAsErrorException laee)
-        {
-            Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}");
-        }
-        catch (Exception ex)
-        {
-            if (Log.HasLoggedErrors)
-                Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}");
-            else
-                Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}");
-        }
-
-        return false;
-    }
-
     private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopState state)
     {
         try
@@ -804,14 +790,14 @@ private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopSta
         }
         catch (LogAsErrorException laee)
         {
-            Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}");
+            Log.LogError($"Precompiling failed for {args.AOTAssembly}: {laee.Message}");
         }
         catch (Exception ex)
         {
             if (Log.HasLoggedErrors)
                 Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}");
             else
-                Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}");
+                Log.LogError($"Precompiling failed for {args.AOTAssembly}: {ex}");
         }
 
         state.Break();
@@ -940,10 +926,8 @@ private static IList<ITaskItem> ConvertAssembliesDictToOrderedList(ConcurrentDic
         List<ITaskItem> outItems = new(originalAssemblies.Count);
         foreach (ITaskItem item in originalAssemblies)
         {
-            if (!dict.TryGetValue(item.GetMetadata("FullPath"), out ITaskItem? dictItem))
-                throw new LogAsErrorException($"Bug: Could not find item in the dict with key {item.ItemSpec}");
-
-            outItems.Add(dictItem);
+            if (dict.TryGetValue(item.GetMetadata("FullPath"), out ITaskItem? dictItem))
+                outItems.Add(dictItem);
         }
         return outItems;
     }
@@ -993,7 +977,7 @@ public FileCache(string? cacheFilePath, TaskLoggingHelper log)
         }
 
         _oldCache ??= new();
-        _newCache = new();
+        _newCache = new(_oldCache.FileHashes);
     }
 
     public bool ShouldCopy(ProxyFile proxyFile, [NotNullWhen(true)] out string? cause)
@@ -1110,6 +1094,10 @@ public enum MonoAotModulesTableLanguage
 
 internal class CompilerCache
 {
+    public CompilerCache() => FileHashes = new();
+    public CompilerCache(IDictionary<string, string> oldHashes)
+        => FileHashes = new(oldHashes);
+
     [JsonPropertyName("file_hashes")]
-    public ConcurrentDictionary<string, string> FileHashes { get; set; } = new();
+    public ConcurrentDictionary<string, string> FileHashes { get; set; }
 }