Skip to content

Commit 0bf8e5d

Browse files
radicallewing
andcommitted
[wasm] Build improvements based on feedback (dotnet#59391)
- Fix for a incremental build issue - Change native build defaults for `Debug` config, to make it faster (thanks @SteveSandersonMS, @mattleibow) - MonoAOTCompiler: - Skip unmanaged assemblies, and emit a warning - PInvokeTableGenerator: - Fix to not fail in presence of native variadic functions, and pinvokes with function pointers (eg. used by sqlite) Co-authored-by: Larry Ewing <lewing@microsoft.com> (cherry picked from commit 80f015d)
1 parent 229d49b commit 0bf8e5d

File tree

17 files changed

+488
-882
lines changed

17 files changed

+488
-882
lines changed

eng/Versions.props

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,7 @@
169169
<SQLitePCLRawbundle_greenVersion>2.0.4</SQLitePCLRawbundle_greenVersion>
170170
<MoqVersion>4.12.0</MoqVersion>
171171
<FsCheckVersion>2.14.3</FsCheckVersion>
172-
<SdkVersionForWorkloadTesting>6.0.100-rc.2.21463.12</SdkVersionForWorkloadTesting>
172+
<SdkVersionForWorkloadTesting>6.0.100-rc.2.21474.31</SdkVersionForWorkloadTesting>
173173
<!-- Docs -->
174174
<MicrosoftPrivateIntellisenseVersion>6.0.0-preview-20210916.1</MicrosoftPrivateIntellisenseVersion>
175175
<!-- ILLink -->

eng/testing/scenarios/BuildWasmAppsJobsList.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ Wasm.Build.Tests.LocalEMSDKTests
1111
Wasm.Build.Tests.MainWithArgsTests
1212
Wasm.Build.Tests.NativeBuildTests
1313
Wasm.Build.Tests.NativeLibraryTests
14+
Wasm.Build.Tests.PInvokeTableGeneratorTests
1415
Wasm.Build.Tests.RebuildTests
1516
Wasm.Build.Tests.SatelliteAssembliesTests
1617
Wasm.Build.Tests.WasmBuildAppTest

src/libraries/workloads-testing.targets

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
</ItemGroup>
1717

1818
<Copy SourceFiles="@(_SourceFiles)" DestinationFolder="$(SdkWithWorkloadForTestingPath)\%(_SourceFiles.RecursiveDir)" />
19-
<Copy SourceFiles="$(MonoProjectRoot)\wasm\BlazorOverwrite.targets" DestinationFiles="$(SdkWithWorkloadForTestingPath)\sdk\$(SdkVersionForWorkloadTesting)\Sdks\Microsoft.NET.Sdk.BlazorWebAssembly\targets\Microsoft.NET.Sdk.BlazorWebAssembly.6_0.targets" />
2019

2120
<WriteLinesToFile File="$(SdkWithWorkloadStampPath)" Lines="" Overwrite="true" />
2221
</Target>
@@ -41,7 +40,6 @@
4140
<Exec Condition="$([MSBuild]::IsOSPlatform('windows'))"
4241
Command='powershell -ExecutionPolicy ByPass -NoProfile -command "&amp; $(_DotNetInstallScriptPath) -InstallDir $(SdkWithNoWorkloadForTestingPath) -Version $(SdkVersionForWorkloadTesting)"' />
4342

44-
<Copy SourceFiles="$(MonoProjectRoot)\wasm\BlazorOverwrite.targets" DestinationFiles="$(SdkWithNoWorkloadForTestingPath)\sdk\$(SdkVersionForWorkloadTesting)\Sdks\Microsoft.NET.Sdk.BlazorWebAssembly\targets\Microsoft.NET.Sdk.BlazorWebAssembly.6_0.targets" />
4543
<WriteLinesToFile File="$(SdkWithNoWorkloadStampPath)" Lines="" Overwrite="true" />
4644
</Target>
4745

src/mono/wasm/BlazorOverwrite.targets

Lines changed: 0 additions & 741 deletions
This file was deleted.

src/mono/wasm/build/WasmApp.Native.targets

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@
143143
<PropertyGroup>
144144
<_MonoAotCrossCompilerPath>@(MonoAotCrossCompiler->WithMetadataValue('RuntimeIdentifier','browser-wasm'))</_MonoAotCrossCompilerPath>
145145
<_EmccDefaultFlagsRsp>$([MSBuild]::NormalizePath($(_WasmRuntimePackSrcDir), 'emcc-default.rsp'))</_EmccDefaultFlagsRsp>
146+
<WasmNativeStrip Condition="'$(WasmNativeStrip)' == '' and '$(Configuration)' == 'Debug' and '$(WasmBuildingForNestedPublish)' != 'true'">false</WasmNativeStrip>
146147
<WasmNativeStrip Condition="'$(WasmNativeStrip)' == ''">true</WasmNativeStrip>
147148
<WasmNativeDebugSymbols Condition="'$(WasmNativeDebugSymbols)' == ''">true</WasmNativeDebugSymbols>
148149
<WasmLinkIcalls Condition="'$(WasmLinkIcalls)' == ''">$(WasmBuildNative)</WasmLinkIcalls>
@@ -156,8 +157,7 @@
156157

157158
<_EmccAssertionLevelDefault>0</_EmccAssertionLevelDefault>
158159
<_EmccOptimizationFlagDefault Condition="'$(_WasmDevel)' == 'true'">-O0 -s ASSERTIONS=$(_EmccAssertionLevelDefault)</_EmccOptimizationFlagDefault>
159-
<_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(OS)' != 'Windows_NT' and '$(Configuration)' == 'Debug'">-Os</_EmccOptimizationFlagDefault>
160-
<_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' != 'Debug'">-Oz</_EmccOptimizationFlagDefault>
160+
<_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' == 'Debug' and '$(WasmBuildingForNestedPublish)' != 'true'">-O1</_EmccOptimizationFlagDefault>
161161
<_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == ''">-Oz</_EmccOptimizationFlagDefault>
162162

163163
<EmccCompileOptimizationFlag Condition="'$(EmccCompileOptimizationFlag)' == ''">$(_EmccOptimizationFlagDefault)</EmccCompileOptimizationFlag>
@@ -206,6 +206,9 @@
206206
<_EmccLDFlags Include="@(_EmccCommonFlags)" />
207207
<_EmccLDFlags Include="-s TOTAL_MEMORY=$(EmccTotalMemory)" />
208208

209+
<!-- ILLinker should have removed unused imports, so error for Publish -->
210+
<_EmccLDFlags Include="-s ERROR_ON_UNDEFINED_SYMBOLS=0" Condition="'$(WasmBuildingForNestedPublish)' != 'true'" />
211+
209212
<_DriverCDependencies Include="$(_WasmPInvokeHPath);$(_WasmICallTablePath)" />
210213
<_DriverCDependencies Include="$(_DriverGenCPath)" Condition="'$(_DriverGenCNeeded)' == 'true'" />
211214

@@ -296,7 +299,8 @@
296299
Inputs="@(_BitcodeFile);$(_EmccDefaultFlagsRsp);$(_EmccCompileBitcodeRsp)"
297300
Outputs="@(_BitcodeFile->'%(ObjectFile)')"
298301
Condition="'$(_WasmShouldAOT)' == 'true' and @(_BitcodeFile->Count()) > 0"
299-
DependsOnTargets="_WasmWriteRspForCompilingBitcode">
302+
DependsOnTargets="_WasmWriteRspForCompilingBitcode"
303+
Returns="@(FileWrites)">
300304

301305
<ItemGroup>
302306
<_BitCodeFile Dependencies="%(_BitCodeFile.Dependencies);$(_EmccDefaultFlagsRsp);$(_EmccCompileBitcodeRsp)" />
@@ -362,7 +366,8 @@
362366
<Target Name="_WasmLinkDotNet"
363367
Inputs="@(_WasmLinkDependencies);$(_EmccDefaultFlagsRsp);$(_EmccLinkRsp)"
364368
Outputs="$(_WasmIntermediateOutputPath)dotnet.js;$(_WasmIntermediateOutputPath)dotnet.wasm"
365-
DependsOnTargets="_WasmSelectRuntimeComponentsForLinking;_WasmCompileAssemblyBitCodeFilesForAOT;_WasmWriteRspFilesForLinking">
369+
DependsOnTargets="_WasmSelectRuntimeComponentsForLinking;_WasmCompileAssemblyBitCodeFilesForAOT;_WasmWriteRspFilesForLinking"
370+
Returns="@(FileWrites)" >
366371

367372
<Message Text="Linking with emcc. This may take a while ..." Importance="High" />
368373
<Message Text="Running emcc with @(_EmccLinkStepArgs->'%(Identity)', ' ')" Importance="Low" />

src/tasks/AotCompilerTask/MonoAOTCompiler.cs

Lines changed: 69 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,13 @@ private bool ProcessAndValidateArguments()
325325
throw new LogAsErrorException($"'{nameof(OutputType)}=Library' can not be used with '{nameof(UseStaticLinking)}=true'.");
326326
}
327327

328+
foreach (var asmItem in Assemblies)
329+
{
330+
string? fullPath = asmItem.GetMetadata("FullPath");
331+
if (!File.Exists(fullPath))
332+
throw new LogAsErrorException($"Could not find {fullPath} to AOT");
333+
}
334+
328335
return !Log.HasLoggedErrors;
329336
}
330337

@@ -339,6 +346,12 @@ public override bool Execute()
339346
Log.LogError(laee.Message);
340347
return false;
341348
}
349+
finally
350+
{
351+
if (_cache != null && _cache.Save(CacheFilePath!))
352+
_fileWrites.Add(CacheFilePath!);
353+
FileWrites = _fileWrites.ToArray();
354+
}
342355
}
343356

344357
private bool ExecuteInternal()
@@ -347,6 +360,7 @@ private bool ExecuteInternal()
347360
return false;
348361

349362
_assembliesToCompile = EnsureAndGetAssembliesInTheSameDir(Assemblies);
363+
_assembliesToCompile = FilterAssemblies(_assembliesToCompile);
350364

351365
if (!string.IsNullOrEmpty(AotModulesTablePath) && !GenerateAotModulesTable(_assembliesToCompile, Profilers, AotModulesTablePath))
352366
return false;
@@ -372,42 +386,29 @@ private bool ExecuteInternal()
372386
}
373387
else
374388
{
375-
int allowedParallelism = Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount);
389+
int allowedParallelism = DisableParallelAot ? 1 : Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount);
376390
if (BuildEngine is IBuildEngine9 be9)
377391
allowedParallelism = be9.RequestCores(allowedParallelism);
378392

379-
if (DisableParallelAot || allowedParallelism == 1)
393+
ParallelLoopResult result = Parallel.ForEach(
394+
argsList,
395+
new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism },
396+
(args, state) => PrecompileLibraryParallel(args, state));
397+
398+
Log.LogMessage(MessageImportance.High, $"result: {result.IsCompleted}");
399+
if (result.IsCompleted)
380400
{
381-
foreach (var args in argsList)
382-
{
383-
if (!PrecompileLibrarySerial(args))
384-
return !Log.HasLoggedErrors;
385-
}
401+
int numUnchanged = _totalNumAssemblies - _numCompiled;
402+
if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies)
403+
Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies.");
386404
}
387-
else
405+
else if (!Log.HasLoggedErrors)
388406
{
389-
ParallelLoopResult result = Parallel.ForEach(
390-
argsList,
391-
new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism },
392-
(args, state) => PrecompileLibraryParallel(args, state));
393-
394-
if (!result.IsCompleted)
395-
{
396-
return false;
397-
}
407+
Log.LogError($"Precompiling failed due to unknown reasons. Check log for more info");
398408
}
399-
400-
int numUnchanged = _totalNumAssemblies - _numCompiled;
401-
if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies)
402-
Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies.");
403409
}
404410

405411
CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, _assembliesToCompile).ToArray();
406-
407-
if (_cache.Save(CacheFilePath!))
408-
_fileWrites.Add(CacheFilePath!);
409-
FileWrites = _fileWrites.ToArray();
410-
411412
return !Log.HasLoggedErrors;
412413
}
413414

@@ -428,60 +429,68 @@ static bool IsNewerThanOutput(string inFile, string outFile)
428429
(File.GetLastWriteTimeUtc(inFile) > File.GetLastWriteTimeUtc(outFile));
429430
}
430431

431-
private IList<ITaskItem> EnsureAndGetAssembliesInTheSameDir(ITaskItem[] originalAssemblies)
432+
private IList<ITaskItem> FilterAssemblies(IEnumerable<ITaskItem> assemblies)
432433
{
433434
List<ITaskItem> filteredAssemblies = new();
434-
string firstAsmDir = Path.GetDirectoryName(originalAssemblies[0].GetMetadata("FullPath")) ?? string.Empty;
435-
bool allInSameDir = true;
436-
437-
foreach (var origAsm in originalAssemblies)
435+
foreach (var asmItem in assemblies)
438436
{
439-
if (allInSameDir && Path.GetDirectoryName(origAsm.GetMetadata("FullPath")) != firstAsmDir)
440-
allInSameDir = false;
441-
442-
if (ShouldSkip(origAsm))
437+
if (ShouldSkip(asmItem))
443438
{
444439
if (parsedAotMode == MonoAotMode.LLVMOnly)
445-
throw new LogAsErrorException($"Building in AOTMode=LLVMonly is not compatible with excluding any assemblies for AOT. Excluded assembly: {origAsm.ItemSpec}");
440+
throw new LogAsErrorException($"Building in AOTMode=LLVMonly is not compatible with excluding any assemblies for AOT. Excluded assembly: {asmItem.ItemSpec}");
446441

447-
Log.LogMessage(MessageImportance.Low, $"Skipping {origAsm.ItemSpec} because it has %(AOT_InternalForceToInterpret)=true");
442+
Log.LogMessage(MessageImportance.Low, $"Skipping {asmItem.ItemSpec} because it has %(AOT_InternalForceToInterpret)=true");
448443
continue;
449444
}
450445

451-
filteredAssemblies.Add(origAsm);
446+
string assemblyPath = asmItem.GetMetadata("FullPath");
447+
using var assemblyFile = File.OpenRead(assemblyPath);
448+
using PEReader reader = new(assemblyFile, PEStreamOptions.Default);
449+
if (!reader.HasMetadata)
450+
{
451+
Log.LogWarning($"Skipping unmanaged {assemblyPath} for AOT");
452+
continue;
453+
}
454+
455+
filteredAssemblies.Add(asmItem);
452456
}
453457

458+
return filteredAssemblies;
459+
460+
static bool ShouldSkip(ITaskItem asmItem)
461+
=> bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip;
462+
}
463+
464+
private IList<ITaskItem> EnsureAndGetAssembliesInTheSameDir(IList<ITaskItem> assemblies)
465+
{
466+
string firstAsmDir = Path.GetDirectoryName(assemblies.First().GetMetadata("FullPath")) ?? string.Empty;
467+
bool allInSameDir = assemblies.All(asm => Path.GetDirectoryName(asm.GetMetadata("FullPath")) == firstAsmDir);
454468
if (allInSameDir)
455-
return filteredAssemblies;
469+
return assemblies;
456470

457471
// Copy to aot-in
458472

459473
string aotInPath = Path.Combine(IntermediateOutputPath, "aot-in");
460474
Directory.CreateDirectory(aotInPath);
461475

462476
List<ITaskItem> newAssemblies = new();
463-
foreach (var origAsm in originalAssemblies)
477+
foreach (var asmItem in assemblies)
464478
{
465-
string asmPath = origAsm.GetMetadata("FullPath");
479+
string asmPath = asmItem.GetMetadata("FullPath");
466480
string newPath = Path.Combine(aotInPath, Path.GetFileName(asmPath));
467481

468482
// FIXME: delete files not in originalAssemblies though
469483
// FIXME: or .. just delete the whole dir?
470484
if (Utils.CopyIfDifferent(asmPath, newPath, useHash: true))
471485
Log.LogMessage(MessageImportance.Low, $"Copying {asmPath} to {newPath}");
486+
_fileWrites.Add(newPath);
472487

473-
if (!ShouldSkip(origAsm))
474-
{
475-
ITaskItem newAsm = new TaskItem(newPath);
476-
origAsm.CopyMetadataTo(newAsm);
477-
newAssemblies.Add(newAsm);
478-
}
488+
ITaskItem newAsm = new TaskItem(newPath);
489+
asmItem.CopyMetadataTo(newAsm);
490+
newAssemblies.Add(newAsm);
479491
}
480492

481493
return newAssemblies;
482-
483-
static bool ShouldSkip(ITaskItem asmItem)
484-
=> bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip;
485494
}
486495

487496
private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, string? monoPaths)
@@ -773,28 +782,6 @@ private bool PrecompileLibrary(PrecompileArguments args)
773782
return true;
774783
}
775784

776-
private bool PrecompileLibrarySerial(PrecompileArguments args)
777-
{
778-
try
779-
{
780-
if (PrecompileLibrary(args))
781-
return true;
782-
}
783-
catch (LogAsErrorException laee)
784-
{
785-
Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}");
786-
}
787-
catch (Exception ex)
788-
{
789-
if (Log.HasLoggedErrors)
790-
Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}");
791-
else
792-
Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}");
793-
}
794-
795-
return false;
796-
}
797-
798785
private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopState state)
799786
{
800787
try
@@ -804,14 +791,14 @@ private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopSta
804791
}
805792
catch (LogAsErrorException laee)
806793
{
807-
Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}");
794+
Log.LogError($"Precompiling failed for {args.AOTAssembly}: {laee.Message}");
808795
}
809796
catch (Exception ex)
810797
{
811798
if (Log.HasLoggedErrors)
812799
Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}");
813800
else
814-
Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}");
801+
Log.LogError($"Precompiling failed for {args.AOTAssembly}: {ex}");
815802
}
816803

817804
state.Break();
@@ -940,10 +927,8 @@ private static IList<ITaskItem> ConvertAssembliesDictToOrderedList(ConcurrentDic
940927
List<ITaskItem> outItems = new(originalAssemblies.Count);
941928
foreach (ITaskItem item in originalAssemblies)
942929
{
943-
if (!dict.TryGetValue(item.GetMetadata("FullPath"), out ITaskItem? dictItem))
944-
throw new LogAsErrorException($"Bug: Could not find item in the dict with key {item.ItemSpec}");
945-
946-
outItems.Add(dictItem);
930+
if (dict.TryGetValue(item.GetMetadata("FullPath"), out ITaskItem? dictItem))
931+
outItems.Add(dictItem);
947932
}
948933
return outItems;
949934
}
@@ -993,7 +978,7 @@ public FileCache(string? cacheFilePath, TaskLoggingHelper log)
993978
}
994979

995980
_oldCache ??= new();
996-
_newCache = new();
981+
_newCache = new(_oldCache.FileHashes);
997982
}
998983

999984
public bool ShouldCopy(ProxyFile proxyFile, [NotNullWhen(true)] out string? cause)
@@ -1110,6 +1095,10 @@ public enum MonoAotModulesTableLanguage
11101095

11111096
internal class CompilerCache
11121097
{
1098+
public CompilerCache() => FileHashes = new();
1099+
public CompilerCache(IDictionary<string, string> oldHashes)
1100+
=> FileHashes = new(oldHashes);
1101+
11131102
[JsonPropertyName("file_hashes")]
1114-
public ConcurrentDictionary<string, string> FileHashes { get; set; } = new();
1103+
public ConcurrentDictionary<string, string> FileHashes { get; set; }
11151104
}

0 commit comments

Comments
 (0)