From 5f569e3fe74fb83f55b51dd6a18ccda825acf33e Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 11 Sep 2021 03:56:56 -0400 Subject: [PATCH 01/22] Refactor to allow fast-path --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 84 ++++++++++++++------ 1 file changed, 58 insertions(+), 26 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 4ee7389c04df01..6d463af1bc9c1d 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -357,6 +357,10 @@ private bool ExecuteInternal() _cache = new FileCache(CacheFilePath, Log); + List argsList = new(); + foreach (var assemblyItem in _assembliesToCompile) + argsList.Add(GetPrecompileArgumentsFor(assemblyItem, monoPaths)); + //FIXME: check the nothing changed at all case _totalNumAssemblies = _assembliesToCompile.Count; @@ -366,18 +370,18 @@ private bool ExecuteInternal() if (DisableParallelAot || allowedParallelism == 1) { - foreach (var assemblyItem in _assembliesToCompile) + foreach (var args in argsList) { - if (!PrecompileLibrarySerial(assemblyItem, monoPaths)) + if (!PrecompileLibrarySerial(args)) return !Log.HasLoggedErrors; } } else { ParallelLoopResult result = Parallel.ForEach( - _assembliesToCompile, + argsList, new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, - (assemblyItem, state) => PrecompileLibraryParallel(assemblyItem, monoPaths, state)); + (args, state) => PrecompileLibraryParallel(args, state)); if (!result.IsCompleted) { @@ -454,7 +458,7 @@ static bool ShouldSkip(ITaskItem asmItem) => bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip; } - private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) + private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, string? monoPaths) { string assembly = assemblyItem.GetMetadata("FullPath"); string assemblyDir = Path.GetDirectoryName(assembly)!; @@ -463,7 +467,6 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) var processArgs = new List(); bool isDedup = assembly == DedupAssembly; List proxyFiles = new(capacity: 5); - string msgPrefix = $"[{Path.GetFileName(assembly)}] "; var a = assemblyItem.GetMetadata("AotArguments"); if (a != null) @@ -674,16 +677,26 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) sw.WriteLine(responseFileContent); } - string workingDir = assemblyDir; + return new PrecompileArguments(ResponseFilePath: responseFilePath, + EnvironmentVariables: envVariables, + WorkingDir: assemblyDir, + AOTAssembly: aotAssembly, + ProxyFiles: proxyFiles); + } + private bool PrecompileLibrary(PrecompileArguments args) + { + string assembly = args.AOTAssembly.GetMetadata("FullPath"); try { + string msgPrefix = $"[{Path.GetFileName(assembly)}] "; + // run the AOT compiler (int exitCode, string output) = Utils.TryRunProcess(Log, CompilerBinaryPath, - $"--response=\"{responseFilePath}\"", - envVariables, - workingDir, + $"--response=\"{args.ResponseFilePath}\"", + args.EnvironmentVariables, + args.WorkingDir, silent: true, debugMessageImportance: MessageImportance.Low, label: Path.GetFileName(assembly)); @@ -692,9 +705,9 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) // Log the command in a compact format which can be copy pasted { StringBuilder envStr = new StringBuilder(string.Empty); - foreach (KeyValuePair kvp in envVariables) + foreach (KeyValuePair kvp in args.EnvironmentVariables) envStr.Append($"{kvp.Key}={kvp.Value} "); - Log.LogMessage(importance, $"{msgPrefix}Exec (with response file contents expanded) in {workingDir}: {envStr}{CompilerBinaryPath} {responseFileContent}"); + Log.LogMessage(importance, $"{msgPrefix}Exec (with response file contents expanded) in {args.WorkingDir}: {envStr}{CompilerBinaryPath} {File.ReadAllText(args.ResponseFilePath)}"); } Log.LogMessage(importance, output); @@ -713,11 +726,11 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) } finally { - File.Delete(responseFilePath); + File.Delete(args.ResponseFilePath); } bool copied = false; - foreach (var proxyFile in proxyFiles) + foreach (var proxyFile in args.ProxyFiles) { copied |= proxyFile.CopyOutputFileIfChanged(); _fileWrites.Add(proxyFile.TargetFile); @@ -725,54 +738,54 @@ private bool PrecompileLibrary(ITaskItem assemblyItem, string? monoPaths) if (copied) { - string copiedFiles = string.Join(", ", proxyFiles.Select(tf => Path.GetFileName(tf.TargetFile))); + string copiedFiles = string.Join(", ", args.ProxyFiles.Select(tf => Path.GetFileName(tf.TargetFile))); int count = Interlocked.Increment(ref _numCompiled); Log.LogMessage(MessageImportance.High, $"[{count}/{_totalNumAssemblies}] {Path.GetFileName(assembly)} -> {copiedFiles}"); } - compiledAssemblies.GetOrAdd(aotAssembly.ItemSpec, aotAssembly); + compiledAssemblies.GetOrAdd(args.AOTAssembly.ItemSpec, args.AOTAssembly); return true; } - private bool PrecompileLibrarySerial(ITaskItem assemblyItem, string? monoPaths) + private bool PrecompileLibrarySerial(PrecompileArguments args) { try { - if (PrecompileLibrary(assemblyItem, monoPaths)) + if (PrecompileLibrary(args)) return true; } catch (LogAsErrorException laee) { - Log.LogError($"Precompile failed for {assemblyItem}: {laee.Message}"); + Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}"); } catch (Exception ex) { if (Log.HasLoggedErrors) - Log.LogMessage(MessageImportance.Low, $"Precompile failed for {assemblyItem}: {ex}"); + Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}"); else - Log.LogError($"Precompile failed for {assemblyItem}: {ex}"); + Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}"); } return false; } - private void PrecompileLibraryParallel(ITaskItem assemblyItem, string? monoPaths, ParallelLoopState state) + private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopState state) { try { - if (PrecompileLibrary(assemblyItem, monoPaths)) + if (PrecompileLibrary(args)) return; } catch (LogAsErrorException laee) { - Log.LogError($"Precompile failed for {assemblyItem}: {laee.Message}"); + Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}"); } catch (Exception ex) { if (Log.HasLoggedErrors) - Log.LogMessage(MessageImportance.Low, $"Precompile failed for {assemblyItem}: {ex}"); + Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}"); else - Log.LogError($"Precompile failed for {assemblyItem}: {ex}"); + Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}"); } state.Break(); @@ -908,6 +921,24 @@ private static IList ConvertAssembliesDictToOrderedList(ConcurrentDic } return outItems; } + + internal class PrecompileArguments + { + public PrecompileArguments(string ResponseFilePath, IDictionary EnvironmentVariables, string WorkingDir, ITaskItem AOTAssembly, IList ProxyFiles) + { + this.ResponseFilePath = ResponseFilePath; + this.EnvironmentVariables = EnvironmentVariables; + this.WorkingDir = WorkingDir; + this.AOTAssembly = AOTAssembly; + this.ProxyFiles = ProxyFiles; + } + + public string ResponseFilePath { get; private set; } + public IDictionary EnvironmentVariables { get; private set; } + public string WorkingDir { get; private set; } + public ITaskItem AOTAssembly { get; private set; } + public IList ProxyFiles { get; private set; } + } } internal class FileCache @@ -1017,6 +1048,7 @@ public bool CopyOutputFileIfChanged() File.Delete(TempFile); } } + } public enum MonoAotMode From 5b3b91fa0a86fb1bb0398ee4736882c56212c101 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 11 Sep 2021 04:57:41 -0400 Subject: [PATCH 02/22] MonoAOTCompiler: check for nothing-changed case The task needs to run `mono-aot-cross` on all the assemblies, even if a given assembly is older than its output files, so as to correctly handle changes in its dependencies. This commit tries to check if all the input assemblies are up-to-date with their corresponding output files, and avoids doing any work if that's true. --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 70 ++++++++++++++------ 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 6d463af1bc9c1d..59f8edae779585 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -361,47 +361,73 @@ private bool ExecuteInternal() foreach (var assemblyItem in _assembliesToCompile) argsList.Add(GetPrecompileArgumentsFor(assemblyItem, monoPaths)); - //FIXME: check the nothing changed at all case - _totalNumAssemblies = _assembliesToCompile.Count; - int allowedParallelism = Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount); - if (BuildEngine is IBuildEngine9 be9) - allowedParallelism = be9.RequestCores(allowedParallelism); - - if (DisableParallelAot || allowedParallelism == 1) + if (CheckAllUpToDate(argsList)) { + Log.LogMessage(MessageImportance.High, "Everything is up-to-date, nothing to precompile"); + + _fileWrites.AddRange(argsList.SelectMany(args => args.ProxyFiles).Select(pf => pf.TargetFile)); foreach (var args in argsList) - { - if (!PrecompileLibrarySerial(args)) - return !Log.HasLoggedErrors; - } + compiledAssemblies.GetOrAdd(args.AOTAssembly.ItemSpec, args.AOTAssembly); } else { - ParallelLoopResult result = Parallel.ForEach( - argsList, - new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, - (args, state) => PrecompileLibraryParallel(args, state)); + int allowedParallelism = Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount); + if (BuildEngine is IBuildEngine9 be9) + allowedParallelism = be9.RequestCores(allowedParallelism); - if (!result.IsCompleted) + if (DisableParallelAot || allowedParallelism == 1) { - return false; + foreach (var args in argsList) + { + if (!PrecompileLibrarySerial(args)) + return !Log.HasLoggedErrors; + } + } + else + { + ParallelLoopResult result = Parallel.ForEach( + argsList, + new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, + (args, state) => PrecompileLibraryParallel(args, state)); + + if (!result.IsCompleted) + { + return false; + } } + + int numUnchanged = _totalNumAssemblies - _numCompiled; + if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) + Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); } - int numUnchanged = _totalNumAssemblies - _numCompiled; - if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) - Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); + CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, _assembliesToCompile).ToArray(); if (_cache.Save(CacheFilePath!)) _fileWrites.Add(CacheFilePath!); - - CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, _assembliesToCompile).ToArray(); FileWrites = _fileWrites.ToArray(); return !Log.HasLoggedErrors; } + private bool CheckAllUpToDate(IList argsList) + { + foreach (var args in argsList) + { + // compare original assembly vs it's outputs.. all it's outputs! + string assemblyPath = args.AOTAssembly.GetMetadata("FullPath"); + if (args.ProxyFiles.Any(pf => IsNewerThanOutput(assemblyPath, pf.TargetFile))) + return false; + } + + return true; + + static bool IsNewerThanOutput(string inFile, string outFile) + => !File.Exists(inFile) || !File.Exists(outFile) || + (File.GetLastWriteTimeUtc(inFile) > File.GetLastWriteTimeUtc(outFile)); + } + private IList EnsureAndGetAssembliesInTheSameDir(ITaskItem[] originalAssemblies) { List filteredAssemblies = new(); From 28fdcba9ae4bede0162a531aeca0e07b0fee96b0 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 17 Sep 2021 16:25:44 -0400 Subject: [PATCH 03/22] [wasm] Change optimization flag defaults for Debug config `WasmNativeStrip` -> `false` Compile, and link optimization flags for `emcc` to `-O1` --- src/mono/wasm/build/WasmApp.Native.targets | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 19737849bdaeef..9bc59c7ee0e5ac 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -143,6 +143,7 @@ <_MonoAotCrossCompilerPath>@(MonoAotCrossCompiler->WithMetadataValue('RuntimeIdentifier','browser-wasm')) <_EmccDefaultFlagsRsp>$([MSBuild]::NormalizePath($(_WasmRuntimePackSrcDir), 'emcc-default.rsp')) + false true true $(WasmBuildNative) @@ -156,6 +157,7 @@ <_EmccAssertionLevelDefault>0 <_EmccOptimizationFlagDefault Condition="'$(_WasmDevel)' == 'true'">-O0 -s ASSERTIONS=$(_EmccAssertionLevelDefault) + <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' == 'Debug'">-O1 <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(OS)' != 'Windows_NT' and '$(Configuration)' == 'Debug'">-Os <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' != 'Debug'">-Oz <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == ''">-Oz From 6e521721c9ea4537da49a12c21afd9e6c7858987 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Mon, 20 Sep 2021 22:23:48 -0400 Subject: [PATCH 04/22] [wasm] EmccCompile: Fix incremental build, in case of only partial .. rebuilds. Specifically, when a rebuild causes only the *linking* part to be run again. In that case, we were correctly skipping over compiling native files, but didn't add them to `@(FileWrites)`, which caused msbuild's incremental clean logic to treat them as "orphaned" files, and delete them! --- src/mono/wasm/build/WasmApp.Native.targets | 6 +- src/tasks/WasmAppBuilder/EmccCompile.cs | 15 +++-- .../BlazorWasmBuildPublishTests.cs | 1 - .../NativeRebuildTestsBase.cs | 13 +++++ .../NoopNativeRebuildTest.cs | 56 +++++++++++++++++++ 5 files changed, 84 insertions(+), 7 deletions(-) diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 9bc59c7ee0e5ac..2889113200f464 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -298,7 +298,8 @@ Inputs="@(_BitcodeFile);$(_EmccDefaultFlagsRsp);$(_EmccCompileBitcodeRsp)" Outputs="@(_BitcodeFile->'%(ObjectFile)')" Condition="'$(_WasmShouldAOT)' == 'true' and @(_BitcodeFile->Count()) > 0" - DependsOnTargets="_WasmWriteRspForCompilingBitcode"> + DependsOnTargets="_WasmWriteRspForCompilingBitcode" + Returns="@(FileWrites)"> <_BitCodeFile Dependencies="%(_BitCodeFile.Dependencies);$(_EmccDefaultFlagsRsp);$(_EmccCompileBitcodeRsp)" /> @@ -364,7 +365,8 @@ + DependsOnTargets="_WasmSelectRuntimeComponentsForLinking;_WasmCompileAssemblyBitCodeFilesForAOT;_WasmWriteRspFilesForLinking" + Returns="@(FileWrites)" > diff --git a/src/tasks/WasmAppBuilder/EmccCompile.cs b/src/tasks/WasmAppBuilder/EmccCompile.cs index a5d6af6c3b473a..b0bc0ff80050b3 100644 --- a/src/tasks/WasmAppBuilder/EmccCompile.cs +++ b/src/tasks/WasmAppBuilder/EmccCompile.cs @@ -95,6 +95,7 @@ private bool ExecuteActual() if (!ShouldCompile(srcFile, objFile, depFiles, out string reason)) { Log.LogMessage(MessageImportance.Low, $"Skipping {srcFile} because {reason}."); + outputItems.Add(CreateOutputItemFor(srcFile, objFile)); } else { @@ -107,7 +108,8 @@ private bool ExecuteActual() if (_numCompiled == _totalFiles) { // nothing to do! - return true; + OutputFiles = outputItems.ToArray(); + return !Log.HasLoggedErrors; } if (_numCompiled > 0) @@ -200,9 +202,7 @@ bool ProcessSourceFile(string srcFile, string objFile) else Log.LogMessage(MessageImportance.Low, $"Copied {tmpObjFile} to {objFile}"); - ITaskItem newItem = new TaskItem(objFile); - newItem.SetMetadata("SourceFile", srcFile); - outputItems.Add(newItem); + outputItems.Add(CreateOutputItemFor(srcFile, objFile)); int count = Interlocked.Increment(ref _numCompiled); Log.LogMessage(MessageImportance.High, $"[{count}/{_totalFiles}] {Path.GetFileName(srcFile)} -> {Path.GetFileName(objFile)} [took {elapsedSecs:F}s]"); @@ -219,6 +219,13 @@ bool ProcessSourceFile(string srcFile, string objFile) File.Delete(tmpObjFile); } } + + ITaskItem CreateOutputItemFor(string srcFile, string objFile) + { + ITaskItem newItem = new TaskItem(objFile); + newItem.SetMetadata("SourceFile", srcFile); + return newItem; + } } private bool ShouldCompile(string srcFile, string objFile, string[] depFiles, out string reason) diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmBuildPublishTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmBuildPublishTests.cs index 687429a3fca482..66b5c5b2782766 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmBuildPublishTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmBuildPublishTests.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.IO; using System.Linq; using Xunit; diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs index 9047fd062a82e4..cd0d3ac4396349 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NativeRebuildTestsBase.cs @@ -77,6 +77,7 @@ protected string Rebuild(bool nativeRelink, bool invariant, BuildArgs buildArgs, buildArgs = newBuildArgs; _testOutput.WriteLine($"{Environment.NewLine}Rebuilding with no changes ..{Environment.NewLine}"); + Console.WriteLine($"{Environment.NewLine}Rebuilding with no changes ..{Environment.NewLine}"); (_, string output) = BuildProject(buildArgs, id: id, dotnetWasmFromRuntimePack: false, @@ -137,6 +138,18 @@ internal void CompareStat(IDictionary oldStat, IDictionary GetFilesTable(bool unchanged, params string[] baseDirs) + { + var dict = new Dictionary(); + foreach (var baseDir in baseDirs) + { + foreach (var file in Directory.EnumerateFiles(baseDir, "*", new EnumerationOptions { RecurseSubdirectories = true })) + dict[Path.GetFileName(file)] = (file, unchanged); + } + + return dict; + } + internal IDictionary GetFilesTable(BuildArgs buildArgs, BuildPaths paths, bool unchanged) { List files = new() diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs index 96f6d41fa10025..3131dec390ce05 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeRebuildTests/NoopNativeRebuildTest.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.IO; using System.Linq; using Wasm.Build.Tests; using Xunit; @@ -33,5 +34,60 @@ public void NoOpRebuildForNativeBuilds(BuildArgs buildArgs, bool nativeRelink, b CompareStat(originalStat, newStat, pathsDict.Values); RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); } + + [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] + [InlineData("Debug")] + [InlineData("Release")] + public void BlazorNoopRebuild(string config) + { + string id = $"blz_rebuild_{config}"; + string projectFile = CreateBlazorWasmTemplateProject(id); + AddItemsPropertiesToProject(projectFile, extraProperties: "true"); + + string objDir = Path.Combine(_projectDir!, "obj", config, "net6.0", "wasm"); + + BlazorBuild(id, config, NativeFilesType.Relinked); + File.Move(Path.Combine(s_buildEnv.LogRootPath, id, $"{id}-build.binlog"), + Path.Combine(s_buildEnv.LogRootPath, id, $"{id}-build-first.binlog")); + + var pathsDict = GetFilesTable(true, objDir); + pathsDict.Remove("runtime-icall-table.h"); + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // build again + BlazorBuild(id, config, NativeFilesType.Relinked); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + } + + + [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] + [InlineData("Debug")] + [InlineData("Release")] + public void BlazorOnlyLinkRebuild(string config) + { + string id = $"blz_relink_{config}"; + string projectFile = CreateBlazorWasmTemplateProject(id); + AddItemsPropertiesToProject(projectFile, extraProperties: "true"); + + string objDir = Path.Combine(_projectDir!, "obj", config, "net6.0", "wasm"); + + BlazorBuild(id, config, NativeFilesType.Relinked); + File.Move(Path.Combine(s_buildEnv.LogRootPath, id, $"{id}-build.binlog"), + Path.Combine(s_buildEnv.LogRootPath, id, $"{id}-build-first.binlog")); + + var pathsDict = GetFilesTable(true, objDir); + pathsDict.Remove("runtime-icall-table.h"); + pathsDict.UpdateTo(unchanged: false, "dotnet.wasm", "dotnet.js", "emcc-link.rsp"); + + var originalStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + // build again + BlazorBuild(id, config, NativeFilesType.Relinked, "-p:EmccLinkOptimizationFlag=-O1"); + var newStat = StatFiles(pathsDict.Select(kvp => kvp.Value.fullPath)); + + CompareStat(originalStat, newStat, pathsDict.Values); + } } } From 08ae443d68da7e4138c0e3841339b137b3ffaa7b Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 22 Sep 2021 00:35:16 -0400 Subject: [PATCH 05/22] MonoAOTCompiler: Skip unmanaged assemblies, and emit a warning --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 63 ++++++++++++-------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 59f8edae779585..3144d12ee69310 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -325,6 +325,13 @@ private bool ProcessAndValidateArguments() throw new LogAsErrorException($"'{nameof(OutputType)}=Library' can not be used with '{nameof(UseStaticLinking)}=true'."); } + foreach (var asmItem in Assemblies) + { + string? fullPath = asmItem.GetMetadata("FullPath"); + if (!File.Exists(fullPath)) + throw new LogAsErrorException($"Could not find {fullPath} to AOT"); + } + return !Log.HasLoggedErrors; } @@ -347,6 +354,7 @@ private bool ExecuteInternal() return false; _assembliesToCompile = EnsureAndGetAssembliesInTheSameDir(Assemblies); + _assembliesToCompile = FilterAssemblies(_assembliesToCompile); if (!string.IsNullOrEmpty(AotModulesTablePath) && !GenerateAotModulesTable(_assembliesToCompile, Profilers, AotModulesTablePath)) return false; @@ -428,31 +436,43 @@ static bool IsNewerThanOutput(string inFile, string outFile) (File.GetLastWriteTimeUtc(inFile) > File.GetLastWriteTimeUtc(outFile)); } - private IList EnsureAndGetAssembliesInTheSameDir(ITaskItem[] originalAssemblies) + private IList FilterAssemblies(IEnumerable assemblies) { List filteredAssemblies = new(); - string firstAsmDir = Path.GetDirectoryName(originalAssemblies[0].GetMetadata("FullPath")) ?? string.Empty; - bool allInSameDir = true; - - foreach (var origAsm in originalAssemblies) + foreach (var asmItem in assemblies) { - if (allInSameDir && Path.GetDirectoryName(origAsm.GetMetadata("FullPath")) != firstAsmDir) - allInSameDir = false; - - if (ShouldSkip(origAsm)) + if (ShouldSkip(asmItem)) { if (parsedAotMode == MonoAotMode.LLVMOnly) - throw new LogAsErrorException($"Building in AOTMode=LLVMonly is not compatible with excluding any assemblies for AOT. Excluded assembly: {origAsm.ItemSpec}"); + throw new LogAsErrorException($"Building in AOTMode=LLVMonly is not compatible with excluding any assemblies for AOT. Excluded assembly: {asmItem.ItemSpec}"); - Log.LogMessage(MessageImportance.Low, $"Skipping {origAsm.ItemSpec} because it has %(AOT_InternalForceToInterpret)=true"); + Log.LogMessage(MessageImportance.Low, $"Skipping {asmItem.ItemSpec} because it has %(AOT_InternalForceToInterpret)=true"); continue; } - filteredAssemblies.Add(origAsm); + string assemblyPath = asmItem.GetMetadata("FullPath"); + PEReader reader = new(File.OpenRead(assemblyPath), PEStreamOptions.Default); + if (!reader.HasMetadata) + { + Log.LogWarning($"Skipping unmanaged {assemblyPath} for AOT"); + continue; + } + + filteredAssemblies.Add(asmItem); } + return filteredAssemblies; + + static bool ShouldSkip(ITaskItem asmItem) + => bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip; + } + + private IList EnsureAndGetAssembliesInTheSameDir(IList assemblies) + { + string firstAsmDir = Path.GetDirectoryName(assemblies.First().GetMetadata("FullPath")) ?? string.Empty; + bool allInSameDir = assemblies.All(asm => Path.GetDirectoryName(asm.GetMetadata("FullPath")) == firstAsmDir); if (allInSameDir) - return filteredAssemblies; + return assemblies; // Copy to aot-in @@ -460,28 +480,23 @@ private IList EnsureAndGetAssembliesInTheSameDir(ITaskItem[] original Directory.CreateDirectory(aotInPath); List newAssemblies = new(); - foreach (var origAsm in originalAssemblies) + foreach (var asmItem in assemblies) { - string asmPath = origAsm.GetMetadata("FullPath"); + string asmPath = asmItem.GetMetadata("FullPath"); string newPath = Path.Combine(aotInPath, Path.GetFileName(asmPath)); // FIXME: delete files not in originalAssemblies though // FIXME: or .. just delete the whole dir? if (Utils.CopyIfDifferent(asmPath, newPath, useHash: true)) Log.LogMessage(MessageImportance.Low, $"Copying {asmPath} to {newPath}"); + _fileWrites.Add(newPath); - if (!ShouldSkip(origAsm)) - { - ITaskItem newAsm = new TaskItem(newPath); - origAsm.CopyMetadataTo(newAsm); - newAssemblies.Add(newAsm); - } + ITaskItem newAsm = new TaskItem(newPath); + asmItem.CopyMetadataTo(newAsm); + newAssemblies.Add(newAsm); } return newAssemblies; - - static bool ShouldSkip(ITaskItem asmItem) - => bool.TryParse(asmItem.GetMetadata("AOT_InternalForceToInterpret"), out bool skip) && skip; } private PrecompileArguments GetPrecompileArgumentsFor(ITaskItem assemblyItem, string? monoPaths) From 7fe0e69eed591a319cf14d0e33d671ec277485fe Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 22 Sep 2021 19:06:23 +0000 Subject: [PATCH 06/22] Apply suggestions from code review Co-authored-by: Larry Ewing --- src/mono/wasm/build/WasmApp.Native.targets | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 2889113200f464..2bcd4363ef12d3 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -158,8 +158,6 @@ <_EmccAssertionLevelDefault>0 <_EmccOptimizationFlagDefault Condition="'$(_WasmDevel)' == 'true'">-O0 -s ASSERTIONS=$(_EmccAssertionLevelDefault) <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' == 'Debug'">-O1 - <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(OS)' != 'Windows_NT' and '$(Configuration)' == 'Debug'">-Os - <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' != 'Debug'">-Oz <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == ''">-Oz $(_EmccOptimizationFlagDefault) From 1c0a3405cc8e8197616d23281374cd268e1167ee Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 22 Sep 2021 14:57:04 -0400 Subject: [PATCH 07/22] MonoAOTCompiler: write the cache even when some files fail to compile - Ensure that the cache gets written even when some of the files failed to compile - And when some were skipped, and others failed --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 87 +++++++------------- src/tasks/WasmAppBuilder/EmccCompile.cs | 29 ++----- 2 files changed, 41 insertions(+), 75 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 3144d12ee69310..5300a35a2f4ebf 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -346,6 +346,12 @@ public override bool Execute() Log.LogError(laee.Message); return false; } + finally + { + if (_cache != null && _cache.Save(CacheFilePath!)) + _fileWrites.Add(CacheFilePath!); + FileWrites = _fileWrites.ToArray(); + } } private bool ExecuteInternal() @@ -380,42 +386,29 @@ private bool ExecuteInternal() } else { - int allowedParallelism = Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount); + int allowedParallelism = DisableParallelAot ? 1 : Math.Min(_assembliesToCompile.Count, Environment.ProcessorCount); if (BuildEngine is IBuildEngine9 be9) allowedParallelism = be9.RequestCores(allowedParallelism); - if (DisableParallelAot || allowedParallelism == 1) + ParallelLoopResult result = Parallel.ForEach( + argsList, + new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, + (args, state) => PrecompileLibraryParallel(args, state)); + + Log.LogMessage(MessageImportance.High, $"result: {result.IsCompleted}"); + if (result.IsCompleted) { - foreach (var args in argsList) - { - if (!PrecompileLibrarySerial(args)) - return !Log.HasLoggedErrors; - } + int numUnchanged = _totalNumAssemblies - _numCompiled; + if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) + Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); } - else + else if (!Log.HasLoggedErrors) { - ParallelLoopResult result = Parallel.ForEach( - argsList, - new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, - (args, state) => PrecompileLibraryParallel(args, state)); - - if (!result.IsCompleted) - { - return false; - } + Log.LogError($"Precompiling failed due to unknown reasons. Check log for more info"); } - - int numUnchanged = _totalNumAssemblies - _numCompiled; - if (numUnchanged > 0 && numUnchanged != _totalNumAssemblies) - Log.LogMessage(MessageImportance.High, $"[{numUnchanged}/{_totalNumAssemblies}] skipped unchanged assemblies."); } CompiledAssemblies = ConvertAssembliesDictToOrderedList(compiledAssemblies, _assembliesToCompile).ToArray(); - - if (_cache.Save(CacheFilePath!)) - _fileWrites.Add(CacheFilePath!); - FileWrites = _fileWrites.ToArray(); - return !Log.HasLoggedErrors; } @@ -788,28 +781,6 @@ private bool PrecompileLibrary(PrecompileArguments args) return true; } - private bool PrecompileLibrarySerial(PrecompileArguments args) - { - try - { - if (PrecompileLibrary(args)) - return true; - } - catch (LogAsErrorException laee) - { - Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}"); - } - catch (Exception ex) - { - if (Log.HasLoggedErrors) - Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}"); - else - Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}"); - } - - return false; - } - private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopState state) { try @@ -819,14 +790,14 @@ private void PrecompileLibraryParallel(PrecompileArguments args, ParallelLoopSta } catch (LogAsErrorException laee) { - Log.LogError($"Precompile failed for {args.AOTAssembly}: {laee.Message}"); + Log.LogError($"Precompiling failed for {args.AOTAssembly}: {laee.Message}"); } catch (Exception ex) { if (Log.HasLoggedErrors) Log.LogMessage(MessageImportance.Low, $"Precompile failed for {args.AOTAssembly}: {ex}"); else - Log.LogError($"Precompile failed for {args.AOTAssembly}: {ex}"); + Log.LogError($"Precompiling failed for {args.AOTAssembly}: {ex}"); } state.Break(); @@ -955,10 +926,12 @@ private static IList ConvertAssembliesDictToOrderedList(ConcurrentDic List outItems = new(originalAssemblies.Count); foreach (ITaskItem item in originalAssemblies) { - if (!dict.TryGetValue(item.GetMetadata("FullPath"), out ITaskItem? dictItem)) - throw new LogAsErrorException($"Bug: Could not find item in the dict with key {item.ItemSpec}"); + if (dict.TryGetValue(item.GetMetadata("FullPath"), out ITaskItem? dictItem)) + { + // throw new LogAsErrorException($"Bug: Could not find item in the dict with key {item.ItemSpec}"); - outItems.Add(dictItem); + outItems.Add(dictItem); + } } return outItems; } @@ -1008,7 +981,7 @@ public FileCache(string? cacheFilePath, TaskLoggingHelper log) } _oldCache ??= new(); - _newCache = new(); + _newCache = new(_oldCache.FileHashes); } public bool ShouldCopy(ProxyFile proxyFile, [NotNullWhen(true)] out string? cause) @@ -1125,6 +1098,10 @@ public enum MonoAotModulesTableLanguage internal class CompilerCache { + public CompilerCache() => FileHashes = new(); + public CompilerCache(IDictionary oldHashes) + => FileHashes = new(oldHashes); + [JsonPropertyName("file_hashes")] - public ConcurrentDictionary FileHashes { get; set; } = new(); + public ConcurrentDictionary FileHashes { get; set; } } diff --git a/src/tasks/WasmAppBuilder/EmccCompile.cs b/src/tasks/WasmAppBuilder/EmccCompile.cs index b0bc0ff80050b3..66d4743d81b4d4 100644 --- a/src/tasks/WasmAppBuilder/EmccCompile.cs +++ b/src/tasks/WasmAppBuilder/EmccCompile.cs @@ -125,31 +125,20 @@ private bool ExecuteActual() _tempPath = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); Directory.CreateDirectory(_tempPath); - int allowedParallelism = Math.Min(SourceFiles.Length, Environment.ProcessorCount); + int allowedParallelism = DisableParallelCompile ? 1 : Math.Min(SourceFiles.Length, Environment.ProcessorCount); if (BuildEngine is IBuildEngine9 be9) allowedParallelism = be9.RequestCores(allowedParallelism); - if (DisableParallelCompile || allowedParallelism == 1) + ParallelLoopResult result = Parallel.ForEach(filesToCompile, + new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, + (toCompile, state) => { - foreach ((string srcFile, string outFile) in filesToCompile) - { - if (!ProcessSourceFile(srcFile, outFile)) - return false; - } - } - else - { - ParallelLoopResult result = Parallel.ForEach(filesToCompile, - new ParallelOptions { MaxDegreeOfParallelism = allowedParallelism }, - (toCompile, state) => - { - if (!ProcessSourceFile(toCompile.Item1, toCompile.Item2)) - state.Stop(); - }); + if (!ProcessSourceFile(toCompile.Item1, toCompile.Item2)) + state.Stop(); + }); - if (!result.IsCompleted && !Log.HasLoggedErrors) - Log.LogError("Unknown failure occured while compiling. Check logs to get more details."); - } + if (!result.IsCompleted && !Log.HasLoggedErrors) + Log.LogError("Unknown failure occured while compiling. Check logs to get more details."); if (!Log.HasLoggedErrors) { From 7230cd3dd009de2f184f1ab55bbcf7c277e59562 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 22 Sep 2021 18:14:07 -0400 Subject: [PATCH 08/22] Don't set optimization defaults for Debug config, when publishing --- src/mono/wasm/build/WasmApp.Native.targets | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 2bcd4363ef12d3..03734454cc81e9 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -143,7 +143,7 @@ <_MonoAotCrossCompilerPath>@(MonoAotCrossCompiler->WithMetadataValue('RuntimeIdentifier','browser-wasm')) <_EmccDefaultFlagsRsp>$([MSBuild]::NormalizePath($(_WasmRuntimePackSrcDir), 'emcc-default.rsp')) - false + false true true $(WasmBuildNative) @@ -157,7 +157,7 @@ <_EmccAssertionLevelDefault>0 <_EmccOptimizationFlagDefault Condition="'$(_WasmDevel)' == 'true'">-O0 -s ASSERTIONS=$(_EmccAssertionLevelDefault) - <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' == 'Debug'">-O1 + <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == '' and '$(Configuration)' == 'Debug' and '$(WasmBuildingForNestedPublish)' != 'true'">-O1 <_EmccOptimizationFlagDefault Condition="'$(_EmccOptimizationFlagDefault)' == ''">-Oz $(_EmccOptimizationFlagDefault) From c71f9c257333eebc5159e1f48dc689e62d83e960 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 23 Sep 2021 16:25:28 -0400 Subject: [PATCH 09/22] Wasm.Build.Tests: Disable net5.0 because they can't be tested right now https://github.com/dotnet/runtime/issues/59538 --- src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmTests.cs index 6c7e3ed788be56..727e6589c4c6d2 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmTests.cs @@ -79,6 +79,7 @@ private CommandResult PublishForRequiresWorkloadTest(string config, string extra [Theory] [InlineData("Debug")] [InlineData("Release")] + [ActiveIssue("https://github.com/dotnet/runtime/issues/59538")] public void Net50Projects_NativeReference(string config) => BuildNet50Project(config, aot: false, expectError: true, @""); @@ -92,6 +93,7 @@ public void Net50Projects_NativeReference(string config) [Theory] [MemberData(nameof(Net50TestData))] + [ActiveIssue("https://github.com/dotnet/runtime/issues/59538")] public void Net50Projects_AOT(string config, bool aot, bool expectError) => BuildNet50Project(config, aot: aot, expectError: expectError); From 110ed85c3621febc183c282c02b0ca57a6efe137 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 23 Sep 2021 20:52:19 -0400 Subject: [PATCH 10/22] [wasm] Error for undefined symbols only when publishing For non-publish builds, undefined symbols will show up as warnings: ``` EXEC : warning : undefined symbol: sqlite3_column_database_name (referenced by top-level compiled C/C++ code) [/BlazorWebAssemblySqlite.csproj] EXEC : warning : undefined symbol: sqlite3_column_origin_name (referenced by top-level compiled C/C++ code) [/BlazorWebAssemblySqlite.csproj] EXEC : warning : undefined symbol: sqlite3_column_table_name (referenced by top-level compiled C/C++ code) [/BlazorWebAssemblySqlite.csproj] EXEC : warning : undefined symbol: sqlite3_snapshot_cmp (referenced by top-level compiled C/C++ code) [/BlazorWebAssemblySqlite.csproj] EXEC : warning : undefined symbol: sqlite3_snapshot_free (referenced by top-level compiled C/C++ code) [/BlazorWebAssemblySqlite.csproj] EXEC : warning : undefined symbol: sqlite3_snapshot_get (referenced by top-level compiled C/C++ code) [/BlazorWebAssemblySqlite.csproj] EXEC : warning : undefined symbol: sqlite3_snapshot_open (referenced by top-level compiled C/C++ code) [/BlazorWebAssemblySqlite.csproj] EXEC : warning : undefined symbol: sqlite3_snapshot_recover (referenced by top-level compiled C/C++ code) [/BlazorWebAssemblySqlite.csproj] ``` --- src/mono/wasm/build/WasmApp.Native.targets | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mono/wasm/build/WasmApp.Native.targets b/src/mono/wasm/build/WasmApp.Native.targets index 03734454cc81e9..f94ba6e66a4b44 100644 --- a/src/mono/wasm/build/WasmApp.Native.targets +++ b/src/mono/wasm/build/WasmApp.Native.targets @@ -206,6 +206,9 @@ <_EmccLDFlags Include="@(_EmccCommonFlags)" /> <_EmccLDFlags Include="-s TOTAL_MEMORY=$(EmccTotalMemory)" /> + + <_EmccLDFlags Include="-s ERROR_ON_UNDEFINED_SYMBOLS=0" Condition="'$(WasmBuildingForNestedPublish)' != 'true'" /> + <_DriverCDependencies Include="$(_WasmPInvokeHPath);$(_WasmICallTablePath)" /> <_DriverCDependencies Include="$(_DriverGenCPath)" Condition="'$(_DriverGenCNeeded)' == 'true'" /> From d17bc64fe47e6fd668ac1b138e49983a7f0acd44 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Thu, 23 Sep 2021 20:55:12 -0400 Subject: [PATCH 11/22] [wasm] PInvokeTableGenerator: Add support for variadic functions Currently, for a variadic function like: `int sqlite3_config(int, ...);` .. and multiple pinvokes like: ```csharp [DllImport(SQLITE_DLL, ExactSpelling=true, EntryPoint = "sqlite3_config", CallingConvention = CALLING_CONVENTION)] public static extern unsafe int sqlite3_config_none(int op); [DllImport(SQLITE_DLL, ExactSpelling=true, EntryPoint = "sqlite3_config", CallingConvention = CALLING_CONVENTION)] public static extern unsafe int sqlite3_config_int(int op, int val); [DllImport(SQLITE_DLL, ExactSpelling=true, EntryPoint = "sqlite3_config", CallingConvention = CALLING_CONVENTION)] public static extern unsafe int sqlite3_config_log(int op, NativeMethods.callback_log func, hook_handle pvUser); ``` .. we generate: ```c int sqlite3_config (int); int sqlite3_config (int,int); int sqlite3_config (int,int,int); ``` .. which fails to compile. Instead, this patch will generate a variadic declaration with one fixed parameter: ```c // Variadic signature created for // System.Int32 sqlite3_config_none(System.Int32) // System.Int32 sqlite3_config_int(System.Int32, System.Int32) // System.Int32 sqlite3_config_log(System.Int32, SQLitePCL.SQLite3Provider_e_sqlite3+NativeMethods+callback_log, SQLitePCL.hook_handle) int sqlite3_config (int, ...); ``` TODO: functions with different first argument --- .../WasmAppBuilder/PInvokeTableGenerator.cs | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index 825de67d6e2704..aeedc8995d1dcf 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -122,12 +122,23 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules w.WriteLine(); var decls = new HashSet(); - foreach (var pinvoke in pinvokes.OrderBy(l => l.EntryPoint)) + // FIXME: handle sigs with different first args + foreach (var group in pinvokes.OrderBy(l => l.EntryPoint).GroupBy(l => l.EntryPoint)) { + IEnumerable? uniqueSigs = group.Select(l => l.Method.ToString()).Distinct(); + bool treatAsVariadic = uniqueSigs.Count() > 1; + if (treatAsVariadic) + { + w.WriteLine($"// Variadic signature created for"); + foreach (string? method in uniqueSigs) + w.WriteLine($"// {method}"); + } + + PInvoke pinvoke = group.First(); if (modules.ContainsKey(pinvoke.Module)) { try { - var decl = GenPInvokeDecl(pinvoke); + var decl = GenPInvokeDecl(pinvoke, treatAsVariadic); if (decls.Contains(decl)) continue; @@ -205,7 +216,7 @@ private string MapType (Type t) return "int"; } - private string GenPInvokeDecl(PInvoke pinvoke) + private string GenPInvokeDecl(PInvoke pinvoke, bool treatAsVariadic=false) { var sb = new StringBuilder(); var method = pinvoke.Method; @@ -215,15 +226,25 @@ private string GenPInvokeDecl(PInvoke pinvoke) sb.Append($"int {pinvoke.EntryPoint} (int, int, int, int, int);"); return sb.ToString(); } + sb.Append(MapType(method.ReturnType)); sb.Append($" {pinvoke.EntryPoint} ("); - int pindex = 0; - var pars = method.GetParameters(); - foreach (var p in pars) { - if (pindex > 0) - sb.Append(','); - sb.Append(MapType(pars[pindex].ParameterType)); - pindex++; + if (!treatAsVariadic) + { + int pindex = 0; + var pars = method.GetParameters(); + foreach (var p in pars) { + if (pindex > 0) + sb.Append(','); + sb.Append(MapType(pars[pindex].ParameterType)); + pindex++; + } + } + else + { + ParameterInfo firstParam = method.GetParameters()[0]; + sb.Append(MapType(firstParam.ParameterType)); + sb.Append(", ..."); } sb.Append(");"); return sb.ToString(); From 8ecfd1cf2cc71cebc040abe551951cd15988035c Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Fri, 24 Sep 2021 17:16:45 -0400 Subject: [PATCH 12/22] [wasm] Handle pinvokes with function pointers - For pinvokes with function pointers, *no* declaration is added to `pinvoke-table.h`, and a warning is raised: ``` warning : DllImports with function pointers are not supported. Calling them will fail. Managed DllImports: [/Users/radical/dev/r2/artifacts/bin/Wasm.Build.Tests/net6.0-Release/browser-wasm/g3acrk4b.a0o/variadic_g3acrk4b.a0o.csproj] warning : Type: Test, Method: System.Int32 using_sum_one(?) [/Users/radical/dev/r2/artifacts/bin/Wasm.Build.Tests/net6.0-Release/browser-wasm/g3acrk4b.a0o/variadic_g3acrk4b.a0o.csproj] ``` - Also, handle multiple pinvokes with the same number of params, but different types --- .../WasmAppBuilder/PInvokeTableGenerator.cs | 69 +++++++++++--- .../Wasm.Build.Tests/NativeLibraryTests.cs | 89 +++++++++++++++++++ 2 files changed, 144 insertions(+), 14 deletions(-) diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index aeedc8995d1dcf..4aab6213eefa05 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -122,23 +122,39 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules w.WriteLine(); var decls = new HashSet(); - // FIXME: handle sigs with different first args - foreach (var group in pinvokes.OrderBy(l => l.EntryPoint).GroupBy(l => l.EntryPoint)) + foreach (var group in pinvokes.GroupBy(l => l.EntryPoint)) { - IEnumerable? uniqueSigs = group.Select(l => l.Method.ToString()).Distinct(); - bool treatAsVariadic = uniqueSigs.Count() > 1; - if (treatAsVariadic) + bool treatAsVariadic = false; + PInvoke first = group.First(); + + if (!ShouldApplyHackForMethodWithFunctionPointers(first.Method)) { - w.WriteLine($"// Variadic signature created for"); - foreach (string? method in uniqueSigs) - w.WriteLine($"// {method}"); + if (HasFunctionPointerParams(first.Method)) + { + Log.LogWarning($"DllImports with function pointers are not supported. Calling them will fail. Managed DllImports: {Environment.NewLine}{GroupToString(group)}"); + foreach (var pinvoke in group) + pinvoke.Skip = true; + + continue; + } + + int numArgs = first.Method.GetParameters().Length; + treatAsVariadic = group.Count() > 1 && group.Any(p => p.Method.GetParameters().Length != numArgs); + if (treatAsVariadic) + { + w.WriteLine($"// Variadic signature created for"); + foreach (PInvoke pinvoke in group) + w.WriteLine($"// {pinvoke.Method}"); + + Log.LogWarning($"Found a native function ({first.EntryPoint}) with varargs, which is not supported. Calling it will fail at runtime. Module: {first.Module}." + + $" Managed DllImports: {Environment.NewLine}{GroupToString(group)}"); + } } - PInvoke pinvoke = group.First(); - if (modules.ContainsKey(pinvoke.Module)) { + if (modules.ContainsKey(first.Module)) { try { - var decl = GenPInvokeDecl(pinvoke, treatAsVariadic); + var decl = GenPInvokeDecl(first, treatAsVariadic); if (decls.Contains(decl)) continue; @@ -148,8 +164,8 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules catch (NotSupportedException) { // See the FIXME in GenPInvokeDecl - Log.LogWarning($"Cannot handle function pointer arguments/return value in pinvoke method '{pinvoke.Method}' in type '{pinvoke.Method.DeclaringType}'."); - pinvoke.Skip = true; + Log.LogWarning($"Cannot handle function pointer arguments/return value in pinvoke method '{first.Method}' in type '{first.Method.DeclaringType}'."); + first.Skip = true; } } } @@ -197,6 +213,27 @@ static string ModuleNameToId(string name) return fixedName; } + + static bool HasFunctionPointerParams(MethodInfo method) + { + try + { + method.GetParameters(); + } + catch (NotSupportedException nse) when (nse.Message.Contains("function pointer types in signatures is not supported")) + { + return true; + } + catch + { + // not concerned with other exceptions + } + + return false; + } + + static string GroupToString(IGrouping group) + => string.Join(Environment.NewLine, group.Select(p => $" Type: {p.Method.DeclaringType}, Method: {p.Method}")); } private string MapType (Type t) @@ -216,11 +253,14 @@ private string MapType (Type t) return "int"; } + private static bool ShouldApplyHackForMethodWithFunctionPointers(MethodInfo method) => method.Name == "EnumCalendarInfo"; + private string GenPInvokeDecl(PInvoke pinvoke, bool treatAsVariadic=false) { var sb = new StringBuilder(); var method = pinvoke.Method; - if (method.Name == "EnumCalendarInfo") { + if (ShouldApplyHackForMethodWithFunctionPointers(method)) + { // FIXME: System.Reflection.MetadataLoadContext can't decode function pointer types // https://github.com/dotnet/runtime/issues/43791 sb.Append($"int {pinvoke.EntryPoint} (int, int, int, int, int);"); @@ -242,6 +282,7 @@ private string GenPInvokeDecl(PInvoke pinvoke, bool treatAsVariadic=false) } else { + // FIXME: handle sigs with different first args ParameterInfo firstParam = method.GetParameters()[0]; sb.Append(MapType(firstParam.ParameterType)); sb.Append(", ..."); diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs index 3ccf400ee92113..c06759ee2aa257 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.IO; using Xunit; using Xunit.Abstractions; @@ -91,5 +92,93 @@ public static int Main() Assert.Contains("Size: 26462 Height: 599, Width: 499", output); } + + [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] + [BuildAndRun(host: RunHost.V8)] + public void NativeLibraryWithVariadicFunctions(BuildArgs buildArgs, RunHost host, string id) + { + string projectName = $"variadic_{id}"; + string code = @" +using System; +using System.Runtime.InteropServices; +public class Test +{ + public static int Main(string[] args) + { + Console.WriteLine($""Main running""); + if (args.Length > 0) + { + // We don't want to run this, because we can't call variadic functions + Console.WriteLine($""sum_three: {sum_three(7, 14, 21)}""); + Console.WriteLine($""sum_two: {sum_two(3, 6)}""); + Console.WriteLine($""sum_one: {sum_one(5)}""); + } + return 42; + } + + [DllImport(""variadic"", EntryPoint=""sum"")] public static extern int sum_one(int a); + [DllImport(""variadic"", EntryPoint=""sum"")] public static extern int sum_two(int a, int b); + [DllImport(""variadic"", EntryPoint=""sum"")] public static extern int sum_three(int a, int b, int c); +}"; + string filename = "variadic.o"; + buildArgs = buildArgs with { ProjectName = projectName }; + buildArgs = ExpandBuildArgs(buildArgs, extraItems: $""); + + BuildProject(buildArgs, + initProject: () => + { + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), code); + File.Copy(Path.Combine(BuildEnvironment.TestAssetsPath, "native-libs", filename), + Path.Combine(_projectDir!, filename)); + }, + publish: false, + id: id, + dotnetWasmFromRuntimePack: false); + + string output = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + Assert.Contains("Main running", output); + } + + [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] + [BuildAndRun(host: RunHost.V8)] + public void DllImportWithFunctionPointersCompilesWithWarning(BuildArgs buildArgs, RunHost host, string id) + { + string projectName = $"variadic_{id}"; + string code = @" +using System; +using System.Runtime.InteropServices; +public class Test +{ + public static int Main() + { + Console.WriteLine($""Main running""); + return 42; + } + + [DllImport(""variadic"", EntryPoint=""sum"")] public unsafe static extern int using_sum_one(delegate* unmanaged callback); +}"; + string filename = "variadic.o"; + buildArgs = buildArgs with { ProjectName = projectName }; + buildArgs = ExpandBuildArgs(buildArgs, extraItems: $"", extraProperties: "true"); + + Console.WriteLine ($"-- args: {buildArgs}, name: {projectName}"); + + (_, string output) = BuildProject(buildArgs, + initProject: () => + { + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), code); + File.Copy(Path.Combine(BuildEnvironment.TestAssetsPath, "native-libs", filename), + Path.Combine(_projectDir!, filename)); + }, + publish: false, + id: id, + dotnetWasmFromRuntimePack: false); + + Assert.Matches("warning.*function pointers", output); + Assert.Matches("warning.*using_sum_one", output); + + output = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + Assert.Contains("Main running", output); + } } } From f4cb745813c1eaacc63122435cc765e77cdcc7a1 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 25 Sep 2021 01:02:34 -0400 Subject: [PATCH 13/22] [wasm] PInvokeTableGenerator: handle pinvokes with function pointers - pinvokes with function pointers could be for variadic functions too - if there are multiple pinvokes for the same native function, but some of them have function pointers, then ignore those but generate the declaration for the rest - raise better warnings - and emit info about the variadic pinvokes in pinvoke-table.h --- .../scenarios/BuildWasmAppsJobsList.txt | 1 + .../WasmAppBuilder/PInvokeTableGenerator.cs | 155 +++++++++++------- .../Wasm.Build.Tests/NativeLibraryTests.cs | 88 ---------- .../PInvokeTableGeneratorTests.cs | 139 ++++++++++++++++ 4 files changed, 235 insertions(+), 148 deletions(-) create mode 100644 src/tests/BuildWasmApps/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs diff --git a/eng/testing/scenarios/BuildWasmAppsJobsList.txt b/eng/testing/scenarios/BuildWasmAppsJobsList.txt index c4c5a1875c3bfc..8afeea6cdeeb13 100644 --- a/eng/testing/scenarios/BuildWasmAppsJobsList.txt +++ b/eng/testing/scenarios/BuildWasmAppsJobsList.txt @@ -11,6 +11,7 @@ Wasm.Build.Tests.LocalEMSDKTests Wasm.Build.Tests.MainWithArgsTests Wasm.Build.Tests.NativeBuildTests Wasm.Build.Tests.NativeLibraryTests +Wasm.Build.Tests.PInvokeTableGeneratorTests Wasm.Build.Tests.RebuildTests Wasm.Build.Tests.SatelliteAssembliesTests Wasm.Build.Tests.WasmBuildAppTest diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index 4aab6213eefa05..7264944fa5b571 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -3,13 +3,10 @@ using System; using System.Collections.Generic; -using System.Collections.Immutable; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Text; -using System.Text.Json; using System.Reflection; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -121,52 +118,42 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules w.WriteLine("// GENERATED FILE, DO NOT MODIFY"); w.WriteLine(); - var decls = new HashSet(); - foreach (var group in pinvokes.GroupBy(l => l.EntryPoint)) - { - bool treatAsVariadic = false; - PInvoke first = group.First(); + var pinvokesGroupedByEntryPoint = pinvokes.Where(l => modules.ContainsKey(l.Module)) + .OrderBy(l => l.EntryPoint) + .GroupBy(l => l.EntryPoint); - if (!ShouldApplyHackForMethodWithFunctionPointers(first.Method)) + var comparer = new PInvokeComparer(); + foreach (IGrouping group in pinvokesGroupedByEntryPoint) + { + var candidates = group.Distinct(comparer).ToArray(); + PInvoke first = candidates[0]; + if (ShouldTreatAsVariadic(candidates)) { - if (HasFunctionPointerParams(first.Method)) - { - Log.LogWarning($"DllImports with function pointers are not supported. Calling them will fail. Managed DllImports: {Environment.NewLine}{GroupToString(group)}"); - foreach (var pinvoke in group) - pinvoke.Skip = true; + Log.LogWarning($"Found a native function ({first.EntryPoint}) with varargs, which is not supported. Calling it will fail at runtime. Module: {first.Module}." + + $" Managed DllImports: {Environment.NewLine}{CandidatesToString(candidates)}"); - continue; - } - - int numArgs = first.Method.GetParameters().Length; - treatAsVariadic = group.Count() > 1 && group.Any(p => p.Method.GetParameters().Length != numArgs); - if (treatAsVariadic) + string? decl = GenPInvokeDecl(first, treatAsVariadic: true); + if (decl != null) { w.WriteLine($"// Variadic signature created for"); - foreach (PInvoke pinvoke in group) + foreach (PInvoke pinvoke in candidates) w.WriteLine($"// {pinvoke.Method}"); - Log.LogWarning($"Found a native function ({first.EntryPoint}) with varargs, which is not supported. Calling it will fail at runtime. Module: {first.Module}." + - $" Managed DllImports: {Environment.NewLine}{GroupToString(group)}"); + w.WriteLine(decl); } + + continue; } - if (modules.ContainsKey(first.Module)) { - try - { - var decl = GenPInvokeDecl(first, treatAsVariadic); - if (decls.Contains(decl)) - continue; + var decls = new HashSet(); + foreach (var candidate in candidates) + { + var decl = GenPInvokeDecl(candidate, treatAsVariadic: false); + if (decl == null || decls.Contains(decl)) + continue; - w.WriteLine(decl); - decls.Add(decl); - } - catch (NotSupportedException) - { - // See the FIXME in GenPInvokeDecl - Log.LogWarning($"Cannot handle function pointer arguments/return value in pinvoke method '{first.Method}' in type '{first.Method.DeclaringType}'."); - first.Skip = true; - } + w.WriteLine(decl); + decls.Add(decl); } } @@ -214,26 +201,24 @@ static string ModuleNameToId(string name) return fixedName; } - static bool HasFunctionPointerParams(MethodInfo method) + static bool ShouldTreatAsVariadic(PInvoke[] candidates) { - try - { - method.GetParameters(); - } - catch (NotSupportedException nse) when (nse.Message.Contains("function pointer types in signatures is not supported")) - { - return true; - } - catch - { - // not concerned with other exceptions - } - - return false; + if (candidates.Length < 2) + return false; + + PInvoke first = candidates[0]; + if (TryIsMethodGetParametersUnsupported(first.Method, out _)) + return false; + + int firstNumArgs = first.Method.GetParameters().Length; + return candidates + .Skip(1) + .Any(c => !TryIsMethodGetParametersUnsupported(c.Method, out _) && + c.Method.GetParameters().Length != firstNumArgs); } - static string GroupToString(IGrouping group) - => string.Join(Environment.NewLine, group.Select(p => $" Type: {p.Method.DeclaringType}, Method: {p.Method}")); + static string CandidatesToString(IEnumerable group) + => string.Join(Environment.NewLine, group); } private string MapType (Type t) @@ -253,20 +238,46 @@ private string MapType (Type t) return "int"; } - private static bool ShouldApplyHackForMethodWithFunctionPointers(MethodInfo method) => method.Name == "EnumCalendarInfo"; + // FIXME: System.Reflection.MetadataLoadContext can't decode function pointer types + // https://github.com/dotnet/runtime/issues/43791 + private static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotNullWhen(true)] out string? reason) + { + try + { + method.GetParameters(); + } + catch (NotSupportedException nse) + { + reason = nse.Message; + return true; + } + catch + { + // not concerned with other exceptions + } + + reason = null; + return false; + } - private string GenPInvokeDecl(PInvoke pinvoke, bool treatAsVariadic=false) + private string? GenPInvokeDecl(PInvoke pinvoke, bool treatAsVariadic) { var sb = new StringBuilder(); var method = pinvoke.Method; - if (ShouldApplyHackForMethodWithFunctionPointers(method)) - { + if (method.Name == "EnumCalendarInfo") { // FIXME: System.Reflection.MetadataLoadContext can't decode function pointer types // https://github.com/dotnet/runtime/issues/43791 sb.Append($"int {pinvoke.EntryPoint} (int, int, int, int, int);"); return sb.ToString(); } + if (TryIsMethodGetParametersUnsupported(pinvoke.Method, out string? reason)) + { + Log.LogWarning($"Skipping the following DllImport because '{reason}'. {Environment.NewLine} {pinvoke.Method}"); + pinvoke.Skip = true; + return null; + } + sb.Append(MapType(method.ReturnType)); sb.Append($" {pinvoke.EntryPoint} ("); if (!treatAsVariadic) @@ -428,7 +439,7 @@ private static bool IsBlittable (Type type) private static void Error (string msg) => throw new LogAsErrorException(msg); } -internal class PInvoke +internal class PInvoke : IEquatable { public PInvoke(string entryPoint, string module, MethodInfo method) { @@ -441,6 +452,30 @@ public PInvoke(string entryPoint, string module, MethodInfo method) public string Module; public MethodInfo Method; public bool Skip; + + public bool Equals(PInvoke? other) + => other != null && + string.Equals(EntryPoint, other.EntryPoint, StringComparison.Ordinal) && + string.Equals(Module, other.Module, StringComparison.Ordinal) && + string.Equals(Method.ToString(), other.Method.ToString(), StringComparison.Ordinal); + + public override string ToString() => $"{{ EntryPoint: {EntryPoint}, Module: {Module}, Method: {Method}, Skip: {Skip} }}"; +} + +internal class PInvokeComparer : IEqualityComparer +{ + public bool Equals(PInvoke? x, PInvoke? y) + { + if (x == null && y == null) + return true; + if (x == null || y == null) + return false; + + return x.Equals(y); + } + + public int GetHashCode(PInvoke pinvoke) + => $"{pinvoke.EntryPoint}{pinvoke.Module}{pinvoke.Method}".GetHashCode(); } internal class PInvokeCallback diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs index c06759ee2aa257..eb306286ba5e0d 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/NativeLibraryTests.cs @@ -92,93 +92,5 @@ public static int Main() Assert.Contains("Size: 26462 Height: 599, Width: 499", output); } - - [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] - [BuildAndRun(host: RunHost.V8)] - public void NativeLibraryWithVariadicFunctions(BuildArgs buildArgs, RunHost host, string id) - { - string projectName = $"variadic_{id}"; - string code = @" -using System; -using System.Runtime.InteropServices; -public class Test -{ - public static int Main(string[] args) - { - Console.WriteLine($""Main running""); - if (args.Length > 0) - { - // We don't want to run this, because we can't call variadic functions - Console.WriteLine($""sum_three: {sum_three(7, 14, 21)}""); - Console.WriteLine($""sum_two: {sum_two(3, 6)}""); - Console.WriteLine($""sum_one: {sum_one(5)}""); - } - return 42; - } - - [DllImport(""variadic"", EntryPoint=""sum"")] public static extern int sum_one(int a); - [DllImport(""variadic"", EntryPoint=""sum"")] public static extern int sum_two(int a, int b); - [DllImport(""variadic"", EntryPoint=""sum"")] public static extern int sum_three(int a, int b, int c); -}"; - string filename = "variadic.o"; - buildArgs = buildArgs with { ProjectName = projectName }; - buildArgs = ExpandBuildArgs(buildArgs, extraItems: $""); - - BuildProject(buildArgs, - initProject: () => - { - File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), code); - File.Copy(Path.Combine(BuildEnvironment.TestAssetsPath, "native-libs", filename), - Path.Combine(_projectDir!, filename)); - }, - publish: false, - id: id, - dotnetWasmFromRuntimePack: false); - - string output = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); - Assert.Contains("Main running", output); - } - - [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] - [BuildAndRun(host: RunHost.V8)] - public void DllImportWithFunctionPointersCompilesWithWarning(BuildArgs buildArgs, RunHost host, string id) - { - string projectName = $"variadic_{id}"; - string code = @" -using System; -using System.Runtime.InteropServices; -public class Test -{ - public static int Main() - { - Console.WriteLine($""Main running""); - return 42; - } - - [DllImport(""variadic"", EntryPoint=""sum"")] public unsafe static extern int using_sum_one(delegate* unmanaged callback); -}"; - string filename = "variadic.o"; - buildArgs = buildArgs with { ProjectName = projectName }; - buildArgs = ExpandBuildArgs(buildArgs, extraItems: $"", extraProperties: "true"); - - Console.WriteLine ($"-- args: {buildArgs}, name: {projectName}"); - - (_, string output) = BuildProject(buildArgs, - initProject: () => - { - File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), code); - File.Copy(Path.Combine(BuildEnvironment.TestAssetsPath, "native-libs", filename), - Path.Combine(_projectDir!, filename)); - }, - publish: false, - id: id, - dotnetWasmFromRuntimePack: false); - - Assert.Matches("warning.*function pointers", output); - Assert.Matches("warning.*using_sum_one", output); - - output = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); - Assert.Contains("Main running", output); - } } } diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs new file mode 100644 index 00000000000000..27783ae85c95ad --- /dev/null +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs @@ -0,0 +1,139 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using Xunit; +using Xunit.Abstractions; + +#nullable enable + +namespace Wasm.Build.Tests +{ + public class PInvokeTableGeneratorTests : BuildTestBase + { + public PInvokeTableGeneratorTests(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) + : base(output, buildContext) + { + } + + [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] + [BuildAndRun(host: RunHost.V8)] + public void NativeLibraryWithVariadicFunctions(BuildArgs buildArgs, RunHost host, string id) + { + string code = @" + using System; + using System.Runtime.InteropServices; + public class Test + { + public static int Main(string[] args) + { + Console.WriteLine($""Main running""); + if (args.Length > 0) + { + // We don't want to run this, because we can't call variadic functions + Console.WriteLine($""sum_three: {sum_three(7, 14, 21)}""); + Console.WriteLine($""sum_two: {sum_two(3, 6)}""); + Console.WriteLine($""sum_one: {sum_one(5)}""); + } + return 42; + } + + [DllImport(""variadic"", EntryPoint=""sum"")] public static extern int sum_one(int a); + [DllImport(""variadic"", EntryPoint=""sum"")] public static extern int sum_two(int a, int b); + [DllImport(""variadic"", EntryPoint=""sum"")] public static extern int sum_three(int a, int b, int c); + }"; + + (buildArgs, string output) = BuildForVariadicFunctionTests(code, + buildArgs with { ProjectName = $"variadic_{buildArgs.Config}_{id}" }, + id); + Assert.Matches("warning.*native function.*sum.*varargs", output); + Assert.Matches("warning.*sum_(one|two|three)", output); + + output = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + Assert.Contains("Main running", output); + } + + [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] + [BuildAndRun(host: RunHost.V8)] + public void DllImportWithFunctionPointersCompilesWithWarning(BuildArgs buildArgs, RunHost host, string id) + { + string code = @" + using System; + using System.Runtime.InteropServices; + public class Test + { + public static int Main() + { + Console.WriteLine($""Main running""); + return 42; + } + + [DllImport(""variadic"", EntryPoint=""sum"")] + public unsafe static extern int using_sum_one(delegate* unmanaged callback); + + [DllImport(""variadic"", EntryPoint=""sum"")] + public static extern int sum_one(int a, int b); + }"; + + (buildArgs, string output) = BuildForVariadicFunctionTests(code, + buildArgs with { ProjectName = $"fnptr_{buildArgs.Config}_{id}" }, + id); + Assert.Matches("warning.*Skipping.*because.*function pointer", output); + Assert.Matches("warning.*using_sum_one", output); + + output = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + Assert.Contains("Main running", output); + } + + [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] + [BuildAndRun(host: RunHost.V8)] + public void DllImportWithFunctionPointers_ForVariadicFunction_CompilesWithWarning(BuildArgs buildArgs, RunHost host, string id) + { + string code = @" + using System; + using System.Runtime.InteropServices; + public class Test + { + public static int Main() + { + Console.WriteLine($""Main running""); + return 42; + } + + [DllImport(""variadic"", EntryPoint=""sum"")] + public unsafe static extern int using_sum_one(delegate* unmanaged callback); + }"; + + (buildArgs, string output) = BuildForVariadicFunctionTests(code, + buildArgs with { ProjectName = $"fnptr_variadic_{buildArgs.Config}_{id}" }, + id); + Assert.Matches("warning.*Skipping.*because.*function pointer", output); + Assert.Matches("warning.*using_sum_one", output); + + output = RunAndTestWasmApp(buildArgs, buildDir: _projectDir, expectedExitCode: 42, host: host, id: id); + Assert.Contains("Main running", output); + } + + private (BuildArgs, string) BuildForVariadicFunctionTests(string programText, BuildArgs buildArgs, string id) + { + string filename = "variadic.o"; + buildArgs = ExpandBuildArgs(buildArgs, + extraItems: $"", + extraProperties: "true<_WasmDevel>true"); + + (_, string output) = BuildProject(buildArgs, + initProject: () => + { + File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), programText); + File.Copy(Path.Combine(BuildEnvironment.TestAssetsPath, "native-libs", filename), + Path.Combine(_projectDir!, filename)); + }, + publish: buildArgs.AOT, + id: id, + dotnetWasmFromRuntimePack: false); + + return (buildArgs, output); + } + } +} From aac1537eaa9d8aac370f0bc8972c09ca0069c780 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 25 Sep 2021 01:03:48 -0400 Subject: [PATCH 14/22] add missing variadic.{c,o} --- .../testassets/native-libs/variadic.c | 14 ++++++++++++++ .../testassets/native-libs/variadic.o | Bin 0 -> 549 bytes 2 files changed, 14 insertions(+) create mode 100644 src/tests/BuildWasmApps/testassets/native-libs/variadic.c create mode 100644 src/tests/BuildWasmApps/testassets/native-libs/variadic.o diff --git a/src/tests/BuildWasmApps/testassets/native-libs/variadic.c b/src/tests/BuildWasmApps/testassets/native-libs/variadic.c new file mode 100644 index 00000000000000..cd4009439d19da --- /dev/null +++ b/src/tests/BuildWasmApps/testassets/native-libs/variadic.c @@ -0,0 +1,14 @@ +#include + +int sum(int n, ...) +{ + int result = 0; + va_list ptr; + va_start(ptr, n); + + for (int i = 0; i < n; i++) + result += va_arg(ptr, int); + + va_end(ptr); + return result; +} diff --git a/src/tests/BuildWasmApps/testassets/native-libs/variadic.o b/src/tests/BuildWasmApps/testassets/native-libs/variadic.o new file mode 100644 index 0000000000000000000000000000000000000000..b4558ce351979341e3109ac1861ae44fea2c41b6 GIT binary patch literal 549 zcmXYu%Wm306ozNUCI)N*@;*#)(cy6YoQGaiABFVu##SwXvLrEa?J z8+6sDs?^8m8B%F9^Pm5m`OaKeP^O$Q#=q`%JI3FO?UrxFZyg9jWlzoNG>I}56w_3t zdGRb5`)`-)KrW}NJj&LpFt*(IvL~?m-_M%#hx5+%hO8Y!F=?Rz0*ob^U;^`nGhtXE zS@A`6V&VbGgBUHaunJY+zF<^PBezCUJQxpP-U^0wy*Ow?9UZW-0S+|Y;^Bm26Aq#I z0b9^|Y;av=Vw=E6(1xQJkBM~b3lrL!a-#E9JcUy@iSdkbXLO904(G(WfOEKr@sbjk z)ZSsXO{gmhuVU;#hm_ub4cASZ{nR(K=#7?9HW$_JIuy)gM!88vi?JWts23{9rThBb zn@59N7H7Zo8`V}t9&V&6O74)9sw`F5ojvo4OoD9wjrWIc&+Wmjm#@}d>3#H~!jpkq zC~qFEJ-I0IG}@%@JkRHeD)UVtl`HeKt3J_}WI@uUvc+b$_mbo(?e3>8^@^2T!)QDj jjK+h}a4;Jx8o@XWM*X0#26uyS9QOKmvvBCk`&a(}ARK*H literal 0 HcmV?d00001 From f48d6b20b939c7e649da26db9f4abb6878eb8ecc Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Sat, 25 Sep 2021 17:25:35 -0400 Subject: [PATCH 15/22] [wasm] Add test for issue dotnet#59255 --- .../BlazorWasmBuildPublishTests.cs | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmBuildPublishTests.cs b/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmBuildPublishTests.cs index 66b5c5b2782766..d4fb7cf49875cf 100644 --- a/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmBuildPublishTests.cs +++ b/src/tests/BuildWasmApps/Wasm.Build.Tests/BlazorWasmBuildPublishTests.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; +using System.Text.RegularExpressions; using Xunit; using Xunit.Abstractions; @@ -170,6 +171,60 @@ public void WithNativeReference_AOTOnCommandLine(string config) BlazorPublish(id, config, expectedFileType: NativeFilesType.Relinked); } + [ConditionalTheory(typeof(BuildTestBase), nameof(IsUsingWorkloads))] + [InlineData("Debug")] + [InlineData("Release")] + public void WithDllImportInMainAssembly(string config) + { + // Based on https://github.com/dotnet/runtime/issues/59255 + string id = $"blz_dllimp_{config}"; + string projectFile = CreateProjectWithNativeReference(id); + string nativeSource = @" + #include + + extern ""C"" { + int cpp_add(int a, int b) { + return a + b; + } + }"; + + File.WriteAllText(Path.Combine(_projectDir!, "mylib.cpp"), nativeSource); + + string myDllImportCs = @$" + using System.Runtime.InteropServices; + namespace {id}; + + public static class MyDllImports + {{ + [DllImport(""mylib"")] + public static extern int cpp_add(int a, int b); + }}"; + + File.WriteAllText(Path.Combine(_projectDir!, "Pages", "MyDllImport.cs"), myDllImportCs); + + AddItemsPropertiesToProject(projectFile, extraItems: @""); + + BlazorBuild(id, config, expectedFileType: NativeFilesType.Relinked); + CheckNativeFileLinked(forPublish: false); + + BlazorPublish(id, config, expectedFileType: NativeFilesType.Relinked); + CheckNativeFileLinked(forPublish: true); + + void CheckNativeFileLinked(bool forPublish) + { + // very crude way to check that the native file was linked in + // needed because we don't run the blazor app yet + string objBuildDir = Path.Combine(_projectDir!, "obj", config, "net6.0", "wasm", forPublish ? "for-publish" : "for-build"); + string pinvokeTableHPath = Path.Combine(objBuildDir, "pinvoke-table.h"); + Assert.True(File.Exists(pinvokeTableHPath), $"Could not find {pinvokeTableHPath}"); + + string pinvokeTableHContents = File.ReadAllText(pinvokeTableHPath); + string pattern = $"\"cpp_add\".*{id}"; + Assert.True(Regex.IsMatch(pinvokeTableHContents, pattern), + $"Could not find {pattern} in {pinvokeTableHPath}"); + } + } + private string CreateProjectWithNativeReference(string id) { CreateBlazorWasmTemplateProject(id); From 9dd19763e1a06da1ae438dab69b1eccada9ab4a0 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 22 Sep 2021 13:16:41 -0400 Subject: [PATCH 16/22] Bump sdk for workload testing to 6.0.100-rc.2.21474.31 .. and remove the Blazor sdk targets hack used for testing, since the updated SDK has the fixes. --- eng/Versions.props | 2 +- src/libraries/workloads-testing.targets | 2 - src/mono/wasm/BlazorOverwrite.targets | 741 ------------------------ 3 files changed, 1 insertion(+), 744 deletions(-) delete mode 100644 src/mono/wasm/BlazorOverwrite.targets diff --git a/eng/Versions.props b/eng/Versions.props index a7c00e5d82ee79..7b78ac2770c964 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -166,7 +166,7 @@ 2.0.4 4.12.0 2.14.3 - 6.0.100-rc.2.21463.12 + 6.0.100-rc.2.21474.31 6.0.0-preview-20210916.1 diff --git a/src/libraries/workloads-testing.targets b/src/libraries/workloads-testing.targets index d3dc57267a608a..6a8fa5055bbbb3 100644 --- a/src/libraries/workloads-testing.targets +++ b/src/libraries/workloads-testing.targets @@ -16,7 +16,6 @@ - @@ -41,7 +40,6 @@ - diff --git a/src/mono/wasm/BlazorOverwrite.targets b/src/mono/wasm/BlazorOverwrite.targets deleted file mode 100644 index a276d385723725..00000000000000 --- a/src/mono/wasm/BlazorOverwrite.targets +++ /dev/null @@ -1,741 +0,0 @@ - - - - - true - - - true - - - - - $(MSBuildThisFileDirectory)..\ - <_BlazorWebAssemblySdkTasksTFM Condition=" '$(MSBuildRuntimeType)' == 'Core'">net6.0 - <_BlazorWebAssemblySdkTasksTFM Condition=" '$(MSBuildRuntimeType)' != 'Core'">net472 - <_BlazorWebAssemblySdkTasksAssembly>$(BlazorWebAssemblySdkDirectoryRoot)tools\$(_BlazorWebAssemblySdkTasksTFM)\Microsoft.NET.Sdk.BlazorWebAssembly.Tasks.dll - <_BlazorWebAssemblySdkToolAssembly>$(BlazorWebAssemblySdkDirectoryRoot)tools\net6.0\Microsoft.NET.Sdk.BlazorWebAssembly.Tool.dll - - - - - - - - - - - - - - true - true - - - false - false - true - false - false - false - <_AggressiveAttributeTrimming Condition="'$(_AggressiveAttributeTrimming)' == ''">true - false - true - - - false - false - false - false - true - - - false - - <_TargetingNET60OrLater Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp' AND $([MSBuild]::VersionGreaterThanOrEquals('$(TargetFrameworkVersion)', '6.0'))">true - - - - false - - true - true - false - ComputeFilesToPublish;_GatherWasmFilesToPublish;$(WasmNestedPublishAppDependsOn) - <_ScrambleDotnetJsFileNameAfterThisTarget Condition="'$(UsingBrowserRuntimeWorkload)' != 'true'">ResolveRuntimePackAssets - <_ScrambleDotnetJsFileNameAfterThisTarget Condition="'$(UsingBrowserRuntimeWorkload)' == 'true'">WasmBuildApp - - - Publish - - - - - - - - - - - - $(ResolveStaticWebAssetsInputsDependsOn); - _AddBlazorWasmStaticWebAssets; - - - - _GenerateBuildBlazorBootJson; - $(StaticWebAssetsPrepareForRunDependsOn) - - - - $(ResolvePublishStaticWebAssetsDependsOn); - ProcessPublishFilesForBlazor; - ComputeBlazorExtensions; - _AddPublishBlazorBootJsonToStaticWebAssets; - - - - $(GenerateStaticWebAssetsPublishManifestDependsOn); - GeneratePublishBlazorBootJson; - - - - - - - - - - <_DotNetJsVersion>$(BundledNETCoreAppPackageVersion) - <_DotNetJsVersion Condition="'$(RuntimeFrameworkVersion)' != ''">$(RuntimeFrameworkVersion) - <_BlazorDotnetJsFileName>dotnet.$(_DotNetJsVersion).js - - - - <_DotNetJsItem Remove="@(_DotNetJsItem)" /> - <_DotNetJsItem Include="@(WasmNativeAsset)" Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.js'"> - _framework/$(_BlazorDotnetJsFileName) - native - true - - - - - <_DotnetJsStaticWebAssetCandidate Remove="@(_DotnetJsStaticWebAssetCandidate)" /> - <_DotnetJsCopyCandidates Remove="@(_DotnetJsCopyCandidates)" /> - - - - - - - - - - - - <_DotnetJsStaticWebAsset Include="@(_DotnetJsStaticWebAssetCandidate->'%(ContentRoot)_framework\dotnet.js')" /> - <_BlazorStaticWebAsset Include="@(_DotnetJsStaticWebAsset)" /> - - - - - - - <_DotNetJsVersion>$(BundledNETCoreAppPackageVersion) - <_DotNetJsVersion Condition="'$(RuntimeFrameworkVersion)' != ''">$(RuntimeFrameworkVersion) - <_BlazorDotnetJsFileName>dotnet.$(_DotNetJsVersion).js - - - - <_DotNetJsItem Include="@(ReferenceCopyLocalPaths)" Condition="'%(ReferenceCopyLocalPaths.DestinationSubPath)' == 'dotnet.js' AND '%(ReferenceCopyLocalPaths.AssetType)' == 'native'"> - _framework/$(_BlazorDotnetJsFileName) - - - - - <_DotNetJsItem Remove="@(_DotNetJsItem)" /> - <_DotNetJsItem Include="@(WasmNativeAsset)" Condition="'%(WasmNativeAsset.FileName)%(WasmNativeAsset.Extension)' == 'dotnet.js'"> - _framework/$(_BlazorDotnetJsFileName) - native - true - - - - - - - - - - - - - <_DotnetJsStaticWebAsset Include="@(_DotnetJsStaticWebAssetCandidate->'%(ContentRoot)_framework\dotnet.js')" /> - <_BlazorStaticWebAsset Include="@(_DotnetJsStaticWebAsset)" /> - - - - - - - - - - - - - - - - <_BlazorEnableTimeZoneSupport>$(BlazorEnableTimeZoneSupport) - <_BlazorEnableTimeZoneSupport Condition="'$(_BlazorEnableTimeZoneSupport)' == ''">true - <_BlazorInvariantGlobalization>$(InvariantGlobalization) - <_BlazorInvariantGlobalization Condition="'$(_BlazorInvariantGlobalization)' == ''">true - <_BlazorCopyOutputSymbolsToOutputDirectory>$(CopyOutputSymbolsToOutputDirectory) - <_BlazorCopyOutputSymbolsToOutputDirectory Condition="'$(_BlazorCopyOutputSymbolsToOutputDirectory)'==''">true - <_BlazorWebAssemblyLoadAllGlobalizationData>$(BlazorWebAssemblyLoadAllGlobalizationData) - <_BlazorWebAssemblyLoadAllGlobalizationData Condition="'$(_BlazorWebAssemblyLoadAllGlobalizationData)' == ''">false - - - $(OutputPath)$(PublishDirName)\ - - - - - - <_BlazorJSFile Include="$(BlazorWebAssemblyJSPath)" /> - <_BlazorJSFile Include="$(BlazorWebAssemblyJSMapPath)" Condition="Exists('$(BlazorWebAssemblyJSMapPath)')" /> - <_BlazorJsFile> - _framework/%(Filename)%(Extension) - - - - - - <_BlazorConfigFileCandidates Include="@(StaticWebAsset)" Condition="'%(SourceType)' == 'Discovered'" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_BlazorBuildGZipCompressDirectory>$(IntermediateOutputPath)build-gz\ - - - - - - - - - - - - - - - - <_BlazorBuildGZipCompressedFile> - %(RelatedAsset) - - - <_BlazorGzipStaticWebAsset Include="@(_BlazorBuildGZipCompressedFile->'%(FullPath)')" /> - - - - <_BlazorBuildBootJsonPath>$(IntermediateOutputPath)blazor.boot.json - - - - <_BuildBlazorBootJson - Include="$(_BlazorBuildBootJsonPath)" - RelativePath="_framework/blazor.boot.json" /> - - - - - - - - - - - - - - - - - - - - <_BlazorBuildBootJsonPath>$(IntermediateOutputPath)blazor.boot.json - <_BlazorWebAssemblyLoadAllGlobalizationData Condition="'$(BlazorWebAssemblyLoadAllGlobalizationData)' == ''">false - - - - <_BlazorJsModuleCandidatesForBuild - Include="@(StaticWebAsset)" - Condition="'%(StaticWebAsset.AssetTraitName)' == 'JSModule' and '%(StaticWebAsset.AssetTraitValue)' == 'JSLibraryModule' and '%(AssetKind)' != 'Publish'" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_BlazorTypeGranularTrimmerDescriptorFile>$(IntermediateOutputPath)typegranularity.trimmerdescriptor.xml - - - - <_BlazorTypeGranularAssembly - Include="@(ManagedAssemblyToLink)" - Condition="'%(Extension)' == '.dll' AND $([System.String]::Copy('%(Filename)').StartsWith('Microsoft.AspNetCore.'))"> - false - all - - - - - - - - - - - - - - - - - <_BlazorAotEnabled>$(UsingBrowserRuntimeWorkload) - <_BlazorAotEnabled Condition="'$(_BlazorAotEnabled)' == ''">false - <_BlazorLinkerEnabled>$(PublishTrimmed) - <_BlazorLinkerEnabled Condition="'$(_BlazorLinkerEnabled)' == ''">true - - - - - - <_BlazorPublishPrefilteredAssets - Include="@(StaticWebAsset)" - Condition="'%(StaticWebAsset.AssetTraitName)' == 'BlazorWebAssemblyResource' or '%(StaticWebAsset.AssetTraitName)' == 'Culture' or '%(AssetRole)' == 'Alternative'" /> - - - - - - - - - - - - - - - - - <_BlazorExtensionsCandidate Include="@(BlazorPublishExtension->'%(FullPath)')"> - $(PackageId) - Computed - $(PublishDir)wwwroot - $(StaticWebAssetBasePath) - %(BlazorPublishExtension.RelativePath) - Publish - All - Primary - BlazorWebAssemblyResource - extension:%(BlazorPublishExtension.ExtensionName) - Never - PreserveNewest - %(BlazorPublishExtension.Identity) - - - - - - - - - - - - - - - - - - - <_PublishBlazorBootJson - Include="$(IntermediateOutputPath)blazor.publish.boot.json" - RelativePath="_framework/blazor.boot.json" /> - - - - - - - - - - - <_BlazorPublishAsset - Include="@(StaticWebAsset)" - Condition="'%(AssetKind)' != 'Build' and '%(StaticWebAsset.AssetTraitValue)' != 'manifest' and ('%(StaticWebAsset.AssetTraitName)' == 'BlazorWebAssemblyResource' or '%(StaticWebAsset.AssetTraitName)' == 'Culture') and '%(StaticWebAsset.AssetTraitValue)' != 'boot'" /> - - <_BlazorPublishConfigFile - Include="@(StaticWebAsset)" - Condition="'%(StaticWebAsset.AssetTraitName)' == 'BlazorWebAssemblyResource' and '%(StaticWebAsset.AssetTraitValue)' == 'settings'"/> - - <_BlazorJsModuleCandidatesForPublish - Include="@(StaticWebAsset)" - Condition="'%(StaticWebAsset.AssetTraitName)' == 'JSModule' and '%(StaticWebAsset.AssetTraitValue)' == 'JSLibraryModule' and '%(AssetKind)' != 'Build'" /> - - - <_BlazorPublishAsset Remove="@(_BlazorExtensionsCandidatesForPublish)" /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - <_CompressedFileOutputPath>$(IntermediateOutputPath)compress\ - <_BlazorWebAssemblyBrotliIncremental>true - - - - <_DotNetHostDirectory>$(NetCoreRoot) - <_DotNetHostFileName>dotnet - <_DotNetHostFileName Condition="'$(OS)' == 'Windows_NT'">dotnet.exe - - - - - - - - <_GzipFileToCompressForPublish Include="@(StaticWebAsset)" - Condition="'%(AssetKind)' != 'Build' and ('%(StaticWebAsset.AssetTraitName)' == 'BlazorWebAssemblyResource' or '%(StaticWebAsset.AssetTraitName)' == 'Culture')" > - %(Identity) - Alternative - Content-Encoding - gzip - - - <_BrotliFileToCompressForPublish Include="@(_GzipFileToCompressForPublish)" Condition="'%(AssetKind)' != 'Build'"> - br - - - - <_AlreadyGzipCompressedAssets - Include="@(StaticWebAsset)" - Condition="'%(AssetKind)' != 'Build' and ('%(StaticWebAsset.AssetTraitName)' == 'Content-Encoding' and '%(StaticWebAsset.AssetTraitValue)' == 'gzip')" /> - <_GzipFileToCompressForPublish Remove="@(_AlreadyGzipCompressedAssets->'%(RelatedAsset)')" /> - - - - - - - - - - - - - - - - <_BlazorPublishGZipCompressedFile> - %(RelatedAsset) - - <_BlazorPublishBrotliCompressedFile> - %(RelatedAsset) - - - - - - - - From 712dbb3280ed84d212410c77a43b4be9593fb372 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Mon, 27 Sep 2021 13:07:12 -0400 Subject: [PATCH 17/22] MonoAOTCompiler: Check the hash for the file also, for "all up-to-date" check --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 5300a35a2f4ebf..cd25ab41e287fc 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -418,8 +418,13 @@ private bool CheckAllUpToDate(IList argsList) { // compare original assembly vs it's outputs.. all it's outputs! string assemblyPath = args.AOTAssembly.GetMetadata("FullPath"); - if (args.ProxyFiles.Any(pf => IsNewerThanOutput(assemblyPath, pf.TargetFile))) + string assemblyHash = Utils.ComputeHash(assemblyPath); + + if (args.ProxyFiles.Any(pf => IsNewerThanOutput(assemblyPath, pf.TargetFile) && + Utils.ComputeHash(pf.TargetFile) != assemblyHash)) + { return false; + } } return true; From dc09bf6124c5e8186b4ebdbfbf8341af8a337bfc Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Mon, 27 Sep 2021 16:14:03 -0400 Subject: [PATCH 18/22] Revert "MonoAOTCompiler: Check the hash for the file also, for "all up-to-date" check" Newer timestamp, but unchanged file is an indication of a build issue, and doesn't need to be handled in this task. This reverts commit 712dbb3280ed84d212410c77a43b4be9593fb372. --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index cd25ab41e287fc..5300a35a2f4ebf 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -418,13 +418,8 @@ private bool CheckAllUpToDate(IList argsList) { // compare original assembly vs it's outputs.. all it's outputs! string assemblyPath = args.AOTAssembly.GetMetadata("FullPath"); - string assemblyHash = Utils.ComputeHash(assemblyPath); - - if (args.ProxyFiles.Any(pf => IsNewerThanOutput(assemblyPath, pf.TargetFile) && - Utils.ComputeHash(pf.TargetFile) != assemblyHash)) - { + if (args.ProxyFiles.Any(pf => IsNewerThanOutput(assemblyPath, pf.TargetFile))) return false; - } } return true; From 41c7476775ac81361157879a567857df4b42db43 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Mon, 27 Sep 2021 19:58:23 -0400 Subject: [PATCH 19/22] PInvokeTableGenerator: don't generate any decl for variadic functions Based on feedback from Zoltan --- .../WasmAppBuilder/PInvokeTableGenerator.cs | 47 ++++++------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index 7264944fa5b571..de66de13177438 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -129,18 +129,12 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules PInvoke first = candidates[0]; if (ShouldTreatAsVariadic(candidates)) { - Log.LogWarning($"Found a native function ({first.EntryPoint}) with varargs, which is not supported. Calling it will fail at runtime. Module: {first.Module}." + - $" Managed DllImports: {Environment.NewLine}{CandidatesToString(candidates)}"); + string imports = string.Join(Environment.NewLine, candidates.Select(p => $" {p.Method}")); + Log.LogWarning($"Found a native function ({first.EntryPoint}) with varargs, which is not supported. Calling it will fail at runtime. Native library: {first.Module}." + + $" Managed DllImports: {Environment.NewLine}{imports}"); - string? decl = GenPInvokeDecl(first, treatAsVariadic: true); - if (decl != null) - { - w.WriteLine($"// Variadic signature created for"); - foreach (PInvoke pinvoke in candidates) - w.WriteLine($"// {pinvoke.Method}"); - - w.WriteLine(decl); - } + foreach (var c in candidates) + c.Skip = true; continue; } @@ -148,7 +142,7 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules var decls = new HashSet(); foreach (var candidate in candidates) { - var decl = GenPInvokeDecl(candidate, treatAsVariadic: false); + var decl = GenPInvokeDecl(candidate); if (decl == null || decls.Contains(decl)) continue; @@ -216,9 +210,6 @@ static bool ShouldTreatAsVariadic(PInvoke[] candidates) .Any(c => !TryIsMethodGetParametersUnsupported(c.Method, out _) && c.Method.GetParameters().Length != firstNumArgs); } - - static string CandidatesToString(IEnumerable group) - => string.Join(Environment.NewLine, group); } private string MapType (Type t) @@ -260,7 +251,7 @@ private static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotN return false; } - private string? GenPInvokeDecl(PInvoke pinvoke, bool treatAsVariadic) + private string? GenPInvokeDecl(PInvoke pinvoke) { var sb = new StringBuilder(); var method = pinvoke.Method; @@ -280,23 +271,13 @@ private static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotN sb.Append(MapType(method.ReturnType)); sb.Append($" {pinvoke.EntryPoint} ("); - if (!treatAsVariadic) - { - int pindex = 0; - var pars = method.GetParameters(); - foreach (var p in pars) { - if (pindex > 0) - sb.Append(','); - sb.Append(MapType(pars[pindex].ParameterType)); - pindex++; - } - } - else - { - // FIXME: handle sigs with different first args - ParameterInfo firstParam = method.GetParameters()[0]; - sb.Append(MapType(firstParam.ParameterType)); - sb.Append(", ..."); + int pindex = 0; + var pars = method.GetParameters(); + foreach (var p in pars) { + if (pindex > 0) + sb.Append(','); + sb.Append(MapType(pars[pindex].ParameterType)); + pindex++; } sb.Append(");"); return sb.ToString(); From dcbcc3550ee781f263a11be62320c4639efc20e9 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Sep 2021 12:40:46 -0400 Subject: [PATCH 20/22] Add missing `using` for disposable objects. --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 3 ++- src/tasks/WasmAppBuilder/IcallTableGenerator.cs | 9 +++------ src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 5300a35a2f4ebf..8454bad78b7823 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -444,7 +444,8 @@ private IList FilterAssemblies(IEnumerable assemblies) } string assemblyPath = asmItem.GetMetadata("FullPath"); - PEReader reader = new(File.OpenRead(assemblyPath), PEStreamOptions.Default); + using var assemblyFile = File.OpenRead(assemblyPath); + using PEReader reader = new(assemblyFile, PEStreamOptions.Default); if (!reader.HasMetadata) { Log.LogWarning($"Skipping unmanaged {assemblyPath} for AOT"); diff --git a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs index cc4d8af1ac7faa..b218334db66bc4 100644 --- a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs @@ -45,7 +45,7 @@ public void GenIcallTable(string runtimeIcallTableFile, string[] assemblies) { ReadTable (runtimeIcallTableFile); var resolver = new PathAssemblyResolver(assemblies); - var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); + using var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); foreach (var aname in assemblies) { var a = mlc.LoadFromAssemblyPath(aname); @@ -107,11 +107,8 @@ private void EmitTable (StreamWriter w) // Read the icall table generated by mono --print-icall-table private void ReadTable (string filename) { - JsonDocument json; - using (var stream = File.Open (filename, FileMode.Open)) - { - json = JsonDocument.Parse (stream); - } + using var stream = File.Open (filename, FileMode.Open); + using JsonDocument json = JsonDocument.Parse (stream); var arr = json.RootElement; foreach (var v in arr.EnumerateArray ()) diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index de66de13177438..af4d30faccc9f4 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -62,7 +62,7 @@ public void GenPInvokeTable(string[] pinvokeModules, string[] assemblies) var callbacks = new List(); var resolver = new PathAssemblyResolver(assemblies); - var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); + using var mlc = new MetadataLoadContext(resolver, "System.Private.CoreLib"); foreach (var aname in assemblies) { var a = mlc.LoadFromAssemblyPath(aname); From 1f57975d60afea74ae9263bc66198d207d55e394 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Sep 2021 12:41:13 -0400 Subject: [PATCH 21/22] cleanup --- src/tasks/AotCompilerTask/MonoAOTCompiler.cs | 4 ---- src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs | 14 +++++++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs index 8454bad78b7823..230e8ce3664729 100644 --- a/src/tasks/AotCompilerTask/MonoAOTCompiler.cs +++ b/src/tasks/AotCompilerTask/MonoAOTCompiler.cs @@ -928,11 +928,7 @@ private static IList ConvertAssembliesDictToOrderedList(ConcurrentDic foreach (ITaskItem item in originalAssemblies) { if (dict.TryGetValue(item.GetMetadata("FullPath"), out ITaskItem? dictItem)) - { - // throw new LogAsErrorException($"Bug: Could not find item in the dict with key {item.ItemSpec}"); - outItems.Add(dictItem); - } } return outItems; } diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index af4d30faccc9f4..feb2dda21d8528 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -118,9 +118,10 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules w.WriteLine("// GENERATED FILE, DO NOT MODIFY"); w.WriteLine(); - var pinvokesGroupedByEntryPoint = pinvokes.Where(l => modules.ContainsKey(l.Module)) - .OrderBy(l => l.EntryPoint) - .GroupBy(l => l.EntryPoint); + var pinvokesGroupedByEntryPoint = pinvokes + .Where(l => modules.ContainsKey(l.Module)) + .OrderBy(l => l.EntryPoint) + .GroupBy(l => l.EntryPoint); var comparer = new PInvokeComparer(); foreach (IGrouping group in pinvokesGroupedByEntryPoint) @@ -129,8 +130,11 @@ private void EmitPInvokeTable(StreamWriter w, Dictionary modules PInvoke first = candidates[0]; if (ShouldTreatAsVariadic(candidates)) { - string imports = string.Join(Environment.NewLine, candidates.Select(p => $" {p.Method}")); - Log.LogWarning($"Found a native function ({first.EntryPoint}) with varargs, which is not supported. Calling it will fail at runtime. Native library: {first.Module}." + + string imports = string.Join(Environment.NewLine, + candidates.Select( + p => $" {p.Method} (in [{p.Method.DeclaringType?.Assembly.GetName().Name}] {p.Method.DeclaringType})")); + Log.LogWarning($"Found a native function ({first.EntryPoint}) with varargs in {first.Module}." + + " Calling such functions is not supported, and will fail at runtime." + $" Managed DllImports: {Environment.NewLine}{imports}"); foreach (var c in candidates) From 20a67a11f22c52868a9090aebea9858a845f6b51 Mon Sep 17 00:00:00 2001 From: Ankit Jain Date: Wed, 29 Sep 2021 13:31:50 -0400 Subject: [PATCH 22/22] address feedback from @lewing --- src/tasks/WasmAppBuilder/IcallTableGenerator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs index b218334db66bc4..a9f5f95161575d 100644 --- a/src/tasks/WasmAppBuilder/IcallTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/IcallTableGenerator.cs @@ -107,7 +107,7 @@ private void EmitTable (StreamWriter w) // Read the icall table generated by mono --print-icall-table private void ReadTable (string filename) { - using var stream = File.Open (filename, FileMode.Open); + using var stream = File.OpenRead (filename); using JsonDocument json = JsonDocument.Parse (stream); var arr = json.RootElement;