From fbc1d08f0948800bf8408dc441e711cee0cd2943 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 25 Jun 2018 17:37:23 +1000 Subject: [PATCH 01/13] Initial commit with sample test for Nuget with newtonsoft --- samples/BenchmarkDotNet.Samples/IntroNuget.cs | 52 +++++++++++++++++++ .../Environments/InfrastructureResolver.cs | 3 +- .../Jobs/InfrastructureMode.cs | 7 +++ src/BenchmarkDotNet/Jobs/JobExtensions.cs | 27 +++++++++- src/BenchmarkDotNet/Jobs/NugetReference.cs | 26 ++++++++++ .../Toolchains/DotNetCli/DotNetCliBuilder.cs | 17 ++++++ .../DotNetCli/DotNetCliCommandExecutor.cs | 4 +- .../DotNetCli/DotNetCliGenerator.cs | 19 ++++++- 8 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 samples/BenchmarkDotNet.Samples/IntroNuget.cs create mode 100644 src/BenchmarkDotNet/Jobs/NugetReference.cs diff --git a/samples/BenchmarkDotNet.Samples/IntroNuget.cs b/samples/BenchmarkDotNet.Samples/IntroNuget.cs new file mode 100644 index 0000000000..1e3d4c6f12 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroNuget.cs @@ -0,0 +1,52 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Threading; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters.Csv; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Toolchains.CsProj; + +namespace BenchmarkDotNet.Samples + + [Config(typeof(Config))] + public class IntroNuget + { + private class Config : ManualConfig + { + public Config() + { + //TODO: So far only implemented with any toolchain using DotNetCliBuilder + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "8.0.1").WithId("8.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "7.0.1").WithId("7.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "6.0.1").WithId("6.0.1")); + Add(DefaultConfig.Instance.GetColumnProviders().ToArray()); + Add(DefaultConfig.Instance.GetLoggers().ToArray()); + Add(CsvExporter.Default); + } + } + + [GlobalSetup] + public void Setup() + { + var jsonConvertMethod = Assembly.Load("Newtonsoft.Json").GetType("Newtonsoft.Json.JsonConvert") + .GetMethods().Where(x => x.Name == "SerializeObject" && x.GetParameters().Length == 1) + .First(); + + Serializer = s => (string)jsonConvertMethod.Invoke(null, new object[] { s }); + } + + public Func Serializer { get; private set; } + + [Benchmark] + public void SerializeAnonymousObject() => Serializer(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + } +} diff --git a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs index 3e84fc8db8..4b63addf0d 100644 --- a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs +++ b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs @@ -17,7 +17,8 @@ private InfrastructureResolver() Register(InfrastructureMode.EngineFactoryCharacteristic, () => new EngineFactory()); Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName); - Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty); + Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty); + Register(InfrastructureMode.NugetReferencesCharacteristic, Array.Empty); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs index 6ede02ee91..14908fd06e 100644 --- a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs +++ b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs @@ -16,6 +16,7 @@ public sealed class InfrastructureMode : JobMode public static readonly Characteristic EngineFactoryCharacteristic = CreateCharacteristic(nameof(EngineFactory)); public static readonly Characteristic BuildConfigurationCharacteristic = CreateCharacteristic(nameof(BuildConfiguration)); public static readonly Characteristic> ArgumentsCharacteristic = CreateCharacteristic>(nameof(Arguments)); + public static readonly Characteristic> NugetReferencesCharacteristic = CreateCharacteristic>(nameof(NugetReferences)); public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessToolchain.Instance); public static readonly InfrastructureMode InProcessDontLogOutput = new InfrastructureMode(InProcessToolchain.DontLogOutput); @@ -60,5 +61,11 @@ public IReadOnlyList Arguments get => ArgumentsCharacteristic[this]; set => ArgumentsCharacteristic[this] = value; } + + public IReadOnlyList NugetReferences + { + get => NugetReferencesCharacteristic[this]; + set => NugetReferencesCharacteristic[this] = value; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 55f3140007..efd97171e2 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -159,8 +159,33 @@ public static class JobExtensions public static Job WithCustomBuildConfiguration(this Job job, string buildConfiguration) => job.WithCore(j => j.Infrastructure.BuildConfiguration = buildConfiguration); public static Job With(this Job job, IReadOnlyList environmentVariables) => job.WithCore(j => j.Environment.EnvironmentVariables = environmentVariables); public static Job With(this Job job, IReadOnlyList arguments) => job.WithCore(j => j.Infrastructure.Arguments = arguments); + + /// + /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process + /// + /// + /// The Nuget package name + /// The Nuget package version + /// + public static Job WithNuget(this Job job, string packageName, string packageVersion) => job.WithCore(j => j.Infrastructure.NugetReferences = new List { new NugetReference(packageName, packageVersion) }); + + /// + /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process + /// + /// + /// The Nuget package name, the latest version will be resolved + /// + public static Job WithNuget(this Job job, string packageName) => job.WithCore(j => j.Infrastructure.NugetReferences = new List { new NugetReference(packageName, string.Empty) }); + + /// + /// Runs the job with a specific Nuget dependencies which will be resolved during the Job build process + /// + /// + /// A collection of Nuget dependencies + /// + public static Job WithNuget(this Job job, IReadOnlyList nugetReferences) => job.WithCore(j => j.Infrastructure.NugetReferences = nugetReferences); - // Accuracy + // Accuracy /// /// Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error). /// The default value is 0.02. diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs new file mode 100644 index 0000000000..69d2b4c389 --- /dev/null +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -0,0 +1,26 @@ +using System; + +namespace BenchmarkDotNet.Jobs +{ + public class NugetReference + { + public NugetReference(string packageName, string packageVersion) + { + if (string.IsNullOrWhiteSpace(packageName)) + throw new ArgumentException("message", nameof(packageName)); + + PackageName = packageName; + PackageVersion = packageVersion; + if (!string.IsNullOrWhiteSpace(PackageVersion) && !Version.TryParse(PackageVersion, out var version)) + throw new InvalidOperationException($"Invalid version specified: {packageVersion}"); + } + + public string PackageName { get; } + public string PackageVersion { get; } + + public override string ToString() + { + return $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}"; + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index ee973c635e..56dd13ec5d 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; @@ -29,6 +30,22 @@ public virtual string GetBuildCommand(string frameworkMoniker, bool justTheProje public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) { + //TODO: This doesn't actually use the generate script in the DotNetCliGenerator to do the building... so instead we'll do the nuget checks here as well for now + // see: https://github.com/dotnet/BenchmarkDotNet/issues/804 + var nugetCommands = DotNetCliGenerator.GetNugetPackageCliCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); + foreach(var command in nugetCommands) + { + var addPackageResult = DotNetCliCommandExecutor.ExecuteCommand( + CustomDotNetCliPath, + command, + generateResult.ArtifactsPaths.BuildArtifactsDirectoryPath, + logger, + useSharedCompilation: null); //exclude the UseSharedCompilation command since that's not compatible with a dotnet add package + + if (!addPackageResult.IsSuccess) + return BuildResult.Failure(generateResult, new Exception(addPackageResult.ProblemDescription)); + } + var extraArguments = DotNetCliGenerator.GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); var restoreResult = DotNetCliCommandExecutor.ExecuteCommand( diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index 73e13e618e..a462aa855e 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -47,9 +47,9 @@ public static CommandResult Failure(TimeSpan time, string standardError, string internal static CommandResult ExecuteCommand( string customDotNetCliPath, string commandWithArguments, string workingDirectory, ILogger logger, - IReadOnlyList environmentVariables = null, bool useSharedCompilation = false) + IReadOnlyList environmentVariables = null, bool? useSharedCompilation = false) { - commandWithArguments = $"{commandWithArguments} /p:UseSharedCompilation={useSharedCompilation.ToString().ToLowerInvariant()}"; + commandWithArguments = $"{commandWithArguments}{(useSharedCompilation.HasValue ? $" /p:UseSharedCompilation={useSharedCompilation.ToString().ToLowerInvariant()}" : string.Empty)}"; using (var process = new Process { StartInfo = BuildStartInfo(customDotNetCliPath, workingDirectory, commandWithArguments, environmentVariables) }) { diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs index 5992fe3d60..3d256e1c78 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using BenchmarkDotNet.Characteristics; @@ -89,8 +90,12 @@ protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths) protected override void GenerateBuildScript(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) { - string content = $"call dotnet {Builder.RestoreCommand} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}{Environment.NewLine}" + - $"call dotnet {Builder.GetBuildCommand(TargetFrameworkMoniker, false, buildPartition.BuildConfiguration)} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}"; + //TODO: This build script isn't actually used, so this logic is also duplicated in the DotNetCliBuilder, see https://github.com/dotnet/BenchmarkDotNet/issues/804 + + var nugetPackages = string.Join("call dotnet ", GetNugetPackageCliCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)); + string content = string.IsNullOrWhiteSpace(nugetPackages) ? string.Empty : (nugetPackages + Environment.NewLine) + + $"call dotnet {Builder.RestoreCommand} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}{Environment.NewLine}" + + $"call dotnet {Builder.GetBuildCommand(TargetFrameworkMoniker, false, buildPartition.BuildConfiguration)} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}"; File.WriteAllText(artifactsPaths.BuildScriptFilePath, content); } @@ -115,5 +120,15 @@ internal static string GetCustomArguments(BenchmarkCase benchmarkCase, IResolver return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation)); } + + internal static IEnumerable GetNugetPackageCliCommands(BenchmarkCase benchmarkCase, IResolver resolver) + { + if (!benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic)) + return Enumerable.Empty(); + + var nugetRefs = benchmarkCase.Job.ResolveValue(InfrastructureMode.NugetReferencesCharacteristic, resolver).OfType(); + + return nugetRefs.Select(x => $"add package {x.PackageName}{(string.IsNullOrWhiteSpace(x.PackageVersion) ? string.Empty : " -v " + x.PackageVersion)}"); + } } } \ No newline at end of file From 3de64ced20744adaf109027f0271c6e8dc855d09 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 25 Jun 2018 21:09:29 +1000 Subject: [PATCH 02/13] missing brace (oops!) --- samples/BenchmarkDotNet.Samples/IntroNuget.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/BenchmarkDotNet.Samples/IntroNuget.cs b/samples/BenchmarkDotNet.Samples/IntroNuget.cs index 1e3d4c6f12..d2adeb7f81 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNuget.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNuget.cs @@ -13,7 +13,7 @@ using BenchmarkDotNet.Toolchains.CsProj; namespace BenchmarkDotNet.Samples - +{ [Config(typeof(Config))] public class IntroNuget { From bc04fae316b061e47b908158f762f6ad93a05d63 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 22 Oct 2018 12:11:50 +1100 Subject: [PATCH 03/13] adds notes --- src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs index 8b057b46d6..db21ea42f9 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs @@ -27,6 +27,8 @@ protected override string[] GetArtifactsToCleanup(ArtifactsPaths artifactsPaths) protected override void GenerateBuildScript(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) { + //TODO: For nuget refs i think we need to use the MsBuildArgument/Argument to tell msbuild to fetch nuget refs? + var prefix = RuntimeInformation.IsWindows() ? "" : "#!/bin/bash\n"; var list = new List(); if (!RuntimeInformation.IsWindows()) @@ -47,6 +49,7 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif prefix + string.Join(" ", list)); } + //TODO: We need to get nuget dependencies in here somehow internal static IEnumerable GetAllReferences(BenchmarkCase benchmarkCase) => benchmarkCase.Descriptor.Type.GetTypeInfo().Assembly .GetReferencedAssemblies() From f54d2bde8c1f0187468a3b21b2ada6676aebeee2 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 22 Oct 2018 13:20:37 +1100 Subject: [PATCH 04/13] Updated code for nuget support merged with latest code in project --- samples/BenchmarkDotNet.Samples/IntroNuget.cs | 2 +- src/BenchmarkDotNet/Jobs/NugetReference.cs | 5 +-- .../Toolchains/DotNetCli/DotNetCliBuilder.cs | 34 +++++----------- .../Toolchains/DotNetCli/DotNetCliCommand.cs | 39 ++++++++++++++++++- .../DotNetCli/DotNetCliGenerator.cs | 16 +++----- 5 files changed, 55 insertions(+), 41 deletions(-) diff --git a/samples/BenchmarkDotNet.Samples/IntroNuget.cs b/samples/BenchmarkDotNet.Samples/IntroNuget.cs index d2adeb7f81..085032cd3d 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNuget.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNuget.cs @@ -22,7 +22,7 @@ private class Config : ManualConfig public Config() { //TODO: So far only implemented with any toolchain using DotNetCliBuilder - Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.2").WithId("11.0.2")); Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "8.0.1").WithId("8.0.1")); diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs index 69d2b4c389..eb9c019dd2 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReference.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -18,9 +18,6 @@ public NugetReference(string packageName, string packageVersion) public string PackageName { get; } public string PackageVersion { get; } - public override string ToString() - { - return $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}"; - } + public override string ToString() => $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}"; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index 56c8c406b6..4292cafde4 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -23,29 +23,15 @@ public DotNetCliBuilder(string targetFrameworkMoniker, string customDotNetCliPat } public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) - => new DotNetCliCommand( - //TODO: This doesn't actually use the generate script in the DotNetCliGenerator to do the building... so instead we'll do the nuget checks here as well for now - // see: https://github.com/dotnet/BenchmarkDotNet/issues/804 - var nugetCommands = DotNetCliGenerator.GetNugetPackageCliCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); - foreach(var command in nugetCommands) - { - var addPackageResult = DotNetCliCommandExecutor.ExecuteCommand( - CustomDotNetCliPath, - command, - generateResult.ArtifactsPaths.BuildArtifactsDirectoryPath, - logger, - useSharedCompilation: null); //exclude the UseSharedCompilation command since that's not compatible with a dotnet add package - - if (!addPackageResult.IsSuccess) - return BuildResult.Failure(generateResult, new Exception(addPackageResult.ProblemDescription)); - } - - CustomDotNetCliPath, - string.Empty, - generateResult, - logger, - buildPartition, - Array.Empty()) - .RestoreThenBuild(); + { + return new DotNetCliCommand( + CustomDotNetCliPath, + string.Empty, + generateResult, + logger, + buildPartition, + Array.Empty()) + .RestoreThenBuild(); + } } } diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index 305d130f1f..c92f4dca3a 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -44,6 +44,11 @@ public DotNetCliCommand WithArguments(string arguments) [PublicAPI] public BuildResult RestoreThenBuild() { + var packagesResult = AddPackages(); + + if (!packagesResult.IsSuccess) + return BuildResult.Failure(GenerateResult, new Exception(packagesResult.ProblemDescription)); + var restoreResult = Restore(); if (!restoreResult.IsSuccess) @@ -60,6 +65,11 @@ public BuildResult RestoreThenBuild() [PublicAPI] public BuildResult RestoreThenBuildThenPublish() { + var packagesResult = AddPackages(); + + if (!packagesResult.IsSuccess) + return BuildResult.Failure(GenerateResult, new Exception(packagesResult.ProblemDescription)); + var restoreResult = Restore(); if (!restoreResult.IsSuccess) @@ -76,6 +86,20 @@ public BuildResult RestoreThenBuildThenPublish() return Publish().ToBuildResult(GenerateResult); } + public DotNetCliCommandResult AddPackages() + { + var executionTime = new TimeSpan(0); + var stdOutput = new StringBuilder(); + foreach (var cmd in GetAddPackagesCommands(BuildPartition)) + { + var result = DotNetCliCommandExecutor.Execute(WithArguments(cmd)); + if (!result.IsSuccess) return result; + executionTime.Add(result.ExecutionTime); + stdOutput.Append(result.StandardOutput); + } + return DotNetCliCommandResult.Success(executionTime, stdOutput.ToString()); + } + public DotNetCliCommandResult Restore() => DotNetCliCommandExecutor.Execute(WithArguments( GetRestoreCommand(GenerateResult.ArtifactsPaths, BuildPartition, Arguments))); @@ -91,7 +115,10 @@ public DotNetCliCommandResult BuildNoDependencies() public DotNetCliCommandResult Publish() => DotNetCliCommandExecutor.Execute(WithArguments( GetPublishCommand(BuildPartition, Arguments))); - + + internal static IEnumerable GetAddPackagesCommands(BuildPartition buildPartition) + => GetNugetAddPackageCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); + internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string extraArguments = null) => new StringBuilder(100) .Append("restore ") @@ -126,5 +153,15 @@ private static string GetCustomMsBuildArguments(BenchmarkCase benchmarkCase, IRe return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation)); } + + private static IEnumerable GetNugetAddPackageCommands(BenchmarkCase benchmarkCase, IResolver resolver) + { + if (!benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic)) + return Enumerable.Empty(); + + var nugetRefs = benchmarkCase.Job.ResolveValue(InfrastructureMode.NugetReferencesCharacteristic, resolver).OfType(); + + return nugetRefs.Select(x => $"add package {x.PackageName}{(string.IsNullOrWhiteSpace(x.PackageVersion) ? string.Empty : " -v " + x.PackageVersion)}"); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs index 070eebdfcd..e277a7aa87 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs @@ -2,6 +2,8 @@ using System.IO; using System.Linq; using System.Text; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -84,7 +86,7 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition)}") .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetBuildCommand(buildPartition)}") .ToString(); - + File.WriteAllText(artifactsPaths.BuildScriptFilePath, content); } @@ -99,18 +101,10 @@ private static bool IsRootSolutionFolder(DirectoryInfo directoryInfo) .GetFileSystemInfos() .Any(fileInfo => fileInfo.Extension == ".sln" || fileInfo.Name == "global.json"); - - - } + - internal static IEnumerable GetNugetPackageCliCommands(BenchmarkCase benchmarkCase, IResolver resolver) - { - if (!benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic)) - return Enumerable.Empty(); + } - var nugetRefs = benchmarkCase.Job.ResolveValue(InfrastructureMode.NugetReferencesCharacteristic, resolver).OfType(); - return nugetRefs.Select(x => $"add package {x.PackageName}{(string.IsNullOrWhiteSpace(x.PackageVersion) ? string.Empty : " -v " + x.PackageVersion)}"); - } } \ No newline at end of file From eafad74c3716948be265e93d10de191f8eee72b4 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 24 Oct 2018 00:21:37 +1100 Subject: [PATCH 05/13] reverts uneeded change to DotNetCliBuilder --- .../Toolchains/DotNetCli/DotNetCliBuilder.cs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index 4292cafde4..bc8bdfc6b7 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -23,15 +23,13 @@ public DotNetCliBuilder(string targetFrameworkMoniker, string customDotNetCliPat } public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) - { - return new DotNetCliCommand( - CustomDotNetCliPath, - string.Empty, - generateResult, - logger, - buildPartition, - Array.Empty()) - .RestoreThenBuild(); - } + => new DotNetCliCommand( + CustomDotNetCliPath, + string.Empty, + generateResult, + logger, + buildPartition, + Array.Empty()) + .RestoreThenBuild(); } } From 8067f37efd18cbbfaafb13cc8451fc6e6fa98256 Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 24 Oct 2018 00:22:36 +1100 Subject: [PATCH 06/13] reverts uneeded change to DotNetCliBuilder --- src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index bc8bdfc6b7..cb1d665106 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -1,6 +1,5 @@ using System; using BenchmarkDotNet.Jobs; -using System.Linq; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; From d337a04b903c0e4979f015d489cd6fa0f460d64b Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 24 Oct 2018 00:24:53 +1100 Subject: [PATCH 07/13] reverts uneeded change to DotNetCliGenerator --- .../Toolchains/DotNetCli/DotNetCliGenerator.cs | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs index e277a7aa87..b34fa74342 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs @@ -1,9 +1,6 @@ -using System.Collections.Generic; -using System.IO; +using System.IO; using System.Linq; using System.Text; -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -99,12 +96,6 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif private static bool IsRootSolutionFolder(DirectoryInfo directoryInfo) => directoryInfo .GetFileSystemInfos() - .Any(fileInfo => fileInfo.Extension == ".sln" || fileInfo.Name == "global.json"); - - - - + .Any(fileInfo => fileInfo.Extension == ".sln" || fileInfo.Name == "global.json"); } - - } \ No newline at end of file From feb4dd0eb49bc3c8413fc333e7b8eaa35ff673bd Mon Sep 17 00:00:00 2001 From: Shannon Date: Wed, 24 Oct 2018 15:27:12 +1100 Subject: [PATCH 08/13] Fixes test --- tests/BenchmarkDotNet.Tests/JobTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/BenchmarkDotNet.Tests/JobTests.cs b/tests/BenchmarkDotNet.Tests/JobTests.cs index 0c05e43db5..33bf61b2b9 100644 --- a/tests/BenchmarkDotNet.Tests/JobTests.cs +++ b/tests/BenchmarkDotNet.Tests/JobTests.cs @@ -415,7 +415,7 @@ public static void Test07GetCharacteristics() Assert.Equal("Id;Accuracy;AnalyzeLaunchVariance;EvaluateOverhead;" + "MaxAbsoluteError;MaxRelativeError;MinInvokeCount;MinIterationTime;OutlierMode;Environment;Affinity;EnvironmentVariables;" + "Jit;Platform;Runtime;Gc;AllowVeryLargeObjects;Concurrent;CpuGroups;Force;HeapAffinitizeMask;HeapCount;NoAffinitize;" + - "RetainVm;Server;Infrastructure;Arguments;BuildConfiguration;Clock;EngineFactory;Toolchain;Meta;Baseline;IsDefault;IsMutator;Run;InvocationCount;IterationCount;IterationTime;" + + "RetainVm;Server;Infrastructure;Arguments;BuildConfiguration;Clock;EngineFactory;NugetReferences;Toolchain;Meta;Baseline;IsDefault;IsMutator;Run;InvocationCount;IterationCount;IterationTime;" + "LaunchCount;MaxIterationCount;MaxWarmupIterationCount;MinIterationCount;MinWarmupIterationCount;RunStrategy;UnrollFactor;WarmupCount", string.Join(";", a)); } From 8936840d391bc33191c44c91665bbd66b27fd59d Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 25 Oct 2018 11:26:58 +1100 Subject: [PATCH 09/13] Updates IntroNuget, adds comments, adds ref to newtonsoft --- .../BenchmarkDotNet.Samples.csproj | 3 ++ samples/BenchmarkDotNet.Samples/IntroNuget.cs | 32 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 2c0023e44a..be85e6c7b6 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -17,6 +17,9 @@ + + + diff --git a/samples/BenchmarkDotNet.Samples/IntroNuget.cs b/samples/BenchmarkDotNet.Samples/IntroNuget.cs index 085032cd3d..c46021e7f2 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNuget.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNuget.cs @@ -11,9 +11,16 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Toolchains.CsProj; +using Newtonsoft.Json; namespace BenchmarkDotNet.Samples { + /// + /// Benchmarks between various versions of a Nuget package + /// + /// + /// Only supported with the DotNetCliBuilder toolchain + /// [Config(typeof(Config))] public class IntroNuget { @@ -21,32 +28,23 @@ private class Config : ManualConfig { public Config() { - //TODO: So far only implemented with any toolchain using DotNetCliBuilder + //Specify jobs with different versions of the same Nuget package to benchmark. + //The Nuget versions referenced on these jobs must be greater or equal to the + //same Nuget version referenced in this benchmark project. + //Example: This benchmark project references Newtonsoft.Json 9.0.1 Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.2").WithId("11.0.2")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.2").WithId("10.0.2")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.1").WithId("10.0.1")); Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); - Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "8.0.1").WithId("8.0.1")); - Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "7.0.1").WithId("7.0.1")); - Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "6.0.1").WithId("6.0.1")); Add(DefaultConfig.Instance.GetColumnProviders().ToArray()); Add(DefaultConfig.Instance.GetLoggers().ToArray()); Add(CsvExporter.Default); } } - [GlobalSetup] - public void Setup() - { - var jsonConvertMethod = Assembly.Load("Newtonsoft.Json").GetType("Newtonsoft.Json.JsonConvert") - .GetMethods().Where(x => x.Name == "SerializeObject" && x.GetParameters().Length == 1) - .First(); - - Serializer = s => (string)jsonConvertMethod.Invoke(null, new object[] { s }); - } - - public Func Serializer { get; private set; } - [Benchmark] - public void SerializeAnonymousObject() => Serializer(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + public void SerializeAnonymousObject() => JsonConvert.SerializeObject(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); } } From 1e784582cb5d246b202a63eacb4f337d40cf2bd6 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 25 Oct 2018 12:39:03 +1100 Subject: [PATCH 10/13] Adds integration and unit test, ensures that multiple packages can be added but not multiple of the same name, fixes version validation for pre-releases --- .../Jobs/InfrastructureMode.cs | 4 +- src/BenchmarkDotNet/Jobs/JobExtensions.cs | 6 +- src/BenchmarkDotNet/Jobs/NugetReference.cs | 64 ++++++++++++++++++- .../BenchmarkDotNet.IntegrationTests.csproj | 1 + .../NugetReferenceTests.cs | 35 ++++++++++ tests/BenchmarkDotNet.Tests/JobTests.cs | 21 ++++++ 6 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs diff --git a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs index 8211eee9bc..a2a6252744 100644 --- a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs +++ b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs @@ -18,7 +18,7 @@ public sealed class InfrastructureMode : JobMode public static readonly Characteristic EngineFactoryCharacteristic = CreateCharacteristic(nameof(EngineFactory)); public static readonly Characteristic BuildConfigurationCharacteristic = CreateCharacteristic(nameof(BuildConfiguration)); public static readonly Characteristic> ArgumentsCharacteristic = CreateCharacteristic>(nameof(Arguments)); - public static readonly Characteristic> NugetReferencesCharacteristic = CreateCharacteristic>(nameof(NugetReferences)); + public static readonly Characteristic> NugetReferencesCharacteristic = CreateCharacteristic>(nameof(NugetReferences)); public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessToolchain.Instance); public static readonly InfrastructureMode InProcessDontLogOutput = new InfrastructureMode(InProcessToolchain.DontLogOutput); @@ -64,7 +64,7 @@ public IReadOnlyList Arguments set => ArgumentsCharacteristic[this] = value; } - public IReadOnlyList NugetReferences + public IReadOnlyCollection NugetReferences { get => NugetReferencesCharacteristic[this]; set => NugetReferencesCharacteristic[this] = value; diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 838b38c275..5d222269d5 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -192,7 +192,7 @@ public static Job With(this Job job, IReadOnlyList environm /// The Nuget package name /// The Nuget package version /// - public static Job WithNuget(this Job job, string packageName, string packageVersion) => job.WithCore(j => j.Infrastructure.NugetReferences = new List { new NugetReference(packageName, packageVersion) }); + public static Job WithNuget(this Job job, string packageName, string packageVersion) => job.WithCore(j => j.Infrastructure.NugetReferences = new HashSet(j.Infrastructure.NugetReferences ?? Array.Empty()) { new NugetReference(packageName, packageVersion) }); /// /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process @@ -200,7 +200,7 @@ public static Job With(this Job job, IReadOnlyList environm /// /// The Nuget package name, the latest version will be resolved /// - public static Job WithNuget(this Job job, string packageName) => job.WithCore(j => j.Infrastructure.NugetReferences = new List { new NugetReference(packageName, string.Empty) }); + public static Job WithNuget(this Job job, string packageName) => job.WithCore(j => j.Infrastructure.NugetReferences = new HashSet(j.Infrastructure.NugetReferences ?? Array.Empty()) { new NugetReference(packageName, string.Empty) }); /// /// Runs the job with a specific Nuget dependencies which will be resolved during the Job build process @@ -208,7 +208,7 @@ public static Job With(this Job job, IReadOnlyList environm /// /// A collection of Nuget dependencies /// - public static Job WithNuget(this Job job, IReadOnlyList nugetReferences) => job.WithCore(j => j.Infrastructure.NugetReferences = nugetReferences); + public static Job WithNuget(this Job job, IReadOnlyCollection nugetReferences) => job.WithCore(j => j.Infrastructure.NugetReferences = nugetReferences); // Accuracy /// diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs index eb9c019dd2..2fc7370229 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReference.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -1,8 +1,10 @@ using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; namespace BenchmarkDotNet.Jobs { - public class NugetReference + public class NugetReference : IEquatable { public NugetReference(string packageName, string packageVersion) { @@ -10,14 +12,70 @@ public NugetReference(string packageName, string packageVersion) throw new ArgumentException("message", nameof(packageName)); PackageName = packageName; - PackageVersion = packageVersion; - if (!string.IsNullOrWhiteSpace(PackageVersion) && !Version.TryParse(PackageVersion, out var version)) + + if (!string.IsNullOrWhiteSpace(PackageVersion) && !IsValidVersion(packageVersion)) throw new InvalidOperationException($"Invalid version specified: {packageVersion}"); + + PackageVersion = packageVersion; + } public string PackageName { get; } public string PackageVersion { get; } + public override bool Equals(object obj) + { + return Equals(obj as NugetReference); + } + + /// + /// Object is equals when the package name is the same + /// + /// + /// + /// + /// There can only be one package reference of the same name regardless of version + /// + public bool Equals(NugetReference other) + { + return other != null && + PackageName == other.PackageName; + } + + public override int GetHashCode() + { + return 557888800 + EqualityComparer.Default.GetHashCode(PackageName); + } + public override string ToString() => $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}"; + + /// + /// Tries to validate the version string + /// + /// + /// + private bool IsValidVersion(string version) + { + if (string.IsNullOrWhiteSpace(version)) return false; + //TODO: There is a great nuget package for this called `semver` however we probably + // don't want to add another dependency here so this will do some rudimentary validation + // and if that fails, then the actual add package command will fail anyways. + var parts = version.Split('-'); + if (parts.Length == 0) return false; + if (!Version.TryParse(parts[0], out var _)) return false; + for (int i = 1; i < parts.Length; i++) + { + if (!PreReleaseValidator.IsMatch(parts[i])) return false; + } + return true; + } + + /// + /// Used to validate all pre-release parts of a semver version + /// + /// + /// Allows alphanumeric chars, ".", "+", "-" + /// + private static readonly Regex PreReleaseValidator = new Regex(@"^[0-9A-Za-z\-\+\.]+$", RegexOptions.Compiled); } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 08de06345c..b2c605c32f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -28,6 +28,7 @@ + diff --git a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs new file mode 100644 index 0000000000..d57a797dee --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs @@ -0,0 +1,35 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using Xunit; +using Xunit.Abstractions; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Attributes; +using Newtonsoft.Json; +using System; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class NugetReferenceTests : BenchmarkTestExecutor + { + public NugetReferenceTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void UserCanSpecifyCustomNuGetPackageDependency() + { + var toolchain = RuntimeInformation.IsFullFramework + ? CsProjClassicNetToolchain.Current.Value // this .NET toolchain will do the right thing, the default RoslynToolchain does not support it + : CsProjCoreToolchain.Current.Value; + + var job = Job.Dry.With(toolchain).WithNuget("Newtonsoft.Json", "11.0.2"); + var config = CreateSimpleConfig(job: job); + + CanExecute(config); + } + + public class WithCallToNewtonsoft + { + [Benchmark] public void SerializeAnonymousObject() => JsonConvert.SerializeObject(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/JobTests.cs b/tests/BenchmarkDotNet.Tests/JobTests.cs index 33bf61b2b9..e7a62181b9 100644 --- a/tests/BenchmarkDotNet.Tests/JobTests.cs +++ b/tests/BenchmarkDotNet.Tests/JobTests.cs @@ -459,6 +459,27 @@ public static void AllJobModesPropertyNamesMatchCharacteristicNames() // it't ma } } + [Fact] + public static void WithNuget() + { + var j = new Job("SomeId"); + + //.WithNuget extensions + + j = j.Freeze().WithNuget("Newtonsoft.Json"); + Assert.Equal(1, j.Infrastructure.NugetReferences.Count); + + j = j.WithNuget("AutoMapper", "7.0.1"); + Assert.Equal(2, j.Infrastructure.NugetReferences.Count); //appends + + j = j.WithNuget("AutoMapper"); + Assert.Equal(2, j.Infrastructure.NugetReferences.Count); //does not append, same package + j = j.WithNuget("AutoMapper", "7.0.0-alpha-0001"); + Assert.Equal(2, j.Infrastructure.NugetReferences.Count); //does not append, same package + + Assert.Equal("AutoMapper 7.0.1", j.Infrastructure.NugetReferences.ElementAt(1).ToString()); //first package reference in wins + } + private static bool IsSubclassOfobModeOfItself(Type type) { Type jobModeOfT; From 7343f2c114abfd8bb96789722afa1865ea90da72 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 25 Oct 2018 13:01:41 +1100 Subject: [PATCH 11/13] Adds check to RoslynToolchain for nuget dependencies, adds unit tests to show that the RoslynToolchain doesn't support Nuget refs --- .../Toolchains/Roslyn/Generator.cs | 2 -- .../Toolchains/Roslyn/RoslynToolchain.cs | 6 ++++ .../NugetReferenceTests.cs | 29 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs index d8df3c7a90..cbdb9046f1 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs @@ -27,7 +27,6 @@ protected override string[] GetArtifactsToCleanup(ArtifactsPaths artifactsPaths) protected override void GenerateBuildScript(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) { - //TODO: For nuget refs i think we need to use the MsBuildArgument/Argument to tell msbuild to fetch nuget refs? string prefix = RuntimeInformation.IsWindows() ? "" : "#!/bin/bash\n"; var list = new List(); if (!RuntimeInformation.IsWindows()) @@ -48,7 +47,6 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif prefix + string.Join(" ", list)); } - //TODO: We need to get nuget dependencies in here somehow internal static IEnumerable GetAllReferences(BenchmarkCase benchmarkCase) => benchmarkCase.Descriptor.Type.GetTypeInfo().Assembly .GetReferencedAssemblies() diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs index 9ce9f07936..8efd67bab1 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs @@ -40,6 +40,12 @@ public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IR return false; } + if (benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic)) + { + logger.WriteLineError("The Roslyn toolchain does not allow specifying Nuget package dependencies"); + return false; + } + return true; } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs index d57a797dee..d3d7a8304b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs @@ -7,6 +7,10 @@ using BenchmarkDotNet.Attributes; using Newtonsoft.Json; using System; +using System.Linq; +using BenchmarkDotNet.Toolchains.Roslyn; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Loggers; namespace BenchmarkDotNet.IntegrationTests { @@ -27,6 +31,31 @@ public void UserCanSpecifyCustomNuGetPackageDependency() CanExecute(config); } + [Fact] + public void RoslynToolchainDoesNotSupportNuGetPackageDependency() + { + var toolchain = RoslynToolchain.Instance; + + var unsupportedJob = Job.Dry.With(toolchain).WithNuget("Newtonsoft.Json", "11.0.2"); + var unsupportedJobConfig = CreateSimpleConfig(job: unsupportedJob); + var unsupportedJobBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), unsupportedJobConfig); + var unsupportedJobLogger = new CompositeLogger(unsupportedJobConfig.GetLoggers().ToArray()); + foreach (var benchmarkCase in unsupportedJobBenchmark.BenchmarksCases) + { + Assert.False(toolchain.IsSupported(benchmarkCase, unsupportedJobLogger, BenchmarkRunner.DefaultResolver)); + } + + var supportedJob = Job.Dry.With(toolchain); + var supportedConfig = CreateSimpleConfig(job: supportedJob); + var supportedBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), supportedConfig); + var supportedLogger = new CompositeLogger(supportedConfig.GetLoggers().ToArray()); + foreach (var benchmarkCase in supportedBenchmark.BenchmarksCases) + { + Assert.True(toolchain.IsSupported(benchmarkCase, supportedLogger, BenchmarkRunner.DefaultResolver)); + } + + } + public class WithCallToNewtonsoft { [Benchmark] public void SerializeAnonymousObject() => JsonConvert.SerializeObject(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); From 4ce6b5fa33afdc20956480a2f45033dc6ab1562e Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 25 Oct 2018 13:03:27 +1100 Subject: [PATCH 12/13] Removes the OfType since it's not required --- src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index c92f4dca3a..6875ea11c2 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -159,7 +159,7 @@ private static IEnumerable GetNugetAddPackageCommands(BenchmarkCase benc if (!benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic)) return Enumerable.Empty(); - var nugetRefs = benchmarkCase.Job.ResolveValue(InfrastructureMode.NugetReferencesCharacteristic, resolver).OfType(); + var nugetRefs = benchmarkCase.Job.ResolveValue(InfrastructureMode.NugetReferencesCharacteristic, resolver); return nugetRefs.Select(x => $"add package {x.PackageName}{(string.IsNullOrWhiteSpace(x.PackageVersion) ? string.Empty : " -v " + x.PackageVersion)}"); } From 0d6c46483b5b02761c5480a135b6ff99f34b5eb3 Mon Sep 17 00:00:00 2001 From: Shannon Date: Thu, 25 Oct 2018 13:07:13 +1100 Subject: [PATCH 13/13] removes TODO note --- src/BenchmarkDotNet/Jobs/NugetReference.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs index 2fc7370229..76d12ecd08 100644 --- a/src/BenchmarkDotNet/Jobs/NugetReference.cs +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -57,7 +57,7 @@ public override int GetHashCode() private bool IsValidVersion(string version) { if (string.IsNullOrWhiteSpace(version)) return false; - //TODO: There is a great nuget package for this called `semver` however we probably + //There is a great nuget package for semver validation called `semver` however we probably // don't want to add another dependency here so this will do some rudimentary validation // and if that fails, then the actual add package command will fail anyways. var parts = version.Split('-');