diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 2c0023e44a..be85e6c7b6 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -17,6 +17,9 @@ + + + diff --git a/samples/BenchmarkDotNet.Samples/IntroNuget.cs b/samples/BenchmarkDotNet.Samples/IntroNuget.cs new file mode 100644 index 0000000000..c46021e7f2 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroNuget.cs @@ -0,0 +1,50 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Threading; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters.Csv; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Toolchains.CsProj; +using Newtonsoft.Json; + +namespace BenchmarkDotNet.Samples +{ + /// + /// Benchmarks between various versions of a Nuget package + /// + /// + /// Only supported with the DotNetCliBuilder toolchain + /// + [Config(typeof(Config))] + public class IntroNuget + { + private class Config : ManualConfig + { + public Config() + { + //Specify jobs with different versions of the same Nuget package to benchmark. + //The Nuget versions referenced on these jobs must be greater or equal to the + //same Nuget version referenced in this benchmark project. + //Example: This benchmark project references Newtonsoft.Json 9.0.1 + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.2").WithId("11.0.2")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.2").WithId("10.0.2")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.1").WithId("10.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); + Add(DefaultConfig.Instance.GetColumnProviders().ToArray()); + Add(DefaultConfig.Instance.GetLoggers().ToArray()); + Add(CsvExporter.Default); + } + } + + [Benchmark] + public void SerializeAnonymousObject() => JsonConvert.SerializeObject(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + } +} diff --git a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs index dab445f9c7..cf6163877a 100644 --- a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs +++ b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs @@ -16,7 +16,8 @@ private InfrastructureResolver() Register(InfrastructureMode.EngineFactoryCharacteristic, () => new EngineFactory()); Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName); - Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty); + Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty); + Register(InfrastructureMode.NugetReferencesCharacteristic, Array.Empty); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs index a7e32974ca..a2a6252744 100644 --- a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs +++ b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs @@ -18,6 +18,7 @@ public sealed class InfrastructureMode : JobMode public static readonly Characteristic EngineFactoryCharacteristic = CreateCharacteristic(nameof(EngineFactory)); public static readonly Characteristic BuildConfigurationCharacteristic = CreateCharacteristic(nameof(BuildConfiguration)); public static readonly Characteristic> ArgumentsCharacteristic = CreateCharacteristic>(nameof(Arguments)); + public static readonly Characteristic> NugetReferencesCharacteristic = CreateCharacteristic>(nameof(NugetReferences)); public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessToolchain.Instance); public static readonly InfrastructureMode InProcessDontLogOutput = new InfrastructureMode(InProcessToolchain.DontLogOutput); @@ -62,5 +63,11 @@ public IReadOnlyList Arguments get => ArgumentsCharacteristic[this]; set => ArgumentsCharacteristic[this] = value; } + + public IReadOnlyCollection NugetReferences + { + get => NugetReferencesCharacteristic[this]; + set => NugetReferencesCharacteristic[this] = value; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 5f43cb048d..5d222269d5 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -184,6 +184,31 @@ public static Job With(this Job job, IReadOnlyList environm job.WithCore(j => j.Environment.EnvironmentVariables = environmentVariables); public static Job With(this Job job, IReadOnlyList arguments) => job.WithCore(j => j.Infrastructure.Arguments = arguments); + + /// + /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process + /// + /// + /// The Nuget package name + /// The Nuget package version + /// + public static Job WithNuget(this Job job, string packageName, string packageVersion) => job.WithCore(j => j.Infrastructure.NugetReferences = new HashSet(j.Infrastructure.NugetReferences ?? Array.Empty()) { new NugetReference(packageName, packageVersion) }); + + /// + /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process + /// + /// + /// The Nuget package name, the latest version will be resolved + /// + public static Job WithNuget(this Job job, string packageName) => job.WithCore(j => j.Infrastructure.NugetReferences = new HashSet(j.Infrastructure.NugetReferences ?? Array.Empty()) { new NugetReference(packageName, string.Empty) }); + + /// + /// Runs the job with a specific Nuget dependencies which will be resolved during the Job build process + /// + /// + /// A collection of Nuget dependencies + /// + public static Job WithNuget(this Job job, IReadOnlyCollection nugetReferences) => job.WithCore(j => j.Infrastructure.NugetReferences = nugetReferences); // Accuracy /// diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs new file mode 100644 index 0000000000..76d12ecd08 --- /dev/null +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -0,0 +1,81 @@ +using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; + +namespace BenchmarkDotNet.Jobs +{ + public class NugetReference : IEquatable + { + public NugetReference(string packageName, string packageVersion) + { + if (string.IsNullOrWhiteSpace(packageName)) + throw new ArgumentException("message", nameof(packageName)); + + PackageName = packageName; + + if (!string.IsNullOrWhiteSpace(PackageVersion) && !IsValidVersion(packageVersion)) + throw new InvalidOperationException($"Invalid version specified: {packageVersion}"); + + PackageVersion = packageVersion; + + } + + public string PackageName { get; } + public string PackageVersion { get; } + + public override bool Equals(object obj) + { + return Equals(obj as NugetReference); + } + + /// + /// Object is equals when the package name is the same + /// + /// + /// + /// + /// There can only be one package reference of the same name regardless of version + /// + public bool Equals(NugetReference other) + { + return other != null && + PackageName == other.PackageName; + } + + public override int GetHashCode() + { + return 557888800 + EqualityComparer.Default.GetHashCode(PackageName); + } + + public override string ToString() => $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}"; + + /// + /// Tries to validate the version string + /// + /// + /// + private bool IsValidVersion(string version) + { + if (string.IsNullOrWhiteSpace(version)) return false; + //There is a great nuget package for semver validation called `semver` however we probably + // don't want to add another dependency here so this will do some rudimentary validation + // and if that fails, then the actual add package command will fail anyways. + var parts = version.Split('-'); + if (parts.Length == 0) return false; + if (!Version.TryParse(parts[0], out var _)) return false; + for (int i = 1; i < parts.Length; i++) + { + if (!PreReleaseValidator.IsMatch(parts[i])) return false; + } + return true; + } + + /// + /// Used to validate all pre-release parts of a semver version + /// + /// + /// Allows alphanumeric chars, ".", "+", "-" + /// + private static readonly Regex PreReleaseValidator = new Regex(@"^[0-9A-Za-z\-\+\.]+$", RegexOptions.Compiled); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index 6c3121d85e..cb1d665106 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -23,10 +23,10 @@ public DotNetCliBuilder(string targetFrameworkMoniker, string customDotNetCliPat public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) => new DotNetCliCommand( - CustomDotNetCliPath, - string.Empty, - generateResult, - logger, + CustomDotNetCliPath, + string.Empty, + generateResult, + logger, buildPartition, Array.Empty()) .RestoreThenBuild(); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index 305d130f1f..6875ea11c2 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -44,6 +44,11 @@ public DotNetCliCommand WithArguments(string arguments) [PublicAPI] public BuildResult RestoreThenBuild() { + var packagesResult = AddPackages(); + + if (!packagesResult.IsSuccess) + return BuildResult.Failure(GenerateResult, new Exception(packagesResult.ProblemDescription)); + var restoreResult = Restore(); if (!restoreResult.IsSuccess) @@ -60,6 +65,11 @@ public BuildResult RestoreThenBuild() [PublicAPI] public BuildResult RestoreThenBuildThenPublish() { + var packagesResult = AddPackages(); + + if (!packagesResult.IsSuccess) + return BuildResult.Failure(GenerateResult, new Exception(packagesResult.ProblemDescription)); + var restoreResult = Restore(); if (!restoreResult.IsSuccess) @@ -76,6 +86,20 @@ public BuildResult RestoreThenBuildThenPublish() return Publish().ToBuildResult(GenerateResult); } + public DotNetCliCommandResult AddPackages() + { + var executionTime = new TimeSpan(0); + var stdOutput = new StringBuilder(); + foreach (var cmd in GetAddPackagesCommands(BuildPartition)) + { + var result = DotNetCliCommandExecutor.Execute(WithArguments(cmd)); + if (!result.IsSuccess) return result; + executionTime.Add(result.ExecutionTime); + stdOutput.Append(result.StandardOutput); + } + return DotNetCliCommandResult.Success(executionTime, stdOutput.ToString()); + } + public DotNetCliCommandResult Restore() => DotNetCliCommandExecutor.Execute(WithArguments( GetRestoreCommand(GenerateResult.ArtifactsPaths, BuildPartition, Arguments))); @@ -91,7 +115,10 @@ public DotNetCliCommandResult BuildNoDependencies() public DotNetCliCommandResult Publish() => DotNetCliCommandExecutor.Execute(WithArguments( GetPublishCommand(BuildPartition, Arguments))); - + + internal static IEnumerable GetAddPackagesCommands(BuildPartition buildPartition) + => GetNugetAddPackageCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); + internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string extraArguments = null) => new StringBuilder(100) .Append("restore ") @@ -126,5 +153,15 @@ private static string GetCustomMsBuildArguments(BenchmarkCase benchmarkCase, IRe return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation)); } + + private static IEnumerable GetNugetAddPackageCommands(BenchmarkCase benchmarkCase, IResolver resolver) + { + if (!benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic)) + return Enumerable.Empty(); + + var nugetRefs = benchmarkCase.Job.ResolveValue(InfrastructureMode.NugetReferencesCharacteristic, resolver); + + return nugetRefs.Select(x => $"add package {x.PackageName}{(string.IsNullOrWhiteSpace(x.PackageVersion) ? string.Empty : " -v " + x.PackageVersion)}"); + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs index 8d147f41f2..b34fa74342 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs @@ -83,7 +83,7 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition)}") .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetBuildCommand(buildPartition)}") .ToString(); - + File.WriteAllText(artifactsPaths.BuildScriptFilePath, content); } @@ -96,10 +96,6 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif private static bool IsRootSolutionFolder(DirectoryInfo directoryInfo) => directoryInfo .GetFileSystemInfos() - .Any(fileInfo => fileInfo.Extension == ".sln" || fileInfo.Name == "global.json"); - - - - + .Any(fileInfo => fileInfo.Extension == ".sln" || fileInfo.Name == "global.json"); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs index 9ce9f07936..8efd67bab1 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs @@ -40,6 +40,12 @@ public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IR return false; } + if (benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic)) + { + logger.WriteLineError("The Roslyn toolchain does not allow specifying Nuget package dependencies"); + return false; + } + return true; } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 08de06345c..b2c605c32f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -28,6 +28,7 @@ + diff --git a/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs new file mode 100644 index 0000000000..d3d7a8304b --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/NugetReferenceTests.cs @@ -0,0 +1,64 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using Xunit; +using Xunit.Abstractions; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Attributes; +using Newtonsoft.Json; +using System; +using System.Linq; +using BenchmarkDotNet.Toolchains.Roslyn; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Loggers; + +namespace BenchmarkDotNet.IntegrationTests +{ + public class NugetReferenceTests : BenchmarkTestExecutor + { + public NugetReferenceTests(ITestOutputHelper output) : base(output) { } + + [Fact] + public void UserCanSpecifyCustomNuGetPackageDependency() + { + var toolchain = RuntimeInformation.IsFullFramework + ? CsProjClassicNetToolchain.Current.Value // this .NET toolchain will do the right thing, the default RoslynToolchain does not support it + : CsProjCoreToolchain.Current.Value; + + var job = Job.Dry.With(toolchain).WithNuget("Newtonsoft.Json", "11.0.2"); + var config = CreateSimpleConfig(job: job); + + CanExecute(config); + } + + [Fact] + public void RoslynToolchainDoesNotSupportNuGetPackageDependency() + { + var toolchain = RoslynToolchain.Instance; + + var unsupportedJob = Job.Dry.With(toolchain).WithNuget("Newtonsoft.Json", "11.0.2"); + var unsupportedJobConfig = CreateSimpleConfig(job: unsupportedJob); + var unsupportedJobBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), unsupportedJobConfig); + var unsupportedJobLogger = new CompositeLogger(unsupportedJobConfig.GetLoggers().ToArray()); + foreach (var benchmarkCase in unsupportedJobBenchmark.BenchmarksCases) + { + Assert.False(toolchain.IsSupported(benchmarkCase, unsupportedJobLogger, BenchmarkRunner.DefaultResolver)); + } + + var supportedJob = Job.Dry.With(toolchain); + var supportedConfig = CreateSimpleConfig(job: supportedJob); + var supportedBenchmark = BenchmarkConverter.TypeToBenchmarks(typeof(WithCallToNewtonsoft), supportedConfig); + var supportedLogger = new CompositeLogger(supportedConfig.GetLoggers().ToArray()); + foreach (var benchmarkCase in supportedBenchmark.BenchmarksCases) + { + Assert.True(toolchain.IsSupported(benchmarkCase, supportedLogger, BenchmarkRunner.DefaultResolver)); + } + + } + + public class WithCallToNewtonsoft + { + [Benchmark] public void SerializeAnonymousObject() => JsonConvert.SerializeObject(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + } + } +} diff --git a/tests/BenchmarkDotNet.Tests/JobTests.cs b/tests/BenchmarkDotNet.Tests/JobTests.cs index 0c05e43db5..e7a62181b9 100644 --- a/tests/BenchmarkDotNet.Tests/JobTests.cs +++ b/tests/BenchmarkDotNet.Tests/JobTests.cs @@ -415,7 +415,7 @@ public static void Test07GetCharacteristics() Assert.Equal("Id;Accuracy;AnalyzeLaunchVariance;EvaluateOverhead;" + "MaxAbsoluteError;MaxRelativeError;MinInvokeCount;MinIterationTime;OutlierMode;Environment;Affinity;EnvironmentVariables;" + "Jit;Platform;Runtime;Gc;AllowVeryLargeObjects;Concurrent;CpuGroups;Force;HeapAffinitizeMask;HeapCount;NoAffinitize;" + - "RetainVm;Server;Infrastructure;Arguments;BuildConfiguration;Clock;EngineFactory;Toolchain;Meta;Baseline;IsDefault;IsMutator;Run;InvocationCount;IterationCount;IterationTime;" + + "RetainVm;Server;Infrastructure;Arguments;BuildConfiguration;Clock;EngineFactory;NugetReferences;Toolchain;Meta;Baseline;IsDefault;IsMutator;Run;InvocationCount;IterationCount;IterationTime;" + "LaunchCount;MaxIterationCount;MaxWarmupIterationCount;MinIterationCount;MinWarmupIterationCount;RunStrategy;UnrollFactor;WarmupCount", string.Join(";", a)); } @@ -459,6 +459,27 @@ public static void AllJobModesPropertyNamesMatchCharacteristicNames() // it't ma } } + [Fact] + public static void WithNuget() + { + var j = new Job("SomeId"); + + //.WithNuget extensions + + j = j.Freeze().WithNuget("Newtonsoft.Json"); + Assert.Equal(1, j.Infrastructure.NugetReferences.Count); + + j = j.WithNuget("AutoMapper", "7.0.1"); + Assert.Equal(2, j.Infrastructure.NugetReferences.Count); //appends + + j = j.WithNuget("AutoMapper"); + Assert.Equal(2, j.Infrastructure.NugetReferences.Count); //does not append, same package + j = j.WithNuget("AutoMapper", "7.0.0-alpha-0001"); + Assert.Equal(2, j.Infrastructure.NugetReferences.Count); //does not append, same package + + Assert.Equal("AutoMapper 7.0.1", j.Infrastructure.NugetReferences.ElementAt(1).ToString()); //first package reference in wins + } + private static bool IsSubclassOfobModeOfItself(Type type) { Type jobModeOfT;