Enables benchmarking betweeen different Nuget packages

samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj

@@ -17,6 +17,9 @@
   <ItemGroup Condition=" '$(TargetFrameworkIdentifier)' == '.NETFramework' ">
     <Reference Include="System.Reflection" />
   </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="9.0.1" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
     <ProjectReference Include="..\..\src\BenchmarkDotNet.Diagnostics.Windows\BenchmarkDotNet.Diagnostics.Windows.csproj" />

samples/BenchmarkDotNet.Samples/IntroNuget.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Linq;
+using System.Reflection;
+using System.Threading;
+using BenchmarkDotNet.Analysers;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Exporters.Csv;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Toolchains.CsProj;
+using Newtonsoft.Json;
+
+namespace BenchmarkDotNet.Samples
+{
+    /// <summary>
+    /// Benchmarks between various versions of a Nuget package
+    /// </summary>
+    /// <remarks>
+    /// Only supported with the DotNetCliBuilder toolchain
+    /// </remarks>
+    [Config(typeof(Config))]
+    public class IntroNuget
+    {
+        private class Config : ManualConfig
+        {
+            public Config()
+            {
+                //Specify jobs with different versions of the same Nuget package to benchmark.
+                //The Nuget versions referenced on these jobs must be greater or equal to the 
+                //same Nuget version referenced in this benchmark project.
+                //Example: This benchmark project references Newtonsoft.Json 9.0.1
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.2").WithId("11.0.2"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.1").WithId("11.0.1"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.3").WithId("10.0.3"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.2").WithId("10.0.2"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.1").WithId("10.0.1"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "9.0.1").WithId("9.0.1"));
+                Add(DefaultConfig.Instance.GetColumnProviders().ToArray());
+                Add(DefaultConfig.Instance.GetLoggers().ToArray());
+                Add(CsvExporter.Default);
+            }
+        }
+
+        [Benchmark]
+        public void SerializeAnonymousObject() => JsonConvert.SerializeObject(new { hello = "world", price = 1.99, now = DateTime.UtcNow });
+    }
+}

src/BenchmarkDotNet/Environments/InfrastructureResolver.cs

@@ -16,7 +16,8 @@ private InfrastructureResolver()
             Register(InfrastructureMode.EngineFactoryCharacteristic, () => new EngineFactory());
             Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName);
 
-            Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty<Argument>);            
+            Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty<Argument>);
+            Register(InfrastructureMode.NugetReferencesCharacteristic, Array.Empty<NugetReference>);
         }
     }
 }

src/BenchmarkDotNet/Jobs/InfrastructureMode.cs

@@ -18,6 +18,7 @@ public sealed class InfrastructureMode : JobMode<InfrastructureMode>
         public static readonly Characteristic<IEngineFactory> EngineFactoryCharacteristic = CreateCharacteristic<IEngineFactory>(nameof(EngineFactory));
         public static readonly Characteristic<string> BuildConfigurationCharacteristic = CreateCharacteristic<string>(nameof(BuildConfiguration));        
         public static readonly Characteristic<IReadOnlyList<Argument>> ArgumentsCharacteristic = CreateCharacteristic<IReadOnlyList<Argument>>(nameof(Arguments));
+        public static readonly Characteristic<IReadOnlyCollection<NugetReference>> NugetReferencesCharacteristic = CreateCharacteristic<IReadOnlyCollection<NugetReference>>(nameof(NugetReferences));
 
         public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessToolchain.Instance);
         public static readonly InfrastructureMode InProcessDontLogOutput = new InfrastructureMode(InProcessToolchain.DontLogOutput);
@@ -62,5 +63,11 @@ public IReadOnlyList<Argument> Arguments
             get => ArgumentsCharacteristic[this];
             set => ArgumentsCharacteristic[this] = value;
         }
+
+        public IReadOnlyCollection<NugetReference> NugetReferences
+        {
+            get => NugetReferencesCharacteristic[this];
+            set => NugetReferencesCharacteristic[this] = value;
+        }
     }
 }

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -184,6 +184,31 @@ public static Job With(this Job job, IReadOnlyList<EnvironmentVariable> environm
             job.WithCore(j => j.Environment.EnvironmentVariables = environmentVariables);
 
         public static Job With(this Job job, IReadOnlyList<Argument> arguments) => job.WithCore(j => j.Infrastructure.Arguments = arguments);
+
+        /// <summary>
+        /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process
+        /// </summary>
+        /// <param name="job"></param>
+        /// <param name="packageName">The Nuget package name</param>
+        /// <param name="packageVersion">The Nuget package version</param>
+        /// <returns></returns>
+        public static Job WithNuget(this Job job, string packageName, string packageVersion) => job.WithCore(j => j.Infrastructure.NugetReferences = new HashSet<NugetReference>(j.Infrastructure.NugetReferences ?? Array.Empty<NugetReference>()) { new NugetReference(packageName, packageVersion) });
+
+        /// <summary>
+        /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process
+        /// </summary>
+        /// <param name="job"></param>
+        /// <param name="packageName">The Nuget package name, the latest version will be resolved</param>
+        /// <returns></returns>
+        public static Job WithNuget(this Job job, string packageName) => job.WithCore(j => j.Infrastructure.NugetReferences = new HashSet<NugetReference>(j.Infrastructure.NugetReferences ?? Array.Empty<NugetReference>()) { new NugetReference(packageName, string.Empty) });
+
+        /// <summary>
+        /// Runs the job with a specific Nuget dependencies which will be resolved during the Job build process
+        /// </summary>
+        /// <param name="job"></param>
+        /// <param name="nugetReferences">A collection of Nuget dependencies</param>
+        /// <returns></returns>
+        public static Job WithNuget(this Job job, IReadOnlyCollection<NugetReference> nugetReferences) => job.WithCore(j => j.Infrastructure.NugetReferences = nugetReferences);
 
         // Accuracy
         /// <summary>

src/BenchmarkDotNet/Jobs/NugetReference.cs

@@ -0,0 +1,81 @@
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+
+namespace BenchmarkDotNet.Jobs
+{
+    public class NugetReference : IEquatable<NugetReference>
+    {
+        public NugetReference(string packageName, string packageVersion)
+        {
+            if (string.IsNullOrWhiteSpace(packageName))
+                throw new ArgumentException("message", nameof(packageName));
+
+            PackageName = packageName;
+
+            if (!string.IsNullOrWhiteSpace(PackageVersion) && !IsValidVersion(packageVersion))
+                throw new InvalidOperationException($"Invalid version specified: {packageVersion}");
+
+            PackageVersion = packageVersion;
+
+        }
+
+        public string PackageName { get; }
+        public string PackageVersion { get; }
+
+        public override bool Equals(object obj)
+        {
+            return Equals(obj as NugetReference);
+        }
+
+        /// <summary>
+        /// Object is equals when the package name is the same
+        /// </summary>
+        /// <param name="other"></param>
+        /// <returns></returns>
+        /// <remarks>
+        /// There can only be one package reference of the same name regardless of version
+        /// </remarks>
+        public bool Equals(NugetReference other)
+        {
+            return other != null &&
+                   PackageName == other.PackageName;
+        }
+
+        public override int GetHashCode()
+        {
+            return 557888800 + EqualityComparer<string>.Default.GetHashCode(PackageName);
+        }
+
+        public override string ToString() => $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}";
+
+        /// <summary>
+        /// Tries to validate the version string
+        /// </summary>
+        /// <param name="version"></param>
+        /// <returns></returns>
+        private bool IsValidVersion(string version)
+        {
+            if (string.IsNullOrWhiteSpace(version)) return false;
+            //There is a great nuget package for semver validation called `semver` however we probably
+            // don't want to add another dependency here so this will do some rudimentary validation
+            // and if that fails, then the actual add package command will fail anyways.
+            var parts = version.Split('-');
+            if (parts.Length == 0) return false;
+            if (!Version.TryParse(parts[0], out var _)) return false;
+            for (int i = 1; i < parts.Length; i++)
+            {
+                if (!PreReleaseValidator.IsMatch(parts[i])) return false;
+            }
+            return true;
+        }
+
+        /// <summary>
+        /// Used to validate all pre-release parts of a semver version
+        /// </summary>
+        /// <remarks>
+        /// Allows alphanumeric chars, ".", "+", "-"
+        /// </remarks>
+        private static readonly Regex PreReleaseValidator = new Regex(@"^[0-9A-Za-z\-\+\.]+$", RegexOptions.Compiled);
+    }
+}

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs

@@ -23,10 +23,10 @@ public DotNetCliBuilder(string targetFrameworkMoniker, string customDotNetCliPat
 
         public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
             => new DotNetCliCommand(
-                    CustomDotNetCliPath, 
-                    string.Empty, 
-                    generateResult, 
-                    logger, 
+                    CustomDotNetCliPath,
+                    string.Empty,
+                    generateResult,
+                    logger,
                     buildPartition,
                     Array.Empty<EnvironmentVariable>())
                 .RestoreThenBuild();

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs

@@ -44,6 +44,11 @@ public DotNetCliCommand WithArguments(string arguments)
         [PublicAPI]
         public BuildResult RestoreThenBuild()
         {
+            var packagesResult = AddPackages();
+
+            if (!packagesResult.IsSuccess)
+                return BuildResult.Failure(GenerateResult, new Exception(packagesResult.ProblemDescription));
+
             var restoreResult = Restore();
 
             if (!restoreResult.IsSuccess)
@@ -60,6 +65,11 @@ public BuildResult RestoreThenBuild()
         [PublicAPI]
         public BuildResult RestoreThenBuildThenPublish()
         {
+            var packagesResult = AddPackages();
+
+            if (!packagesResult.IsSuccess)
+                return BuildResult.Failure(GenerateResult, new Exception(packagesResult.ProblemDescription));
+
             var restoreResult = Restore();
 
             if (!restoreResult.IsSuccess)
@@ -76,6 +86,20 @@ public BuildResult RestoreThenBuildThenPublish()
             return Publish().ToBuildResult(GenerateResult);
         }
 
+        public DotNetCliCommandResult AddPackages()
+        {
+            var executionTime = new TimeSpan(0);
+            var stdOutput = new StringBuilder();
+            foreach (var cmd in GetAddPackagesCommands(BuildPartition))
+            {
+                var result = DotNetCliCommandExecutor.Execute(WithArguments(cmd));
+                if (!result.IsSuccess) return result;
+                executionTime.Add(result.ExecutionTime);
+                stdOutput.Append(result.StandardOutput);
+            }
+            return DotNetCliCommandResult.Success(executionTime, stdOutput.ToString());
+        }
+
         public DotNetCliCommandResult Restore()
             => DotNetCliCommandExecutor.Execute(WithArguments(
                 GetRestoreCommand(GenerateResult.ArtifactsPaths, BuildPartition, Arguments)));
@@ -91,7 +115,10 @@ public DotNetCliCommandResult BuildNoDependencies()
         public DotNetCliCommandResult Publish()
             => DotNetCliCommandExecutor.Execute(WithArguments(
                 GetPublishCommand(BuildPartition, Arguments)));
-
+
+        internal static IEnumerable<string> GetAddPackagesCommands(BuildPartition buildPartition)
+            => GetNugetAddPackageCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver);
+
         internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string extraArguments = null) 
             => new StringBuilder(100)
                 .Append("restore ")
@@ -126,5 +153,15 @@ private static string GetCustomMsBuildArguments(BenchmarkCase benchmarkCase, IRe
 
             return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation));
         }
+
+        private static IEnumerable<string> GetNugetAddPackageCommands(BenchmarkCase benchmarkCase, IResolver resolver)
+        {
+            if (!benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic))
+                return Enumerable.Empty<string>();
+
+            var nugetRefs = benchmarkCase.Job.ResolveValue(InfrastructureMode.NugetReferencesCharacteristic, resolver);
+
+            return nugetRefs.Select(x => $"add package {x.PackageName}{(string.IsNullOrWhiteSpace(x.PackageVersion) ? string.Empty : " -v " + x.PackageVersion)}");
+        }
     }
 }

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs

@@ -83,7 +83,7 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif
                 .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition)}")
                 .AppendLine($"call {CliPath ?? "dotnet"} {DotNetCliCommand.GetBuildCommand(buildPartition)}")
                 .ToString();
-            
+
             File.WriteAllText(artifactsPaths.BuildScriptFilePath, content);
         }
 
@@ -96,10 +96,6 @@ protected override void GenerateBuildScript(BuildPartition buildPartition, Artif
         private static bool IsRootSolutionFolder(DirectoryInfo directoryInfo)
             => directoryInfo
                 .GetFileSystemInfos()
-                .Any(fileInfo => fileInfo.Extension == ".sln" || fileInfo.Name == "global.json");
-
-
-
-
+                .Any(fileInfo => fileInfo.Extension == ".sln" || fileInfo.Name == "global.json");          
     }
 }

src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs

@@ -40,6 +40,12 @@ public override bool IsSupported(BenchmarkCase benchmarkCase, ILogger logger, IR
                 return false;
             }
 
+            if (benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic))
+            {
+                logger.WriteLineError("The Roslyn toolchain does not allow specifying Nuget package dependencies");
+                return false;
+            }
+
             return true;
         }
     }

tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj

@@ -28,6 +28,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.3.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="9.0.1" />
     <PackageReference Include="System.Runtime.CompilerServices.Unsafe" Version="4.5.0" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.3.1" />
     <PackageReference Include="xunit" Version="2.3.1" />