Supporting testing between Nuget dependencies

samples/BenchmarkDotNet.Samples/IntroNuget.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Linq;
+using System.Reflection;
+using System.Threading;
+using BenchmarkDotNet.Analysers;
+using BenchmarkDotNet.Attributes;
+using BenchmarkDotNet.Columns;
+using BenchmarkDotNet.Configs;
+using BenchmarkDotNet.Engines;
+using BenchmarkDotNet.Exporters.Csv;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Loggers;
+using BenchmarkDotNet.Toolchains.CsProj;
+
+namespace BenchmarkDotNet.Samples
+{
+    [Config(typeof(Config))]
+    public class IntroNuget
+    {
+        private class Config : ManualConfig
+        {
+            public Config()
+            {
+                //TODO: So far only implemented with any toolchain using DotNetCliBuilder
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.1").WithId("11.0.1"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.3").WithId("10.0.3"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "9.0.1").WithId("9.0.1"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "8.0.1").WithId("8.0.1"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "7.0.1").WithId("7.0.1"));
+                Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "6.0.1").WithId("6.0.1"));
+                Add(DefaultConfig.Instance.GetColumnProviders().ToArray());
+                Add(DefaultConfig.Instance.GetLoggers().ToArray());
+                Add(CsvExporter.Default);
+            }
+        }
+
+        [GlobalSetup]
+        public void Setup()
+        {
+            var jsonConvertMethod = Assembly.Load("Newtonsoft.Json").GetType("Newtonsoft.Json.JsonConvert")
+                .GetMethods().Where(x => x.Name == "SerializeObject" && x.GetParameters().Length == 1)
+                .First();
+
+            Serializer = s => (string)jsonConvertMethod.Invoke(null, new object[] { s });
+        }
+
+        public Func<object, string> Serializer { get; private set; }
+
+        [Benchmark]
+        public void SerializeAnonymousObject() => Serializer(new { hello = "world", price = 1.99, now = DateTime.UtcNow });
+    }
+}

src/BenchmarkDotNet/Environments/InfrastructureResolver.cs

@@ -17,7 +17,8 @@ private InfrastructureResolver()
             Register(InfrastructureMode.EngineFactoryCharacteristic, () => new EngineFactory());
             Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName);
 
-            Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty<Argument>);            
+            Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty<Argument>);
+            Register(InfrastructureMode.NugetReferencesCharacteristic, Array.Empty<NugetReference>);
         }
     }
 }

src/BenchmarkDotNet/Jobs/InfrastructureMode.cs

@@ -16,6 +16,7 @@ public sealed class InfrastructureMode : JobMode<InfrastructureMode>
         public static readonly Characteristic<IEngineFactory> EngineFactoryCharacteristic = CreateCharacteristic<IEngineFactory>(nameof(EngineFactory));
         public static readonly Characteristic<string> BuildConfigurationCharacteristic = CreateCharacteristic<string>(nameof(BuildConfiguration));        
         public static readonly Characteristic<IReadOnlyList<Argument>> ArgumentsCharacteristic = CreateCharacteristic<IReadOnlyList<Argument>>(nameof(Arguments));
+        public static readonly Characteristic<IReadOnlyList<NugetReference>> NugetReferencesCharacteristic = CreateCharacteristic<IReadOnlyList<NugetReference>>(nameof(NugetReferences));
 
         public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessToolchain.Instance);
         public static readonly InfrastructureMode InProcessDontLogOutput = new InfrastructureMode(InProcessToolchain.DontLogOutput);
@@ -60,5 +61,11 @@ public IReadOnlyList<Argument> Arguments
             get => ArgumentsCharacteristic[this];
             set => ArgumentsCharacteristic[this] = value;
         }
+
+        public IReadOnlyList<NugetReference> NugetReferences
+        {
+            get => NugetReferencesCharacteristic[this];
+            set => NugetReferencesCharacteristic[this] = value;
+        }
     }
 }

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -159,8 +159,33 @@ public static class JobExtensions
         public static Job WithCustomBuildConfiguration(this Job job, string buildConfiguration) => job.WithCore(j => j.Infrastructure.BuildConfiguration = buildConfiguration);
         public static Job With(this Job job, IReadOnlyList<EnvironmentVariable> environmentVariables) => job.WithCore(j => j.Environment.EnvironmentVariables = environmentVariables);
         public static Job With(this Job job, IReadOnlyList<Argument> arguments) => job.WithCore(j => j.Infrastructure.Arguments = arguments);
+
+        /// <summary>
+        /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process
+        /// </summary>
+        /// <param name="job"></param>
+        /// <param name="packageName">The Nuget package name</param>
+        /// <param name="packageVersion">The Nuget package version</param>
+        /// <returns></returns>
+        public static Job WithNuget(this Job job, string packageName, string packageVersion) => job.WithCore(j => j.Infrastructure.NugetReferences = new List<NugetReference> { new NugetReference(packageName, packageVersion) });
+
+        /// <summary>
+        /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process
+        /// </summary>
+        /// <param name="job"></param>
+        /// <param name="packageName">The Nuget package name, the latest version will be resolved</param>
+        /// <returns></returns>
+        public static Job WithNuget(this Job job, string packageName) => job.WithCore(j => j.Infrastructure.NugetReferences = new List<NugetReference> { new NugetReference(packageName, string.Empty) });
+
+        /// <summary>
+        /// Runs the job with a specific Nuget dependencies which will be resolved during the Job build process
+        /// </summary>
+        /// <param name="job"></param>
+        /// <param name="nugetReferences">A collection of Nuget dependencies</param>
+        /// <returns></returns>
+        public static Job WithNuget(this Job job, IReadOnlyList<NugetReference> nugetReferences) => job.WithCore(j => j.Infrastructure.NugetReferences = nugetReferences);
 
-    // Accuracy
+        // Accuracy
         /// <summary>
         /// Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error).
         /// The default value is 0.02.

src/BenchmarkDotNet/Jobs/NugetReference.cs

@@ -0,0 +1,26 @@
+using System;
+
+namespace BenchmarkDotNet.Jobs
+{
+    public class NugetReference
+    {
+        public NugetReference(string packageName, string packageVersion)
+        {
+            if (string.IsNullOrWhiteSpace(packageName))
+                throw new ArgumentException("message", nameof(packageName));
+
+            PackageName = packageName;
+            PackageVersion = packageVersion;
+            if (!string.IsNullOrWhiteSpace(PackageVersion) && !Version.TryParse(PackageVersion, out var version))
+                throw new InvalidOperationException($"Invalid version specified: {packageVersion}");
+        }
+
+        public string PackageName { get; }
+        public string PackageVersion { get; }
+
+        public override string ToString()
+        {
+            return $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}";
+        }
+    }
+}

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs

@@ -1,5 +1,6 @@
 using System;
 using System.IO;
+using System.Linq;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Toolchains.Results;
@@ -29,6 +30,22 @@ public virtual string GetBuildCommand(string frameworkMoniker, bool justTheProje
 
         public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
         {
+            //TODO: This doesn't actually use the generate script in the DotNetCliGenerator to do the building... so instead we'll do the nuget checks here as well for now
+            // see: https://github.com/dotnet/BenchmarkDotNet/issues/804
+            var nugetCommands = DotNetCliGenerator.GetNugetPackageCliCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver);
+            foreach(var command in nugetCommands)
+            {
+                var addPackageResult = DotNetCliCommandExecutor.ExecuteCommand(
+                    CustomDotNetCliPath,
+                    command,
+                    generateResult.ArtifactsPaths.BuildArtifactsDirectoryPath,
+                    logger,
+                    useSharedCompilation: null); //exclude the UseSharedCompilation command since that's not compatible with a dotnet add package
+
+                if (!addPackageResult.IsSuccess)
+                    return BuildResult.Failure(generateResult, new Exception(addPackageResult.ProblemDescription));
+            }
+
             var extraArguments = DotNetCliGenerator.GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver);
 
             var restoreResult = DotNetCliCommandExecutor.ExecuteCommand(

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs

@@ -47,9 +47,9 @@ public static CommandResult Failure(TimeSpan time, string standardError, string
 
         internal static CommandResult ExecuteCommand(
             string customDotNetCliPath, string commandWithArguments, string workingDirectory, ILogger logger, 
-            IReadOnlyList<EnvironmentVariable> environmentVariables = null, bool useSharedCompilation = false)
+            IReadOnlyList<EnvironmentVariable> environmentVariables = null, bool? useSharedCompilation = false)
         {
-            commandWithArguments = $"{commandWithArguments} /p:UseSharedCompilation={useSharedCompilation.ToString().ToLowerInvariant()}";
+            commandWithArguments = $"{commandWithArguments}{(useSharedCompilation.HasValue ? $" /p:UseSharedCompilation={useSharedCompilation.ToString().ToLowerInvariant()}" : string.Empty)}";
 
             using (var process = new Process { StartInfo = BuildStartInfo(customDotNetCliPath, workingDirectory, commandWithArguments, environmentVariables) })
             {

src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using BenchmarkDotNet.Characteristics;
@@ -89,8 +90,12 @@ protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths)
 
         protected override void GenerateBuildScript(BuildPartition buildPartition, ArtifactsPaths artifactsPaths)
         {
-            string content = $"call dotnet {Builder.RestoreCommand} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}{Environment.NewLine}" +
-                             $"call dotnet {Builder.GetBuildCommand(TargetFrameworkMoniker, false, buildPartition.BuildConfiguration)} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}";
+            //TODO: This build script isn't actually used, so this logic is also duplicated in the DotNetCliBuilder, see https://github.com/dotnet/BenchmarkDotNet/issues/804
+
+            var nugetPackages = string.Join("call dotnet ", GetNugetPackageCliCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver));
+            string content = string.IsNullOrWhiteSpace(nugetPackages) ? string.Empty : (nugetPackages + Environment.NewLine) +
+                $"call dotnet {Builder.RestoreCommand} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}{Environment.NewLine}" +
+                $"call dotnet {Builder.GetBuildCommand(TargetFrameworkMoniker, false, buildPartition.BuildConfiguration)} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}";
 
             File.WriteAllText(artifactsPaths.BuildScriptFilePath, content);
         }
@@ -115,5 +120,15 @@ internal static string GetCustomArguments(BenchmarkCase benchmarkCase, IResolver
 
             return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation));
         }
+
+        internal static IEnumerable<string> GetNugetPackageCliCommands(BenchmarkCase benchmarkCase, IResolver resolver)
+        {
+            if (!benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic))
+                return Enumerable.Empty<string>();
+
+            var nugetRefs = benchmarkCase.Job.ResolveValue(InfrastructureMode.NugetReferencesCharacteristic, resolver).OfType<NugetReference>();
+
+            return nugetRefs.Select(x => $"add package {x.PackageName}{(string.IsNullOrWhiteSpace(x.PackageVersion) ? string.Empty : " -v " + x.PackageVersion)}");
+        }
     }
 }