Partition benchmark run info based on added nuget packages

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -192,15 +192,15 @@ public static Job With(this Job job, IReadOnlyList<EnvironmentVariable> environm
         /// <param name="packageName">The Nuget package name</param>
         /// <param name="packageVersion">The Nuget package version</param>
         /// <returns></returns>
-        public static Job WithNuget(this Job job, string packageName, string packageVersion) => job.WithCore(j => j.Infrastructure.NugetReferences = new HashSet<NugetReference>(j.Infrastructure.NugetReferences ?? Array.Empty<NugetReference>()) { new NugetReference(packageName, packageVersion) });
+        public static Job WithNuget(this Job job, string packageName, string packageVersion) => job.WithCore(j => j.Infrastructure.NugetReferences = new NugetReferenceList(j.Infrastructure.NugetReferences ?? Array.Empty<NugetReference>()) { new NugetReference(packageName, packageVersion) });
 
         /// <summary>
         /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process
         /// </summary>
         /// <param name="job"></param>
         /// <param name="packageName">The Nuget package name, the latest version will be resolved</param>
         /// <returns></returns>
-        public static Job WithNuget(this Job job, string packageName) => job.WithCore(j => j.Infrastructure.NugetReferences = new HashSet<NugetReference>(j.Infrastructure.NugetReferences ?? Array.Empty<NugetReference>()) { new NugetReference(packageName, string.Empty) });
+        public static Job WithNuget(this Job job, string packageName) => job.WithNuget(packageName, string.Empty);
 
         /// <summary>
         /// Runs the job with a specific Nuget dependencies which will be resolved during the Job build process

src/BenchmarkDotNet/Jobs/NugetReference.cs

@@ -29,22 +29,20 @@ public override bool Equals(object obj)
         }
 
         /// <summary>
-        /// Object is equals when the package name is the same
+        /// Object is equals when the package name and version are the same
         /// </summary>
         /// <param name="other"></param>
         /// <returns></returns>
-        /// <remarks>
-        /// There can only be one package reference of the same name regardless of version
-        /// </remarks>
         public bool Equals(NugetReference other)
         {
             return other != null &&
-                   PackageName == other.PackageName;
+                   PackageName == other.PackageName &&
+                   PackageVersion == other.PackageVersion;
         }
 
         public override int GetHashCode()
         {
-            return 557888800 + EqualityComparer<string>.Default.GetHashCode(PackageName);
+            return 557888800 + EqualityComparer<string>.Default.GetHashCode(PackageName) + EqualityComparer<string>.Default.GetHashCode(PackageVersion).GetHashCode();
         }
 
         public override string ToString() => $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}";

src/BenchmarkDotNet/Jobs/NugetReferenceList.cs

@@ -0,0 +1,89 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+
+namespace BenchmarkDotNet.Jobs
+{
+    /// <summary>
+    /// An ordered list of Nuget references. Does not allow duplicate references with the same PackageName.
+    /// </summary>
+    public class NugetReferenceList : IReadOnlyCollection<NugetReference>
+    {
+        private readonly List<NugetReference> references = new List<NugetReference>();
+
+        public NugetReferenceList()
+        {
+        }
+
+        public NugetReferenceList(IReadOnlyCollection<NugetReference> readOnlyCollection)
+        {
+            foreach (var nugetReference in readOnlyCollection)
+            {
+                Add(nugetReference);
+            }
+        }
+
+        public int Count => references.Count;
+
+        public IEnumerator<NugetReference> GetEnumerator() => references.GetEnumerator();
+
+        IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+
+        public void Add(NugetReference reference)
+        {
+            if (reference == null)
+                throw new ArgumentNullException(nameof(reference));
+
+            var insertionIndex = references.BinarySearch(reference, PackageNameComparer.Default);
+            if (0 <= insertionIndex && insertionIndex < Count)
+                throw new ArgumentException($"Nuget package {reference.PackageName} was already added", nameof(reference));
+
+            references.Insert(~insertionIndex, reference);
+        }
+
+        public override bool Equals(object obj)
+        {
+            if (ReferenceEquals(null, obj))
+                return false;
+            if (ReferenceEquals(this, obj))
+                return true;
+            if (obj.GetType() != GetType())
+                return false;
+
+            var otherList = (NugetReferenceList)obj;
+            if (Count != otherList.Count)
+                return false;
+
+            for (int i = 0; i < Count; i++)
+            {
+                if (!references[i].Equals(otherList.references[i]))
+                    return false;
+            }
+
+            return true;
+        }
+
+        public override int GetHashCode()
+        {
+            unchecked
+            {
+                int hashCode = 0;
+                foreach (var nugetReference in references)
+                {
+                    hashCode = hashCode * 397 + nugetReference.GetHashCode();
+                }
+
+                return hashCode;
+            }
+        }
+
+        private class PackageNameComparer : IComparer<NugetReference>
+        {
+            private PackageNameComparer() { }
+
+            public static PackageNameComparer Default { get; } = new PackageNameComparer();
+
+            public int Compare(NugetReference x, NugetReference y) => Comparer<string>.Default.Compare(x.PackageName, y.PackageName);
+        }
+    }
+}

src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs

@@ -45,6 +45,8 @@ public bool Equals(BenchmarkCase x, BenchmarkCase y)
                     return false;
                 if (AreDifferent(jobX.Infrastructure.Arguments, jobY.Infrastructure.Arguments)) // arguments can be anything (Mono runtime settings or MsBuild parameters)
                     return false;
+                if (AreDifferent(jobX.Infrastructure.NugetReferences, jobY.Infrastructure.NugetReferences))
+                    return false;
                 if (!jobX.Environment.Gc.Equals(jobY.Environment.Gc)) // GC settings are per .config/.csproj
                     return false;
 
@@ -78,6 +80,8 @@ public int GetHashCode(BenchmarkCase obj)
                     hashCode ^= job.Infrastructure.BuildConfiguration.GetHashCode();
                 if (job.Infrastructure.Arguments != null && job.Infrastructure.Arguments.Any())
                     hashCode ^= job.Infrastructure.Arguments.GetHashCode();
+                if (job.Infrastructure.NugetReferences != null)
+                    hashCode ^= job.Infrastructure.NugetReferences.GetHashCode();
                 if (!string.IsNullOrEmpty(obj.Descriptor.AdditionalLogic))
                     hashCode ^= obj.Descriptor.AdditionalLogic.GetHashCode();
 

tests/BenchmarkDotNet.Tests/JobTests.cs

@@ -470,14 +470,27 @@ public static void WithNuget()
             Assert.Equal(1, j.Infrastructure.NugetReferences.Count);
 
             j = j.WithNuget("AutoMapper", "7.0.1");
-            Assert.Equal(2, j.Infrastructure.NugetReferences.Count); //appends
+            Assert.Collection(j.Infrastructure.NugetReferences,
+                reference => Assert.Equal(new NugetReference("AutoMapper", "7.0.1"), reference),
+                reference => Assert.Equal(new NugetReference("Newtonsoft.Json", ""), reference));
 
-            j = j.WithNuget("AutoMapper");
-            Assert.Equal(2, j.Infrastructure.NugetReferences.Count); //does not append, same package
-            j = j.WithNuget("AutoMapper", "7.0.0-alpha-0001");
-            Assert.Equal(2, j.Infrastructure.NugetReferences.Count); //does not append, same package
+            Assert.Throws<ArgumentException>(() => j = j.WithNuget("AutoMapper")); //adding is an error, since it's the same package
+            Assert.Throws<ArgumentException>(() => j = j.WithNuget("AutoMapper", "7.0.0-alpha-0001")); //adding is an error, since it's the same package
 
-            Assert.Equal("AutoMapper 7.0.1", j.Infrastructure.NugetReferences.ElementAt(1).ToString()); //first package reference in wins
+            j = j.WithNuget("NLog", "4.5.10"); // ensure we can add at the end of a non-empty list
+            Assert.Collection(j.Infrastructure.NugetReferences,
+                reference => Assert.Equal(new NugetReference("AutoMapper", "7.0.1"), reference),
+                reference => Assert.Equal(new NugetReference("Newtonsoft.Json", ""), reference),
+                reference => Assert.Equal(new NugetReference("NLog", "4.5.10"), reference));
+
+            var expected = new NugetReferenceList(Array.Empty<NugetReference>())
+            {
+                new NugetReference("AutoMapper", "7.0.1"),
+                new NugetReference("Newtonsoft.Json", ""),
+                new NugetReference("NLog", "4.5.10"),
+            };
+
+            Assert.Equal(expected, j.Infrastructure.NugetReferences); // ensure that the list's equality operator returns true when the contents are the same
         }
 
         private static bool IsSubclassOfobModeOfItself(Type type)

tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs

@@ -82,5 +82,48 @@ public void CustomClrBuildJobsAreGroupedByVersion()
             foreach (var grouping in grouped)
                 Assert.Equal(3 * 2, grouping.Count()); // (M1 + M2 + M3) * (Plain1 + Plain2)
         }
+
+        [Fact]
+        public void CustomNugetJobsWithSamePackageVersionAreGroupedTogether()
+        {
+            var job1 = Job.Default.WithNuget("AutoMapper", "7.0.1");
+            var job2 = Job.Default.WithNuget("AutoMapper", "7.0.1");
+
+            var config = ManualConfig.Create(DefaultConfig.Instance)
+                .With(job1)
+                .With(job2);
+
+            var benchmarks1 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain1), config);
+            var benchmarks2 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain2), config);
+
+            var grouped = benchmarks1.BenchmarksCases.Union(benchmarks2.BenchmarksCases)
+                .GroupBy(benchmark => benchmark, new BenchmarkPartitioner.BenchmarkRuntimePropertiesComparer())
+                .ToArray();
+
+            Assert.Single(grouped); // 7.0.1
+
+            foreach (var grouping in grouped)
+                Assert.Equal(2 * 3 * 2, grouping.Count()); // ((job1 + job2) * (M1 + M2 + M3) * (Plain1 + Plain2)
+        }
+
+        [Fact]
+        public void CustomNugetJobsAreGroupedByPackageVersion()
+        {
+            var config = ManualConfig.Create(DefaultConfig.Instance)
+                .With(Job.Default.WithNuget("AutoMapper", "7.0.1"))
+                .With(Job.Default.WithNuget("AutoMapper", "7.0.0-alpha-0001"));
+
+            var benchmarks1 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain1), config);
+            var benchmarks2 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain2), config);
+
+            var grouped = benchmarks1.BenchmarksCases.Union(benchmarks2.BenchmarksCases)
+                .GroupBy(benchmark => benchmark, new BenchmarkPartitioner.BenchmarkRuntimePropertiesComparer())
+                .ToArray();
+
+            Assert.Equal(2, grouped.Length); // 7.0.1 + 7.0.0-alpha-0001
+
+            foreach (var grouping in grouped)
+                Assert.Equal(3 * 2, grouping.Count()); // (M1 + M2 + M3) * (Plain1 + Plain2)
+        }
     }
 }