feat: Add JobOrderPolicy option to sort jobs by numeric order

src/BenchmarkDotNet/Attributes/OrdererAttribute.cs

@@ -9,9 +9,10 @@ public class OrdererAttribute : Attribute, IConfigSource
     {
         public OrdererAttribute(
             SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default,
-            MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared)
+            MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared,
+            JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default)
         {
-            Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy));
+            Config = ManualConfig.CreateEmpty().WithOrderer(new DefaultOrderer(summaryOrderPolicy, methodOrderPolicy, jobOrderPolicy));
         }
 
         public IConfig Config { get; }

src/BenchmarkDotNet/Jobs/JobComparer.cs

@@ -1,12 +1,23 @@
-using System;
+using BenchmarkDotNet.Characteristics;
+using BenchmarkDotNet.Order;
+using System;
 using System.Collections.Generic;
-using BenchmarkDotNet.Characteristics;
 
 namespace BenchmarkDotNet.Jobs
 {
     internal class JobComparer : IComparer<Job>, IEqualityComparer<Job>
     {
-        public static readonly JobComparer Instance = new JobComparer();
+        private readonly IComparer<string> Comparer;
+
+        public static readonly JobComparer Instance = new JobComparer(JobOrderPolicy.Default);
+        public static readonly JobComparer Numeric = new JobComparer(JobOrderPolicy.Numeric);
+
+        public JobComparer(JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default)
+        {
+            Comparer = jobOrderPolicy == JobOrderPolicy.Default
+                ? StringComparer.Ordinal
+                : new NumericStringComparer();  // TODO: Use `StringComparer.Create(CultureInfo.InvariantCulture, CompareOptions.NumericOrdering)` for .NET10 or greater.
+        }
 
         public int Compare(Job x, Job y)
         {
@@ -39,7 +50,7 @@ public int Compare(Job x, Job y)
                     continue;
                 }
 
-                int compare = string.CompareOrdinal(
+                int compare = Comparer.Compare(
                     presenter.ToPresentation(x, characteristic),
                     presenter.ToPresentation(y, characteristic));
                 if (compare != 0)
@@ -52,5 +63,79 @@ public int Compare(Job x, Job y)
         public bool Equals(Job x, Job y) => Compare(x, y) == 0;
 
         public int GetHashCode(Job obj) => obj.Id.GetHashCode();
+
+        internal class NumericStringComparer : IComparer<string>
+        {
+            public int Compare(string? x, string? y)
+            {
+                if (ReferenceEquals(x, y)) return 0;
+                if (x == null) return -1;
+                if (y == null) return 1;
+
+                ReadOnlySpan<char> spanX = x.AsSpan();
+                ReadOnlySpan<char> spanY = y.AsSpan();
+
+                int i = 0, j = 0;
+
+                while (i < spanX.Length && j < spanY.Length)
+                {
+                    char cx = spanX[i];
+                    char cy = spanY[j];
+
+                    if (!char.IsDigit(cx) || !char.IsDigit(cy))
+                    {
+                        int cmp = cx.CompareTo(cy);
+                        if (cmp != 0)
+                            return cmp;
+
+                        i++;
+                        j++;
+                        continue;
+                    }
+
+                    int ixStart = i;
+                    int iyStart = j;
+
+                    // Skip leading zeros
+                    while (ixStart < spanX.Length && spanX[ixStart] == '0') ixStart++;
+                    while (iyStart < spanY.Length && spanY[iyStart] == '0') iyStart++;
+
+                    int ix = ixStart;
+                    int iy = iyStart;
+
+                    // Skip digits
+                    while (ix < spanX.Length && char.IsDigit(spanX[ix])) ix++;
+                    while (iy < spanY.Length && char.IsDigit(spanY[iy])) iy++;
+
+                    int lenX = ix - ixStart;
+                    int lenY = iy - iyStart;
+
+                    // Compare by digits length
+                    if (lenX != lenY)
+                        return lenX.CompareTo(lenY);
+
+                    // Compare digits
+                    for (int k = 0; k < lenX; k++)
+                    {
+                        int cmp = spanX[ixStart + k].CompareTo(spanY[iyStart + k]);
+                        if (cmp != 0)
+                            return cmp;
+                    }
+
+                    // Compare by leading zeros
+                    int leadingZerosX = ixStart - i;
+                    int leadingZerosY = iyStart - j;
+                    if (leadingZerosX != leadingZerosY)
+                        return 0; // Leading zero differences are ignored (`CompareOptions.NumericOrdering` behavior of .NET)
+
+                    // Move to the next character after the digits
+                    i = ix;
+                    j = iy;
+                }
+
+                // Compare remaining chars
+                return (spanX.Length - i).CompareTo(spanY.Length - j);
+            }
+        }
     }
 }

src/BenchmarkDotNet/Order/DefaultOrderer.cs

@@ -19,18 +19,22 @@ public class DefaultOrderer : IOrderer
 
         private readonly IComparer<string[]> categoryComparer = CategoryComparer.Instance;
         private readonly IComparer<ParameterInstances> paramsComparer = ParameterComparer.Instance;
-        private readonly IComparer<Job> jobComparer = JobComparer.Instance;
+        private readonly IComparer<Job> jobComparer;
         private readonly IComparer<Descriptor> targetComparer;
 
         public SummaryOrderPolicy SummaryOrderPolicy { get; }
         public MethodOrderPolicy MethodOrderPolicy { get; }
 
         public DefaultOrderer(
             SummaryOrderPolicy summaryOrderPolicy = SummaryOrderPolicy.Default,
-            MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared)
+            MethodOrderPolicy methodOrderPolicy = MethodOrderPolicy.Declared,
+            JobOrderPolicy jobOrderPolicy = JobOrderPolicy.Default)
         {
             SummaryOrderPolicy = summaryOrderPolicy;
             MethodOrderPolicy = methodOrderPolicy;
+            jobComparer = jobOrderPolicy == JobOrderPolicy.Default
+                ? JobComparer.Instance
+                : JobComparer.Numeric;
             targetComparer = new DescriptorComparer(methodOrderPolicy);
         }
 

src/BenchmarkDotNet/Order/JobOrderPolicy.cs

@@ -0,0 +1,14 @@
+namespace BenchmarkDotNet.Order;
+
+public enum JobOrderPolicy
+{
+    /// <summary>
+    /// Compare job characteristics in ordinal order.
+    /// </summary>
+    Default,
+
+    /// <summary>
+    /// Compare job characteristics in numeric order.
+    /// </summary>
+    Numeric,
+}

tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs

@@ -0,0 +1,146 @@
+using BenchmarkDotNet.Environments;
+using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Toolchains;
+using BenchmarkDotNet.Toolchains.CsProj;
+using System.Linq;
+using Xunit;
+
+namespace BenchmarkDotNet.Tests.Order;
+
+public class JobOrderTests
+{
+    [Fact]
+    public void TestJobOrders_ByJobId()
+    {
+        // Arrange
+        Job[] jobs =
+        [
+            Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80)
+                   .WithRuntime(CoreRuntime.Core80)
+                   .WithId("v1.4.1"),
+            Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90)
+                   .WithRuntime(CoreRuntime.Core90)
+                   .WithId("v1.4.10"),
+            Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0)
+                   .WithRuntime(CoreRuntime.Core10_0)
+                   .WithId("v1.4.2"),
+        ];
+
+        // Verify jobs are sorted by JobId's ordinal order.
+        {
+            // Act
+            var comparer = JobComparer.Instance;
+            var results = jobs.OrderBy(x => x, comparer)
+                              .Select(x => x.Job.Id)
+                              .ToArray();
+
+            // Assert
+            Assert.Equal(["v1.4.1", "v1.4.10", "v1.4.2"], results);
+        }
+
+        // Verify jobs are sorted by JobId's numeric order.
+        {
+            // Act
+            var comparer = JobComparer.Numeric;
+            var results = jobs.OrderBy(d => d, comparer)
+                              .Select(x => x.Job.Id)
+                              .ToArray();
+            // Assert
+            Assert.Equal(["v1.4.1", "v1.4.2", "v1.4.10"], results);
+        }
+    }
+
+    [Fact]
+    public void TestJobOrders_ByRuntime()
+    {
+        // Arrange
+        Job[] jobs =
+        [
+            Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0)
+                   .WithRuntime(CoreRuntime.Core80),
+            Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90)
+                   .WithRuntime(CoreRuntime.Core90),
+            Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80)
+                   .WithRuntime(CoreRuntime.Core10_0),
+        ];
+
+        // Act
+        // Verify jobs are sorted by Runtime's numeric order.
+        var results = jobs.OrderBy(d => d, JobComparer.Numeric)
+                          .Select(x => x.Job.Environment.GetRuntime().Name)
+                          .ToArray();
+
+        // Assert
+        var expected = new[]
+        {
+            CoreRuntime.Core80.Name,
+            CoreRuntime.Core90.Name,
+            CoreRuntime.Core10_0.Name
+        };
+        Assert.Equal(expected, results);
+    }
+
+    [Fact]
+    public void TestJobOrders_ByToolchain()
+    {
+        // Arrange
+        Job[] jobs =
+        [
+            Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0),
+            Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90),
+            Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80),
+        ];
+
+        // Act
+        // Verify jobs are sorted by Toolchain's numeric order.
+        var results = jobs.OrderBy(d => d, JobComparer.Numeric)
+                          .Select(x => x.Job.GetToolchain().Name)
+                          .ToArray();
+
+        // Assert
+        var expected = new[]
+        {
+            CsProjCoreToolchain.NetCoreApp80.Name,
+            CsProjCoreToolchain.NetCoreApp90.Name,
+            CsProjCoreToolchain.NetCoreApp10_0.Name,
+        };
+        Assert.Equal(expected, results);
+    }
+
+    [Theory]
+    [InlineData("item1", "item1", 0)]
+    [InlineData("item123", "item123", 0)]
+    // Compare different values
+    [InlineData("item1", "item2", -1)]
+    [InlineData("item2", "item1", 1)]
+    [InlineData("item2", "item10", -1)]
+    [InlineData("item10", "item2", 1)]
+    [InlineData("item1a", "item1b", -1)]
+    [InlineData("item1b", "item1a", 1)]
+    [InlineData("item", "item1", -1)]
+    [InlineData("item10", "item", 1)]
+    [InlineData(".NET 8", ".NET 10", -1)]
+    [InlineData(".NET 10", ".NET 8", 1)]
+    [InlineData("v1.4.1", "v1.4.10", -1)]
+    [InlineData("v1.4.10", "v1.4.2", 1)]
+    // Compare zero paddeed numeric string.
+    [InlineData("item01", "item1", 0)]
+    [InlineData("item001", "item1", 0)]
+    [InlineData("item1", "item001", 0)]
+    [InlineData("item1", "item01", 0)]
+    [InlineData("item9", "item09", 0)]
+    [InlineData(".NET 08", ".NET 10", -1)]
+    [InlineData(".NET 10", ".NET 08", 1)]
+    // Arguments that contains null
+    [InlineData(null, "a", -1)]
+    [InlineData("a", null, 1)]
+    [InlineData(null, null, 0)]
+    public void TestNumericComparer(string? a, string? b, int expectedSign)
+    {
+        int result = new JobComparer.NumericStringComparer().Compare(a, b);
+        Assert.Equal(expectedSign, NormalizeSign(result));
+
+        static int NormalizeSign(int value)
+            => value == 0 ? 0 : value < 0 ? -1 : 1;
+    }
+}