From fbc1d08f0948800bf8408dc441e711cee0cd2943 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 25 Jun 2018 17:37:23 +1000 Subject: [PATCH 1/2] Initial commit with sample test for Nuget with newtonsoft --- samples/BenchmarkDotNet.Samples/IntroNuget.cs | 52 +++++++++++++++++++ .../Environments/InfrastructureResolver.cs | 3 +- .../Jobs/InfrastructureMode.cs | 7 +++ src/BenchmarkDotNet/Jobs/JobExtensions.cs | 27 +++++++++- src/BenchmarkDotNet/Jobs/NugetReference.cs | 26 ++++++++++ .../Toolchains/DotNetCli/DotNetCliBuilder.cs | 17 ++++++ .../DotNetCli/DotNetCliCommandExecutor.cs | 4 +- .../DotNetCli/DotNetCliGenerator.cs | 19 ++++++- 8 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 samples/BenchmarkDotNet.Samples/IntroNuget.cs create mode 100644 src/BenchmarkDotNet/Jobs/NugetReference.cs diff --git a/samples/BenchmarkDotNet.Samples/IntroNuget.cs b/samples/BenchmarkDotNet.Samples/IntroNuget.cs new file mode 100644 index 0000000000..1e3d4c6f12 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroNuget.cs @@ -0,0 +1,52 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Threading; +using BenchmarkDotNet.Analysers; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Exporters.Csv; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Toolchains.CsProj; + +namespace BenchmarkDotNet.Samples + + [Config(typeof(Config))] + public class IntroNuget + { + private class Config : ManualConfig + { + public Config() + { + //TODO: So far only implemented with any toolchain using DotNetCliBuilder + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "11.0.1").WithId("11.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "10.0.3").WithId("10.0.3")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "9.0.1").WithId("9.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "8.0.1").WithId("8.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "7.0.1").WithId("7.0.1")); + Add(Job.MediumRun.With(CsProjCoreToolchain.Current.Value).WithNuget("Newtonsoft.Json", "6.0.1").WithId("6.0.1")); + Add(DefaultConfig.Instance.GetColumnProviders().ToArray()); + Add(DefaultConfig.Instance.GetLoggers().ToArray()); + Add(CsvExporter.Default); + } + } + + [GlobalSetup] + public void Setup() + { + var jsonConvertMethod = Assembly.Load("Newtonsoft.Json").GetType("Newtonsoft.Json.JsonConvert") + .GetMethods().Where(x => x.Name == "SerializeObject" && x.GetParameters().Length == 1) + .First(); + + Serializer = s => (string)jsonConvertMethod.Invoke(null, new object[] { s }); + } + + public Func Serializer { get; private set; } + + [Benchmark] + public void SerializeAnonymousObject() => Serializer(new { hello = "world", price = 1.99, now = DateTime.UtcNow }); + } +} diff --git a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs index 3e84fc8db8..4b63addf0d 100644 --- a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs +++ b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs @@ -17,7 +17,8 @@ private InfrastructureResolver() Register(InfrastructureMode.EngineFactoryCharacteristic, () => new EngineFactory()); Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName); - Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty); + Register(InfrastructureMode.ArgumentsCharacteristic, Array.Empty); + Register(InfrastructureMode.NugetReferencesCharacteristic, Array.Empty); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs index 6ede02ee91..14908fd06e 100644 --- a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs +++ b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs @@ -16,6 +16,7 @@ public sealed class InfrastructureMode : JobMode public static readonly Characteristic EngineFactoryCharacteristic = CreateCharacteristic(nameof(EngineFactory)); public static readonly Characteristic BuildConfigurationCharacteristic = CreateCharacteristic(nameof(BuildConfiguration)); public static readonly Characteristic> ArgumentsCharacteristic = CreateCharacteristic>(nameof(Arguments)); + public static readonly Characteristic> NugetReferencesCharacteristic = CreateCharacteristic>(nameof(NugetReferences)); public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessToolchain.Instance); public static readonly InfrastructureMode InProcessDontLogOutput = new InfrastructureMode(InProcessToolchain.DontLogOutput); @@ -60,5 +61,11 @@ public IReadOnlyList Arguments get => ArgumentsCharacteristic[this]; set => ArgumentsCharacteristic[this] = value; } + + public IReadOnlyList NugetReferences + { + get => NugetReferencesCharacteristic[this]; + set => NugetReferencesCharacteristic[this] = value; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 55f3140007..efd97171e2 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -159,8 +159,33 @@ public static class JobExtensions public static Job WithCustomBuildConfiguration(this Job job, string buildConfiguration) => job.WithCore(j => j.Infrastructure.BuildConfiguration = buildConfiguration); public static Job With(this Job job, IReadOnlyList environmentVariables) => job.WithCore(j => j.Environment.EnvironmentVariables = environmentVariables); public static Job With(this Job job, IReadOnlyList arguments) => job.WithCore(j => j.Infrastructure.Arguments = arguments); + + /// + /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process + /// + /// + /// The Nuget package name + /// The Nuget package version + /// + public static Job WithNuget(this Job job, string packageName, string packageVersion) => job.WithCore(j => j.Infrastructure.NugetReferences = new List { new NugetReference(packageName, packageVersion) }); + + /// + /// Runs the job with a specific Nuget dependency which will be resolved during the Job build process + /// + /// + /// The Nuget package name, the latest version will be resolved + /// + public static Job WithNuget(this Job job, string packageName) => job.WithCore(j => j.Infrastructure.NugetReferences = new List { new NugetReference(packageName, string.Empty) }); + + /// + /// Runs the job with a specific Nuget dependencies which will be resolved during the Job build process + /// + /// + /// A collection of Nuget dependencies + /// + public static Job WithNuget(this Job job, IReadOnlyList nugetReferences) => job.WithCore(j => j.Infrastructure.NugetReferences = nugetReferences); - // Accuracy + // Accuracy /// /// Maximum acceptable error for a benchmark (by default, BenchmarkDotNet continue iterations until the actual error is less than the specified error). /// The default value is 0.02. diff --git a/src/BenchmarkDotNet/Jobs/NugetReference.cs b/src/BenchmarkDotNet/Jobs/NugetReference.cs new file mode 100644 index 0000000000..69d2b4c389 --- /dev/null +++ b/src/BenchmarkDotNet/Jobs/NugetReference.cs @@ -0,0 +1,26 @@ +using System; + +namespace BenchmarkDotNet.Jobs +{ + public class NugetReference + { + public NugetReference(string packageName, string packageVersion) + { + if (string.IsNullOrWhiteSpace(packageName)) + throw new ArgumentException("message", nameof(packageName)); + + PackageName = packageName; + PackageVersion = packageVersion; + if (!string.IsNullOrWhiteSpace(PackageVersion) && !Version.TryParse(PackageVersion, out var version)) + throw new InvalidOperationException($"Invalid version specified: {packageVersion}"); + } + + public string PackageName { get; } + public string PackageVersion { get; } + + public override string ToString() + { + return $"{PackageName}{(string.IsNullOrWhiteSpace(PackageVersion) ? string.Empty : $" {PackageVersion}")}"; + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index ee973c635e..56dd13ec5d 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; @@ -29,6 +30,22 @@ public virtual string GetBuildCommand(string frameworkMoniker, bool justTheProje public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger) { + //TODO: This doesn't actually use the generate script in the DotNetCliGenerator to do the building... so instead we'll do the nuget checks here as well for now + // see: https://github.com/dotnet/BenchmarkDotNet/issues/804 + var nugetCommands = DotNetCliGenerator.GetNugetPackageCliCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); + foreach(var command in nugetCommands) + { + var addPackageResult = DotNetCliCommandExecutor.ExecuteCommand( + CustomDotNetCliPath, + command, + generateResult.ArtifactsPaths.BuildArtifactsDirectoryPath, + logger, + useSharedCompilation: null); //exclude the UseSharedCompilation command since that's not compatible with a dotnet add package + + if (!addPackageResult.IsSuccess) + return BuildResult.Failure(generateResult, new Exception(addPackageResult.ProblemDescription)); + } + var extraArguments = DotNetCliGenerator.GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver); var restoreResult = DotNetCliCommandExecutor.ExecuteCommand( diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index 73e13e618e..a462aa855e 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -47,9 +47,9 @@ public static CommandResult Failure(TimeSpan time, string standardError, string internal static CommandResult ExecuteCommand( string customDotNetCliPath, string commandWithArguments, string workingDirectory, ILogger logger, - IReadOnlyList environmentVariables = null, bool useSharedCompilation = false) + IReadOnlyList environmentVariables = null, bool? useSharedCompilation = false) { - commandWithArguments = $"{commandWithArguments} /p:UseSharedCompilation={useSharedCompilation.ToString().ToLowerInvariant()}"; + commandWithArguments = $"{commandWithArguments}{(useSharedCompilation.HasValue ? $" /p:UseSharedCompilation={useSharedCompilation.ToString().ToLowerInvariant()}" : string.Empty)}"; using (var process = new Process { StartInfo = BuildStartInfo(customDotNetCliPath, workingDirectory, commandWithArguments, environmentVariables) }) { diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs index 5992fe3d60..3d256e1c78 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; using BenchmarkDotNet.Characteristics; @@ -89,8 +90,12 @@ protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths) protected override void GenerateBuildScript(BuildPartition buildPartition, ArtifactsPaths artifactsPaths) { - string content = $"call dotnet {Builder.RestoreCommand} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}{Environment.NewLine}" + - $"call dotnet {Builder.GetBuildCommand(TargetFrameworkMoniker, false, buildPartition.BuildConfiguration)} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}"; + //TODO: This build script isn't actually used, so this logic is also duplicated in the DotNetCliBuilder, see https://github.com/dotnet/BenchmarkDotNet/issues/804 + + var nugetPackages = string.Join("call dotnet ", GetNugetPackageCliCommands(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)); + string content = string.IsNullOrWhiteSpace(nugetPackages) ? string.Empty : (nugetPackages + Environment.NewLine) + + $"call dotnet {Builder.RestoreCommand} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}{Environment.NewLine}" + + $"call dotnet {Builder.GetBuildCommand(TargetFrameworkMoniker, false, buildPartition.BuildConfiguration)} {GetCustomArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)}"; File.WriteAllText(artifactsPaths.BuildScriptFilePath, content); } @@ -115,5 +120,15 @@ internal static string GetCustomArguments(BenchmarkCase benchmarkCase, IResolver return string.Join(" ", msBuildArguments.Select(arg => arg.TextRepresentation)); } + + internal static IEnumerable GetNugetPackageCliCommands(BenchmarkCase benchmarkCase, IResolver resolver) + { + if (!benchmarkCase.Job.HasValue(InfrastructureMode.NugetReferencesCharacteristic)) + return Enumerable.Empty(); + + var nugetRefs = benchmarkCase.Job.ResolveValue(InfrastructureMode.NugetReferencesCharacteristic, resolver).OfType(); + + return nugetRefs.Select(x => $"add package {x.PackageName}{(string.IsNullOrWhiteSpace(x.PackageVersion) ? string.Empty : " -v " + x.PackageVersion)}"); + } } } \ No newline at end of file From 3de64ced20744adaf109027f0271c6e8dc855d09 Mon Sep 17 00:00:00 2001 From: Shannon Date: Mon, 25 Jun 2018 21:09:29 +1000 Subject: [PATCH 2/2] missing brace (oops!) --- samples/BenchmarkDotNet.Samples/IntroNuget.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/BenchmarkDotNet.Samples/IntroNuget.cs b/samples/BenchmarkDotNet.Samples/IntroNuget.cs index 1e3d4c6f12..d2adeb7f81 100644 --- a/samples/BenchmarkDotNet.Samples/IntroNuget.cs +++ b/samples/BenchmarkDotNet.Samples/IntroNuget.cs @@ -13,7 +13,7 @@ using BenchmarkDotNet.Toolchains.CsProj; namespace BenchmarkDotNet.Samples - +{ [Config(typeof(Config))] public class IntroNuget {