Extend M.E.AI.Evaluation.Console to support Azure Storage options

eng/packages/General.props

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup>
+    <PackageVersion Include="Azure.Identity" Version="1.13.2" />
     <PackageVersion Include="Azure.Storage.Files.DataLake" Version="12.21.0" />
     <PackageVersion Include="Azure.AI.Inference" Version="1.0.0-beta.3" />
     <PackageVersion Include="ICSharpCode.Decompiler" Version="8.2.0.7535" />

eng/packages/TestOnly.props

@@ -3,7 +3,6 @@
   <ItemGroup>
     <PackageVersion Include="AutoFixture.AutoMoq" Version="4.17.0" />
     <PackageVersion Include="Azure.AI.OpenAI" Version="2.2.0-beta.2" />
-    <PackageVersion Include="Azure.Identity" Version="1.13.2" />
     <PackageVersion Include="autofixture" Version="4.17.0" />
     <PackageVersion Include="BenchmarkDotNet" Version="0.13.5" />
     <PackageVersion Include="FluentAssertions" Version="6.12.0" />

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs

@@ -1,24 +1,44 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using Azure.Identity;
+using Azure.Storage.Files.DataLake;
 using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
+using Microsoft.Extensions.AI.Evaluation.Reporting;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
 using Microsoft.Extensions.Logging;
 
 namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
 
 internal sealed class CleanCacheCommand(ILogger logger)
 {
-    internal async Task<int> InvokeAsync(DirectoryInfo storageRootDir, CancellationToken cancellationToken = default)
+    internal async Task<int> InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default)
     {
-        string storageRootPath = storageRootDir.FullName;
-        logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
-        logger.LogInformation("Deleting expired cache entries...");
+        IResponseCacheProvider cacheProvider;
 
-        var cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
+        if (storageRootDir is not null)
+        {
+            string storageRootPath = storageRootDir.FullName;
+            logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+            logger.LogInformation("Deleting expired cache entries...");
+
+            cacheProvider = new DiskBasedResponseCacheProvider(storageRootPath);
+        }
+        else if (endpointUri is not null)
+        {
+            logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
+
+            var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
+            cacheProvider = new AzureStorageResponseCacheProvider(fsClient);
+        }
+        else
+        {
+            throw new InvalidOperationException("Either --path or --endpoint must be specified");
+        }
 
         await logger.ExecuteWithCatchAsync(
             () => cacheProvider.DeleteExpiredCacheEntriesAsync(cancellationToken)).ConfigureAwait(false);

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs

@@ -1,11 +1,15 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using Azure.Identity;
+using Azure.Storage.Files.DataLake;
 using Microsoft.Extensions.AI.Evaluation.Console.Utilities;
+using Microsoft.Extensions.AI.Evaluation.Reporting;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Storage;
 using Microsoft.Extensions.Logging;
 
@@ -14,14 +18,31 @@ namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
 internal sealed class CleanResultsCommand(ILogger logger)
 {
     internal async Task<int> InvokeAsync(
-        DirectoryInfo storageRootDir,
+        DirectoryInfo? storageRootDir,
+        Uri? endpointUri,
         int lastN,
         CancellationToken cancellationToken = default)
     {
-        string storageRootPath = storageRootDir.FullName;
-        logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+        IResultStore resultStore;
 
-        var resultStore = new DiskBasedResultStore(storageRootPath);
+        if (storageRootDir is not null)
+        {
+            string storageRootPath = storageRootDir.FullName;
+            logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+
+            resultStore = new DiskBasedResultStore(storageRootPath);
+        }
+        else if (endpointUri is not null)
+        {
+            logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
+
+            var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
+            resultStore = new AzureStorageResultStore(fsClient);
+        }
+        else
+        {
+            throw new InvalidOperationException("Either --path or --endpoint must be specified");
+        }
 
         await logger.ExecuteWithCatchAsync(
             async ValueTask () =>

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs

@@ -3,9 +3,12 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using Azure.Identity;
+using Azure.Storage.Files.DataLake;
 using Microsoft.Extensions.AI.Evaluation.Reporting;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Html;
 using Microsoft.Extensions.AI.Evaluation.Reporting.Formats.Json;
@@ -17,17 +20,36 @@ namespace Microsoft.Extensions.AI.Evaluation.Console.Commands;
 internal sealed partial class ReportCommand(ILogger logger)
 {
     internal async Task<int> InvokeAsync(
-        DirectoryInfo storageRootDir,
+        DirectoryInfo? storageRootDir,
+        Uri? endpointUri,
         FileInfo outputFile,
+        bool openReport,
         int lastN,
         Format format,
         CancellationToken cancellationToken = default)
     {
-        string storageRootPath = storageRootDir.FullName;
-        logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+        IResultStore resultStore;
 
-        var results = new List<ScenarioRunResult>();
-        var resultStore = new DiskBasedResultStore(storageRootPath);
+        if (storageRootDir is not null)
+        {
+            string storageRootPath = storageRootDir.FullName;
+            logger.LogInformation("Storage root path: {storageRootPath}", storageRootPath);
+
+            resultStore = new DiskBasedResultStore(storageRootPath);
+        }
+        else if (endpointUri is not null)
+        {
+            logger.LogInformation("Azure Storage endpoint: {endpointUri}", endpointUri);
+
+            var fsClient = new DataLakeDirectoryClient(endpointUri, new DefaultAzureCredential());
+            resultStore = new AzureStorageResultStore(fsClient);
+        }
+        else
+        {
+            throw new InvalidOperationException("Either --path or --endpoint must be specified");
+        }
+
+        List<ScenarioRunResult> results = [];
 
         await foreach (string executionName in
             resultStore.GetLatestExecutionNamesAsync(lastN, cancellationToken).ConfigureAwait(false))
@@ -38,6 +60,8 @@ internal async Task<int> InvokeAsync(
                     cancellationToken: cancellationToken).ConfigureAwait(false))
             {
                 results.Add(result);
+
+                logger.LogInformation("Execution: {executionName} Scenario: {scenarioName} Iteration: {iterationName}", result.ExecutionName, result.ScenarioName, result.IterationName);
             }
         }
 
@@ -58,6 +82,24 @@ internal async Task<int> InvokeAsync(
         await reportWriter.WriteReportAsync(results, cancellationToken).ConfigureAwait(false);
         logger.LogInformation("Report: {outputFilePath} [{format}]", outputFilePath, format);
 
+        // See the following issues for reasoning behind this check. We want to avoid opening the report
+        // if this process is running as a service or in a CI pipeline.
+        // https://github.com/dotnet/runtime/issues/770#issuecomment-564700467
+        // https://github.com/dotnet/runtime/issues/66530#issuecomment-1065854289
+        bool isRedirected = System.Console.IsInputRedirected && System.Console.IsOutputRedirected && System.Console.IsErrorRedirected;
+        bool isInteractive = Environment.UserInteractive && (OperatingSystem.IsWindows() || !(isRedirected));
+
+        if (openReport && isInteractive)
+        {
+            // Open the generated report in the default browser.
+            _ = Process.Start(
+                new ProcessStartInfo
+                {
+                    FileName = outputFilePath,
+                    UseShellExecute = true
+                });
+        }
+
         return 0;
     }
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj

@@ -24,13 +24,16 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Azure.Identity" />
+    <PackageReference Include="Azure.Storage.Files.DataLake" />
     <PackageReference Include="System.CommandLine" />
     <PackageReference Include="Microsoft.Extensions.Logging.Console" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.Extensions.AI.Evaluation\Microsoft.Extensions.AI.Evaluation.csproj" />
     <ProjectReference Include="..\Microsoft.Extensions.AI.Evaluation.Reporting\CSharp\Microsoft.Extensions.AI.Evaluation.Reporting.csproj" />
+    <ProjectReference Include="..\Microsoft.Extensions.AI.Evaluation.Reporting.Azure\Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj" />
   </ItemGroup>
 
   <ItemGroup>

src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Program.cs

@@ -2,10 +2,11 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 #if DEBUG
-using System.CommandLine.Parsing;
 using System.Diagnostics;
 #endif
+using System;
 using System.CommandLine;
+using System.CommandLine.Parsing;
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.Extensions.AI.Evaluation.Console.Commands;
@@ -15,6 +16,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Console;
 
 internal sealed class Program
 {
+    private const string ShortName = "aieval";
     private const string Name = "Microsoft.Extensions.AI.Evaluation.Console";
     private const string Banner = $"{Name} [{Constants.Version}]";
 
@@ -23,7 +25,7 @@ private static async Task<int> Main(string[] args)
 #pragma warning restore EA0014
     {
         using ILoggerFactory factory = LoggerFactory.Create(builder => builder.AddConsole());
-        ILogger logger = factory.CreateLogger(Name);
+        ILogger logger = factory.CreateLogger(ShortName);
         logger.LogInformation("{banner}", Banner);
 
         var rootCmd = new RootCommand(Banner);
@@ -33,19 +35,54 @@ private static async Task<int> Main(string[] args)
         rootCmd.AddGlobalOption(debugOpt);
 #endif
 
-        var reportCmd = new Command("report", "Generate a report ");
+        var reportCmd = new Command("report", "Generate a report from a result store");
 
         var pathOpt =
             new Option<DirectoryInfo>(
                 ["-p", "--path"],
                 "Root path under which the cache and results are stored")
             {
-                IsRequired = true
+                IsRequired = false
             };
 
+        var endpointOpt =
+            new Option<Uri>(
+                ["--endpoint"],
+                "Endpoint URL under which the cache and results are stored for Azure Data Lake Gen2 storage")
+            {
+                IsRequired = false
+            };
+
+        var openReportOpt =
+            new Option<bool>(
+                ["--open"],
+                getDefaultValue: () => false,
+                "Open the report in the default browser")
+            {
+                IsRequired = false,
+            };
+
+        ValidateSymbolResult<CommandResult> requiresPathOrEndpoint = (CommandResult cmd) =>
+        {
+            bool hasPath = cmd.GetValueForOption(pathOpt) is not null;
+            bool hasEndpoint = cmd.GetValueForOption(endpointOpt) is not null;
+            if (!(hasPath ^ hasEndpoint))
+            {
+                cmd.ErrorMessage = $"Either '{pathOpt.Name}' or '{endpointOpt.Name}' must be specified.";
+            }
+        };
+
         reportCmd.AddOption(pathOpt);
+        reportCmd.AddOption(endpointOpt);
+        reportCmd.AddOption(openReportOpt);
+        reportCmd.AddValidator(requiresPathOrEndpoint);
 
-        var outputOpt = new Option<FileInfo>(["-o", "--output"], "Output filename/path") { IsRequired = true };
+        var outputOpt = new Option<FileInfo>(
+            ["-o", "--output"],
+            "Output filename/path")
+        {
+            IsRequired = true,
+        };
         reportCmd.AddOption(outputOpt);
 
         var lastNOpt = new Option<int>(["-n"], () => 1, "Number of most recent executions to include in the report.");
@@ -60,37 +97,44 @@ private static async Task<int> Main(string[] args)
         reportCmd.AddOption(formatOpt);
 
         reportCmd.SetHandler(
-            (path, output, lastN, format) => new ReportCommand(logger).InvokeAsync(path, output, lastN, format),
+            (path, endpoint, output, openReport, lastN, format) => new ReportCommand(logger).InvokeAsync(path, endpoint, output, openReport, lastN, format),
             pathOpt,
+            endpointOpt,
             outputOpt,
+            openReportOpt,
             lastNOpt,
             formatOpt);
 
         rootCmd.Add(reportCmd);
 
         // TASK: Support more granular filters such as the specific scenario / iteration / execution whose results must
         // be cleaned up.
-        var cleanResults = new Command("cleanResults", "Delete results");
-        cleanResults.AddOption(pathOpt);
+        var cleanResultsCmd = new Command("cleanResults", "Delete results");
+        cleanResultsCmd.AddOption(pathOpt);
+        cleanResultsCmd.AddOption(endpointOpt);
+        cleanResultsCmd.AddValidator(requiresPathOrEndpoint);
 
         var lastNOpt2 = new Option<int>(["-n"], () => 0, "Number of most recent executions to preserve.");
-        cleanResults.AddOption(lastNOpt2);
+        cleanResultsCmd.AddOption(lastNOpt2);
 
-        cleanResults.SetHandler(
-            (path, lastN) => new CleanResultsCommand(logger).InvokeAsync(path, lastN),
+        cleanResultsCmd.SetHandler(
+            (path, endpoint, lastN) => new CleanResultsCommand(logger).InvokeAsync(path, endpoint, lastN),
             pathOpt,
+            endpointOpt,
             lastNOpt2);
 
-        rootCmd.Add(cleanResults);
+        rootCmd.Add(cleanResultsCmd);
 
-        var cleanCache = new Command("cleanCache", "Delete expired cache entries");
-        cleanCache.AddOption(pathOpt);
+        var cleanCacheCmd = new Command("cleanCache", "Delete expired cache entries");
+        cleanCacheCmd.AddOption(pathOpt);
+        cleanCacheCmd.AddOption(endpointOpt);
+        cleanCacheCmd.AddValidator(requiresPathOrEndpoint);
 
-        cleanCache.SetHandler(
-            path => new CleanCacheCommand(logger).InvokeAsync(path),
-            pathOpt);
+        cleanCacheCmd.SetHandler(
+            (path, endpoint) => new CleanCacheCommand(logger).InvokeAsync(path, endpoint),
+            pathOpt, endpointOpt);
 
-        rootCmd.Add(cleanCache);
+        rootCmd.Add(cleanCacheCmd);
 
         // TASK: Support some mechanism to fail a build (i.e. return a failure exit code) based on one or more user
         // specified criteria (e.g., if x% of metrics were deemed 'poor'). Ideally this mechanism would be flexible /
@@ -106,4 +150,5 @@ private static async Task<int> Main(string[] args)
 
         return await rootCmd.InvokeAsync(args).ConfigureAwait(false);
     }
+
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/Directory.Build.targets

@@ -67,7 +67,7 @@ internal static class Constants
     </ItemGroup>
   </Target>
 
-  <!-- Generate a version file that can be accessed from the script that builds the Azure DevOps Extension pacakge. -->
+  <!-- Generate a version file that can be accessed from the script that builds the Azure DevOps Extension package. -->
   <Target Name="StampVSIXPackageVersion" BeforeTargets="DispatchToInnerBuilds">
     <PropertyGroup>
       <_VSIXPackageVersionFile>$(MSBuildThisFileDirectory)\TypeScript\azure-devops-report\VSIXPackageVersion.json</_VSIXPackageVersionFile>