Introduce evaluators for agentic workflows

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/AIToolExtensions.cs

@@ -0,0 +1,44 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI.Evaluation.Quality;
+
+internal static class AIToolExtensions
+{
+    internal static string RenderAsJson(
+        this IEnumerable<AITool> toolDefinitions,
+        JsonSerializerOptions? options = null)
+    {
+        _ = Throw.IfNull(toolDefinitions);
+
+        var toolDefinitionsJsonArray = new JsonArray();
+
+        foreach (AIFunction function in toolDefinitions.OfType<AIFunction>())
+        {
+            JsonNode functionJsonNode =
+                new JsonObject
+                {
+                    ["name"] = function.Name,
+                    ["description"] = function.Description,
+                    ["functionSchema"] = JsonNode.Parse(function.JsonSchema.GetRawText()),
+                };
+
+            if (function.ReturnJsonSchema is not null)
+            {
+                functionJsonNode["functionReturnValueSchema"] =
+                    JsonNode.Parse(function.ReturnJsonSchema.Value.GetRawText());
+            }
+
+            toolDefinitionsJsonArray.Add(functionJsonNode);
+        }
+
+        string renderedToolDefinitions = toolDefinitionsJsonArray.ToJsonString(options);
+        return renderedToolDefinitions;
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/ChatMessageExtensions.cs

@@ -0,0 +1,35 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI.Evaluation.Quality;
+
+internal static class ChatMessageExtensions
+{
+    internal static string RenderAsJson(this IEnumerable<ChatMessage> messages, JsonSerializerOptions? options = null)
+    {
+        _ = Throw.IfNull(messages);
+
+        var messagesJsonArray = new JsonArray();
+
+        foreach (ChatMessage message in messages)
+        {
+            JsonNode? messageJsonNode =
+                JsonSerializer.SerializeToNode(
+                    message,
+                    AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(ChatMessage)));
+
+            if (messageJsonNode is not null)
+            {
+                messagesJsonArray.Add(messageJsonNode);
+            }
+        }
+
+        string renderedMessages = messagesJsonArray.ToJsonString(options);
+        return renderedMessages;
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/ChatResponseExtensions.cs

@@ -0,0 +1,51 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Linq;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI.Evaluation.Quality;
+
+internal static class ChatResponseExtensions
+{
+    internal static string RenderAsJson(this ChatResponse modelResponse, JsonSerializerOptions? options = null)
+    {
+        _ = Throw.IfNull(modelResponse);
+
+        return modelResponse.Messages.RenderAsJson(options);
+    }
+
+    internal static string RenderToolCallsAndResultsAsJson(
+        this ChatResponse modelResponse,
+        JsonSerializerOptions? options = null)
+    {
+        _ = Throw.IfNull(modelResponse);
+
+        var toolCallsAndResultsJsonArray = new JsonArray();
+
+        foreach (AIContent content in modelResponse.Messages.SelectMany(m => m.Contents))
+        {
+            if (content is FunctionCallContent or FunctionResultContent)
+            {
+                Type contentType =
+                    content is FunctionCallContent ? typeof(FunctionCallContent) : typeof(FunctionResultContent);
+
+                JsonNode? toolCallOrResultJsonNode =
+                    JsonSerializer.SerializeToNode(
+                        content,
+                        AIJsonUtilities.DefaultOptions.GetTypeInfo(contentType));
+
+                if (toolCallOrResultJsonNode is not null)
+                {
+                    toolCallsAndResultsJsonArray.Add(toolCallOrResultJsonNode);
+                }
+            }
+        }
+
+        string renderedToolCallsAndResults = toolCallsAndResultsJsonArray.ToJsonString(options);
+        return renderedToolCallsAndResults;
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/CompletenessEvaluator.cs

@@ -87,7 +87,7 @@ public async ValueTask<EvaluationResult> EvaluateAsync(
         {
             metric.AddDiagnostics(
                 EvaluationDiagnostic.Error(
-                    $"A value of type '{nameof(CompletenessEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection."));
+                    $"A value of type {nameof(CompletenessEvaluatorContext)} was not found in the {nameof(additionalContext)} collection."));
 
             return result;
         }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/EquivalenceEvaluator.cs

@@ -86,7 +86,7 @@ public async ValueTask<EvaluationResult> EvaluateAsync(
         {
             metric.AddDiagnostics(
                 EvaluationDiagnostic.Error(
-                    $"A value of type '{nameof(EquivalenceEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection."));
+                    $"A value of type {nameof(EquivalenceEvaluatorContext)} was not found in the {nameof(additionalContext)} collection."));
 
             return result;
         }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/EvaluationMetricExtensions.cs

@@ -33,6 +33,25 @@ internal static EvaluationMetricInterpretation InterpretScore(this NumericMetric
             : new EvaluationMetricInterpretation(rating);
     }
 
+    internal static EvaluationMetricInterpretation InterpretScore(
+        this BooleanMetric metric,
+        bool passValue = true)
+    {
+        EvaluationRating rating = metric.Value switch
+        {
+            null => EvaluationRating.Inconclusive,
+            true => passValue ? EvaluationRating.Exceptional : EvaluationRating.Unacceptable,
+            false => passValue ? EvaluationRating.Unacceptable : EvaluationRating.Exceptional,
+        };
+
+        return metric.Value is bool value && value == passValue
+            ? new EvaluationMetricInterpretation(rating)
+            : new EvaluationMetricInterpretation(
+                rating,
+                failed: true,
+                reason: $"{metric.Name} is not {passValue}.");
+    }
+
     internal static bool TryParseEvaluationResponseWithValue<T>(
         this EvaluationMetric<T> metric,
         ChatResponse evaluationResponse,
@@ -81,7 +100,7 @@ internal static bool TryParseEvaluationResponseWithTags<T>(
 
         static bool TryParseTag(string text, string tagName, [NotNullWhen(true)] out string? tagValue)
         {
-            const RegexOptions Options = RegexOptions.Multiline;
+            const RegexOptions Options = RegexOptions.Singleline;
             Match match = Regex.Match(text, $@"<{tagName}>(?<value>.*?)</{tagName}>", Options);
 
             if (!match.Success || match.Groups["value"] is not Group valueGroup || !valueGroup.Success)
@@ -131,6 +150,11 @@ private static bool TryParseValue<T>(this EvaluationMetric<T> metric, string val
                     booleanMetric.Value = booleanValue;
                     return true;
                 }
+                else if (int.TryParse(valueText, out int intValue) && (intValue is 0 or 1))
+                {
+                    booleanMetric.Value = intValue is 1;
+                    return true;
+                }
                 else
                 {
                     metric.AddDiagnostics(

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/GroundednessEvaluator.cs

@@ -85,7 +85,7 @@ public async ValueTask<EvaluationResult> EvaluateAsync(
         {
             metric.AddDiagnostics(
                 EvaluationDiagnostic.Error(
-                    $"A value of type '{nameof(GroundednessEvaluatorContext)}' was not found in the '{nameof(additionalContext)}' collection."));
+                    $"A value of type {nameof(GroundednessEvaluatorContext)} was not found in the {nameof(additionalContext)} collection."));
 
             return result;
         }