Fix-for-quoting-issue

sdk/evaluation/azure-ai-evaluation/tests/unittests/data/quotation_fix_test_data.jsonl

@@ -0,0 +1 @@
+{"query": "\"test\"", "response":"test","ground_truth":"\"test\""}

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py

@@ -86,6 +86,9 @@ def evaluate_test_data_alphanumeric():
 def questions_file():
     return _get_file("questions.jsonl")
 
+@pytest.fixture
+def quotation_fix_test_data():
+    return _get_file("quotation_fix_test_data.jsonl")
 
 @pytest.fixture
 def questions_wrong_file():
@@ -687,7 +690,17 @@ def test_general_aggregation(self):
         assert "bad_thing.boolean_with_nan" not in aggregation
         assert "bad_thing.boolean_with_none" not in aggregation
 
-    @pytest.mark.skip(reason="Breaking CI by crashing pytest somehow")
+    def test_quotation_fix_test_data(self, quotation_fix_test_data):
+        from test_evaluators.test_inputs_evaluators import QuotationFixEval
+        result = evaluate(
+                data=quotation_fix_test_data,
+                evaluators={
+                    "test_evaluator": QuotationFixEval,
+                }
+            )
+        print(result)
+        assert result is not None
+
     def test_optional_inputs_with_data(self, questions_file, questions_answers_basic_file):
         from test_evaluators.test_inputs_evaluators import HalfOptionalEval, NoInputEval, NonOptionalEval, OptionalEval
 

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py

@@ -25,6 +25,21 @@ def __init__(self):
 
     def __call__(self, query, *, response="default"):
         return {"half_score": 0 if response == "default" else 1}
+
+class QuotationFixEval:
+    def __init__(self):
+        pass
+
+    def __call__(self, query, response, ground_truth):
+        if ground_truth == response:
+            return {
+                "score": 1,
+                "reason": "eq"
+            }
+        return {
+            "score": 2,
+            "reason": "ne"
+        } 
 
 
 class OptionalEval: