From 7947c47fb73977c39cb6e7d9be248113fc7de303 Mon Sep 17 00:00:00 2001
From: Waqas Javed <7674577+w-javed@users.noreply.github.com>
Date: Thu, 10 Jul 2025 22:55:08 -0700
Subject: [PATCH] initial-commit-for-quoting-issue

---
 .../unittests/data/quotation_fix_test_data.jsonl  |  1 +
 .../tests/unittests/test_evaluate.py              | 15 ++++++++++++++-
 .../test_evaluators/test_inputs_evaluators.py     | 15 +++++++++++++++
 3 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 sdk/evaluation/azure-ai-evaluation/tests/unittests/data/quotation_fix_test_data.jsonl

diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/quotation_fix_test_data.jsonl b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/quotation_fix_test_data.jsonl
new file mode 100644
index 000000000000..a72cdcbd1552
--- /dev/null
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/data/quotation_fix_test_data.jsonl
@@ -0,0 +1 @@
+{"query": "\"test\"", "response":"test","ground_truth":"\"test\""}
\ No newline at end of file
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
index 089c2917256e..69f4bf3fccb5 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py
@@ -86,6 +86,9 @@ def evaluate_test_data_alphanumeric():
 def questions_file():
     return _get_file("questions.jsonl")
 
+@pytest.fixture
+def quotation_fix_test_data():
+    return _get_file("quotation_fix_test_data.jsonl")
 
 @pytest.fixture
 def questions_wrong_file():
@@ -687,7 +690,17 @@ def test_general_aggregation(self):
         assert "bad_thing.boolean_with_nan" not in aggregation
         assert "bad_thing.boolean_with_none" not in aggregation
 
-    @pytest.mark.skip(reason="Breaking CI by crashing pytest somehow")
+    def test_quotation_fix_test_data(self, quotation_fix_test_data):
+        from test_evaluators.test_inputs_evaluators import QuotationFixEval
+        result = evaluate(
+                data=quotation_fix_test_data,
+                evaluators={
+                    "test_evaluator": QuotationFixEval,
+                }
+            )
+        print(result)
+        assert result is not None
+        
     def test_optional_inputs_with_data(self, questions_file, questions_answers_basic_file):
         from test_evaluators.test_inputs_evaluators import HalfOptionalEval, NoInputEval, NonOptionalEval, OptionalEval
 
diff --git a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py
index 004aec6741a2..452893f13ae7 100644
--- a/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py
+++ b/sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluators/test_inputs_evaluators.py
@@ -25,6 +25,21 @@ def __init__(self):
 
     def __call__(self, query, *, response="default"):
         return {"half_score": 0 if response == "default" else 1}
+    
+class QuotationFixEval:
+    def __init__(self):
+        pass
+
+    def __call__(self, query, response, ground_truth):
+        if ground_truth == response:
+            return {
+                "score": 1,
+                "reason": "eq"
+            }
+        return {
+            "score": 2,
+            "reason": "ne"
+        } 
 
 
 class OptionalEval: