From fbee46b52d77fa6c721d8991a8f026c79c88fe70 Mon Sep 17 00:00:00 2001 From: Kumar Anirudha Date: Mon, 29 Sep 2025 18:26:55 +0530 Subject: [PATCH] fix: answer_relevancy scoring logic to prevent false zero scores for valid answers --- src/ragas/metrics/_answer_relevance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ragas/metrics/_answer_relevance.py b/src/ragas/metrics/_answer_relevance.py index 420c524c58..42be40afb1 100644 --- a/src/ragas/metrics/_answer_relevance.py +++ b/src/ragas/metrics/_answer_relevance.py @@ -116,7 +116,7 @@ def _calculate_score( ) -> float: question = row["user_input"] gen_questions = [answer.question for answer in answers] - committal = np.any([answer.noncommittal for answer in answers]) + all_noncommittal = np.all([answer.noncommittal for answer in answers]) if all(q == "" for q in gen_questions): logger.warning( "Invalid JSON response. Expected dictionary with key 'question'" @@ -124,7 +124,7 @@ def _calculate_score( score = np.nan else: cosine_sim = self.calculate_similarity(question, gen_questions) - score = cosine_sim.mean() * int(not committal) + score = cosine_sim.mean() * int(not all_noncommittal) return score