Merge pull request #2644 from miafig/fix/faithfulness-penalize-ambiguous-description

add ambiguous claims to contradictions when penalized is true

Author: penguine-ip

deepeval/metrics/faithfulness/faithfulness.py

@@ -172,6 +172,8 @@ async def _a_generate_reason(self, multimodal: bool) -> str:
         for verdict in self.verdicts:
             if verdict.verdict.strip().lower() == "no":
                 contradictions.append(verdict.reason)
+            if verdict.verdict.strip().lower() == "idk" and self.penalize_ambiguous_claims:
+                contradictions.append(f"(Ambiguous) {verdict.reason}")
 
         prompt = self.evaluation_template.generate_reason(
             contradictions=contradictions,
@@ -195,6 +197,8 @@ def _generate_reason(self, multimodal: bool) -> str:
         for verdict in self.verdicts:
             if verdict.verdict.strip().lower() == "no":
                 contradictions.append(verdict.reason)
+            if verdict.verdict.strip().lower() == "idk" and self.penalize_ambiguous_claims:
+                contradictions.append(f"(Ambiguous) {verdict.reason}")
 
         prompt = self.evaluation_template.generate_reason(
             contradictions=contradictions,