Try to get a helpful error out of deepeval.

Author: lkacenja

python_components/evaluation/evaluation/exception/evaluation.py

@@ -137,14 +137,15 @@ async def _ceq_evaluate(self, document, exception) -> Result:
         )
         details = document.llm_context()
         details.append(decision)
-        test_case = LLMTestCase(
-            actual_output=[response],
-            retrieval_context=context,
-            input="\n\n".join(details),
-        )
-        metric.measure(test_case)
-        if type(metric) is None or metric.verdicts is None:
-            raise RuntimeError("Metric measurement failed. This is likely due to rate limiting.")
+        try:
+            test_case = LLMTestCase(
+                actual_output=[response],
+                retrieval_context=context,
+                input="\n\n".join(details),
+            )
+            metric.measure(test_case)
+        except AttributeError:
+            raise RuntimeError("Metric measurement failed. This is likely due to rate limiting or metric performance.")
         details = {
             "verdicts": convert_model_list(metric.verdicts),
             "response": response,
@@ -169,16 +170,16 @@ async def _faithfulness_evaluate(self, document, exception):
         elif exception == "application":
             response = document.ai_exception["why_application"]
             context = APPLICATION_EXCEPTION_CONTEXT
-
-        metric = MultimodalFaithfulnessMetric(model=self.evaluation_model)
-        test_case = MLLMTestCase(
-            input=[],
-            retrieval_context=context + document.images,
-            actual_output=[response],
-        )
-        metric.measure(test_case)
-        if type(metric) is None or metric.truths is None or metric.claims is None or metric.verdicts is None:
-            raise RuntimeError("Metric measurement failed. This is likely due to rate limiting.")
+        try:
+            metric = MultimodalFaithfulnessMetric(model=self.evaluation_model)
+            test_case = MLLMTestCase(
+                input=[],
+                retrieval_context=context + document.images,
+                actual_output=[response],
+            )
+            metric.measure(test_case)
+        except AttributeError:
+            raise RuntimeError("Metric measurement failed. This is likely due to rate limiting or metric performance.")
         details = {
             "truths": metric.truths,
             "claims": metric.claims,

python_components/evaluation/evaluation/summary/evaluation.py

@@ -41,20 +41,15 @@ def evaluate(self, document: Document) -> List[Result]:
         })))
         logger.info("Summarization complete. Performing related evaluations.")
         document.ai_summary = result["summary"]
-        # Begin the DeepEval summary evaluation.
-        metric = MultimodalInputSummarization(model=self.evaluation_model)
-        test_case = MLLMTestCase(
-            input=document.images, actual_output=document.ai_summary
-        )
-        metric.measure(test_case)
-        if (type(metric) is None
-                or metric.truths is None
-                or metric.claims is None
-                or metric.assessment_questions is None
-                or metric.coverage_verdicts is None
-                or metric.alignment_verdicts is None
-        ):
-            raise RuntimeError("Metric measurement failed. This is likely due to rate limiting.")
+        try:
+            # Begin the DeepEval summary evaluation.
+            metric = MultimodalInputSummarization(model=self.evaluation_model)
+            test_case = MLLMTestCase(
+                input=document.images, actual_output=document.ai_summary
+            )
+            metric.measure(test_case)
+        except AttributeError:
+            raise RuntimeError("Metric measurement failed. This is likely due to rate limiting or metric performance.")
         details = {
             "truths": metric.truths,
             "claims": metric.claims,