Gsm8k response

deepeval/benchmarks/gsm8k/gsm8k.py

@@ -1,4 +1,4 @@
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Union, Tuple, Any
 from tqdm import tqdm
 
 from deepeval.dataset import Golden
@@ -95,11 +95,19 @@ def predict(self, model: DeepEvalBaseLLM, golden: Golden) -> Dict:
 
         # Enforced model generation
         try:
-            res: NumberSchema = model.generate(
+            res: Union[NumberSchema, Tuple[Any]] = model.generate(
                 prompt=prompt, schema=NumberSchema
             )
-            prediction = str(res.answer)
-        except TypeError:
+            if isinstance(res, tuple):
+                if len(res) == 1:
+                    res = res[0]
+                else:
+                    raise TypeError
+            elif isinstance(res, NumberSchema):
+                prediction = str(res.answer)
+            else:
+                raise TypeError # Much eaiser to just add confinement instructions
+        except (TypeError, AttributeError):
             prompt += f"\n\n{self.confinement_instructions}"
             prediction = model.generate(prompt)
 

deepeval/cli/main.py

@@ -381,7 +381,7 @@ def unset_local_embeddings_env():
 
 
 #############################################
-# Ollama Integration ########################
+# Gemini Integration ########################
 #############################################
 
 

deepeval/metrics/g_eval/g_eval.py

@@ -348,7 +348,7 @@ def is_successful(self) -> bool:
             self.success = False
         else:
             try:
-                self.score >= self.threshold
+                self.success = self.score >= self.threshold
             except:
                 self.success = False
         return self.success