Merge pull request #64 from get-convex/jordan/add-gemini-pro

Add gemini pro to evals

Author: jordanhunt22

.github/workflows/daily_google_evals.yml

@@ -46,7 +46,7 @@ jobs:
 
     - name: Set Google models
       run: |
-        echo "MODELS=gemini-2.0-flash-lite,gemini-2.0-flash,gemini-1.5-flash" >> $GITHUB_ENV
+        echo "MODELS=gemini-2.0-flash-lite,gemini-2.0-flash,gemini-1.5-flash,gemini-2.5-pro-exp-03-25" >> $GITHUB_ENV
 
     - name: Run evaluations
       env:

runner/eval_convex_coding.py

@@ -206,6 +206,7 @@ def convex_coding_task(model: ModelTemplate, input: str):
     "o3-mini",
     "gemini-2.0-flash-lite",
     "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo",
+    "gemini-2.5-pro-exp-03-25",
 ]
 
 if os.getenv("MODELS") is not None:

runner/models/__init__.py

@@ -132,6 +132,14 @@ class ModelTemplate(BaseModel):
         uses_system_prompt=False,
         provider=ModelProvider.GOOGLE,
     ),
+    ModelTemplate(
+        name="gemini-2.5-pro-exp-03-25",
+        formatted_name="Gemini 2.5 Pro (Experimental)",
+        max_concurrency=int(os.getenv("GOOGLE_CONCURRENCY", "4")),
+        requires_chain_of_thought=False,
+        uses_system_prompt=False,
+        provider=ModelProvider.GOOGLE,
+    ),
 ]
 MODELS_BY_NAME = {model.name: model for model in ALL_MODELS}