back to original model configs

Author: Lingjiao Chen

eureka_ml_insights/configs/model_configs.py

@@ -4,7 +4,6 @@
 
 from eureka_ml_insights.models import (
     AzureOpenAIO1Model,
-    AzureOpenAIModel,
     ClaudeModel,
     DirectOpenAIModel,
     DirectOpenAIO1Model,
@@ -14,7 +13,7 @@
     LLaVAModel,
     MistralServerlessAzureRestEndpointModel,
     RestEndpointModel,
-    #TnRModels,
+    TestModel,
 )
 
 from .config import ModelConfig
@@ -23,22 +22,17 @@
 # in the secret_key_params dictionary. OR you can provide the key name and key vault URL to fetch the key from Azure Key Vault.
 # You don't need to provide both the key_vault_url and local_keys_path. You can provide one of them based on your setup.
 
+
+# Test model
+TEST_MODEL_CONFIG = ModelConfig(TestModel, {})
+
 # OpenAI models
 
-'''
 OPENAI_SECRET_KEY_PARAMS = {
     "key_name": "your_openai_secret_key_name",
     "local_keys_path": "keys/keys.json",
     "key_vault_url": None,
 }
-'''
-
-OPENAI_SECRET_KEY_PARAMS = {
-    "key_name": "openai",
-    "local_keys_path": "keys/aifeval-vault-azure-net.json",
-    "key_vault_url": "https://aifeval.vault.azure.net",
-}
-
 
 OAI_O1_PREVIEW_CONFIG = ModelConfig(
     DirectOpenAIO1Model,
@@ -48,14 +42,6 @@
     },
 )
 
-OAI_O1_MINI_CONFIG = ModelConfig(
-    DirectOpenAIO1Model,
-    {
-        "model_name": "o1-mini-2024-09-12",
-        "secret_key_params": OPENAI_SECRET_KEY_PARAMS,
-    },
-)
-
 OAI_O1_PREVIEW_AUZRE_CONFIG = ModelConfig(
     AzureOpenAIO1Model,
     {
@@ -105,110 +91,12 @@
     },
 )
 
-# Azure OAI models
-## Azure OAI models -- TNR Models
-
-TNR_SECRET_KEY_PARAMS = {
-    "key_name": "tnrllmproxy",
-    "local_keys_path": "keys/aifeval-vault-azure-net.json",
-    "key_vault_url": "https://aifeval.vault.azure.net",
-}
-
-GCRAOAI8SW1_AZURE_OAI_O1_PREVIEW_CONFIG = ModelConfig(
-    AzureOpenAIO1Model,
-    {
-        "url": "https://gcraoai8sw1.openai.azure.com/",
-        "model_name": "o1-preview",
-        "api_version": "2024-08-01-preview",
-    }
-)
-
-GCRAOAI8SW1_AZURE_OAI_O1_MINI_CONFIG = ModelConfig(
-    AzureOpenAIO1Model,
-    {
-        "url": "https://gcraoai8sw1.openai.azure.com/",
-        "model_name": "o1-mini",
-        "api_version": "2024-08-01-preview",
-    }
-)
-
-GCRAOAI8SW1_AZURE_OAI_GPT4O_CONFIG = ModelConfig(
-    AzureOpenAIO1Model,
-    {
-        "url": "https://gcraoai8sw1.openai.azure.com/",
-        "model_name": "gpt-4o",
-        "api_version": "2024-08-01-preview",
-        "temperature": 1.0,
-
-    }
-)
-
-
-GCRAOAI8SW1_AZURE_OAI_GPT4_T1_CONFIG = ModelConfig(
-    AzureOpenAIO1Model,
-    {
-        "url": "https://gcraoai8sw1.openai.azure.com/",
-        "model_name": "gpt-4",
-        "api_version": "2024-08-01-preview",
-        "temperature": 1.0,
-
-    }
-)
-
-AzureOpenAIModel
-
-"""
-TNR_GPT4_1106_PREVIEW_CONFIG = ModelConfig(
-    TnRModels,
-    {
-        "url": "https://trapi.research.microsoft.com/gcr/shared/nj/",
-        "secret_key_params": TNR_SECRET_KEY_PARAMS,
-        "model_name": "gpt-4",
-    },
-)
-
-TNR_GPT4_VISION_PREVIEW_CONFIG = ModelConfig(
-    TnRModels,
-    {
-        "url": "https://trapi.research.microsoft.com/gcr/shared/nj/",
-        "secret_key_params": TNR_SECRET_KEY_PARAMS,
-        "model_name": "gpt-4-turbo-v",
-    },
-)
-
-TNR_GPT4V_TURBO_2024_04_09_CONFIG = ModelConfig(
-    TnRModels,
-    {
-        "url": "https://trapi.research.microsoft.com/gcr/shared/nj/",
-        "secret_key_params": TNR_SECRET_KEY_PARAMS,
-        "model_name": "gpt-4-turbo",
-    },
-)
-
-TNR_GPT4O_2024_05_13_CONFIG = ModelConfig(
-    TnRModels,
-    {
-        "url": "https://trapi.research.microsoft.com/gcr/shared/nj/",
-        "secret_key_params": TNR_SECRET_KEY_PARAMS,
-        "model_name": "gpt-4o",
-    },
-)
-"""
-
 # Gemini models
-'''
 GEMINI_SECRET_KEY_PARAMS = {
     "key_name": "your_gemini_secret_key_name",
     "local_keys_path": "keys/keys.json",
     "key_vault_url": None,
 }
-'''
-
-GEMINI_SECRET_KEY_PARAMS = {
-    "key_name": "aif-eval-gemini-firstproject",
-    "local_keys_path": "keys/aifeval-vault-azure-net.json",
-    "key_vault_url": "https://aifeval.vault.azure.net",
-}
 
 GEMINI_V15_PRO_CONFIG = ModelConfig(
     GeminiModel,
@@ -218,37 +106,6 @@
     },
 )
 
-
-GEMINI_V15_PRO_T1_CONFIG = ModelConfig(
-    GeminiModel,
-    {
-        "model_name": "gemini-1.5-pro",
-        "secret_key_params": GEMINI_SECRET_KEY_PARAMS,
-        "temperature":1.0,
-    },
-)
-
-GEMINI_EXP_1206_T1_CONFIG = ModelConfig(
-    GeminiModel,
-    {
-        "model_name": "gemini-exp-1206",
-        "secret_key_params": GEMINI_SECRET_KEY_PARAMS,
-        "temperature":1.0,
-    },
-)
-
-
-GEMINI_EXP_1121_T1_CONFIG = ModelConfig(
-    GeminiModel,
-    {
-        "model_name": "gemini-exp-1121",
-        "secret_key_params": GEMINI_SECRET_KEY_PARAMS,
-        "temperature":1.0,
-    },
-)
-
-
-
 GEMINI_V1_PRO_CONFIG = ModelConfig(
     GeminiModel,
     {
@@ -258,20 +115,11 @@
 )
 
 # Claude models
-'''
 CLAUDE_SECRET_KEY_PARAMS = {
     "key_name": "your_claude_secret_key_name",
     "local_keys_path": "keys/keys.json",
     "key_vault_url": None,
 }
-'''
-
-CLAUDE_SECRET_KEY_PARAMS = {
-    "key_name": "aif-eval-claude",
-    "local_keys_path": "keys/aifeval-vault-azure-net.json",
-    "key_vault_url": "https://aifeval.vault.azure.net",
-}
-
 
 CLAUDE_3_OPUS_CONFIG = ModelConfig(
     ClaudeModel,
@@ -289,32 +137,6 @@
     },
 )
 
-CLAUDE_3_5_SONNET_T1_CONFIG = ModelConfig(
-    ClaudeModel,
-    {
-        "secret_key_params": CLAUDE_SECRET_KEY_PARAMS,
-        "model_name": "claude-3-5-sonnet-20240620",
-        "temperature":1.0,
-    },
-)
-
-CLAUDE_3_5_SONNET_SEARCH_T1_CONFIG = ModelConfig(
-    ClaudeModel,
-    {
-        "secret_key_params": CLAUDE_SECRET_KEY_PARAMS,
-        "model_name": "claude-3-5-sonnet-20241022",
-        "temperature": 1.0,
-    },
-)
-
-CLAUDE_3_5_SONNET_SEARCH_CONFIG = ModelConfig(
-    ClaudeModel,
-    {
-        "secret_key_params": CLAUDE_SECRET_KEY_PARAMS,
-        "model_name": "claude-3-5-sonnet-20241022",
-    },
-)
-
 # LLAVA models
 LLAVAHF_V16_34B_CONFIG = ModelConfig(
     LLaVAHuggingFaceModel,
@@ -377,51 +199,3 @@
         "model_name": "Mistral-large-2407",
     },
 )
-
-
-
-AIF_NT_MISTRAL_LARGE_2_2407_T1_CONFIG = ModelConfig(
-    MistralServerlessAzureRestEndpointModel,
-    {
-        "url": "https://Mistral-large-2407-aifeval.eastus.models.ai.azure.com/v1/chat/completions",
-        "secret_key_params": {
-            "key_name": "aif-nt-mistral-large-2-2407",
-            "local_keys_path": "keys/aifeval-vault-azure-net.json",
-            "key_vault_url": "https://aifeval.vault.azure.net",
-        },
-        "model_name": "Mistral-large-2407-aifeval",
-                "temperature": 1.0,
-
-    },
-)
-
-
-GCR_LLAMA3_1_70B_INSTRUCT_CONFIG = ModelConfig(
-    RestEndpointModel,
-    {
-        "url": "https://gcr-llama31-70b-instruct.westus3.inference.ml.azure.com/score",
-        "secret_key_params": {
-            "key_name": "meta-llama-3-1-70b-instruct-1",
-            "local_keys_path": "keys/aifeval-vault-azure-net.json",
-            "key_vault_url": "https://aifeval.vault.azure.net",
-        },
-        "model_name": "meta-llama-3-1-70b-instruct-1",
-                        "temperature": 1.0,
-
-    },
-)
-
-AIF_NT_LLAMA3_1_405B_INSTRUCT_CONFIG = ModelConfig(
-    LlamaServerlessAzureRestEndpointModel,
-    {
-        "url": "https://Meta-Llama-3-1-405B-Instruct-aif.eastus.models.ai.azure.com/v1/chat/completions",
-        "secret_key_params": {
-            "key_name": "aif-nt-meta-llama-3-1-405b-instruct-1",
-            "local_keys_path": "keys/aifeval-vault-azure-net.json",
-            "key_vault_url": "https://aifeval.vault.azure.net",
-        },
-        "model_name": "Meta-Llama-3-1-405B-Instruct-aif",
-                        "temperature": 1.0,
-
-    },
-)

eureka_ml_insights/user_configs/aime.py

@@ -23,7 +23,6 @@
     MajorityVoteTransform,
     MultiplyTransform,
     SequenceTransform,
-    SamplerTransform,
 )
 from eureka_ml_insights.data_utils.aime_utils import AIMEExtractAnswer
 from eureka_ml_insights.data_utils.data import DataLoader