[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Author: pre-commit-ci[bot]

conftest.py

@@ -154,11 +154,7 @@ def pytest_addoption(parser: Parser) -> None:
     )
 
     # HuggingFace options
-    hf_group.addoption(
-        "--hf-access-token",
-        default=os.environ.get("HF_ACCESS_TOKEN"),
-        help="HF access token"
-    )
+    hf_group.addoption("--hf-access-token", default=os.environ.get("HF_ACCESS_TOKEN"), help="HF access token")
 
 
 def pytest_cmdline_main(config: Any) -> None:

tests/model_explainability/lm_eval/conftest.py

@@ -47,37 +47,26 @@ def lmevaljob_hf(
             "enabled": True,
         },
         limit="0.01",
-        pod = {
+        pod={
             "container": {
                 "resources": {
-                    "limits": {
-                        "cpu": "1",
-                        "memory": "8Gi",
-                        "nvidia.com/gpu": "1"
-                    },
-                    "requests": {
-                        "cpu": "1",
-                        "memory": "8Gi",
-                        "nvidia.com/gpu": "1"
-                    }
+                    "limits": {"cpu": "1", "memory": "8Gi", "nvidia.com/gpu": "1"},
+                    "requests": {"cpu": "1", "memory": "8Gi", "nvidia.com/gpu": "1"},
                 },
                 "env": [
-                    {   "name": "HF_TOKEN",
+                    {
+                        "name": "HF_TOKEN",
                         "valueFrom": {
                             "secretKeyRef": {
                                 "name": "hf-secret",
                                 "key": "HF_ACCESS_TOKEN",
                             },
                         },
                     },
-                    {
-                        "name": "HF_ALLOW_CODE_EVAL",
-                        "value": "1"
-                    },
+                    {"name": "HF_ALLOW_CODE_EVAL", "value": "1"},
                 ],
             },
         },
-
     ) as job:
         yield job
 
@@ -442,13 +431,14 @@ def lmevaljob_vllm_emulator_pod(
 def lmevaljob_s3_offline_pod(admin_client: DynamicClient, lmevaljob_s3_offline: LMEvalJob) -> Generator[Pod, Any, Any]:
     yield get_lmevaljob_pod(client=admin_client, lmevaljob=lmevaljob_s3_offline)
 
+
 @pytest.fixture(scope="function")
 def lmeval_hf_access_token(
     admin_client: DynamicClient,
     model_namespace: Namespace,
     pytestconfig: Config,
 ) -> Secret:
-    hf_access_token=pytestconfig.option.hf_access_token
+    hf_access_token = pytestconfig.option.hf_access_token
     if not hf_access_token:
         raise MissingParameter(
             "HF access token is not set. "
@@ -461,7 +451,6 @@ def lmeval_hf_access_token(
         string_data={
             "HF_ACCESS_TOKEN": hf_access_token,
         },
-
         wait_for_resource=True,
     ) as secret:
         yield secret

tests/model_explainability/lm_eval/test_lm_eval.py

@@ -10,6 +10,7 @@
 
 LMEVAL_TASKS: List[str] = get_lmeval_tasks(min_downloads=10000)
 
+
 @pytest.mark.parametrize(
     "model_namespace, lmevaljob_hf",
     [
@@ -48,10 +49,9 @@
             },
             id="custom_task",
         ),
-    ], indirect=True,
+    ],
+    indirect=True,
 )
-
-
 def test_lmeval_huggingface_model(admin_client, model_namespace, lmevaljob_hf_pod):
     """Tests that verify running common evaluations (and a custom one) on a model pulled directly from HuggingFace.
     On each test we run a different evaluation task, limiting it to 0.5% of the questions on each eval."""

tests/model_explainability/lm_eval/utils.py

@@ -35,6 +35,7 @@ def get_lmevaljob_pod(client: DynamicClient, lmevaljob: LMEvalJob, timeout: int
 
     return lmeval_pod
 
+
 def get_lmeval_tasks(min_downloads: int = 10000) -> List[str]:
     """
     Gets the list of supported LM-Eval tasks that have above a certain number of minimum downloads on HuggingFace.
@@ -51,22 +52,18 @@ def get_lmeval_tasks(min_downloads: int = 10000) -> List[str]:
     lmeval_tasks = pd.read_csv("tests/model_explainability/lm_eval/data/new_task_list.csv")
 
     # filter for tasks that either exceed (min_downloads OR exist on the OpenLLM leaderboard)
-    # AND exist on LMEval AND do not 
+    # AND exist on LMEval AND do not
 
     filtered_df = lmeval_tasks[
-    lmeval_tasks['Exists'] & (lmeval_tasks['Dataset'] != "MMMU/MMMU") & (
-        (lmeval_tasks['HF dataset downloads'] >= min_downloads) |
-        (lmeval_tasks['OpenLLM leaderboard'])
-        )
+        lmeval_tasks["Exists"]
+        & (lmeval_tasks["Dataset"] != "MMMU/MMMU")
+        & ((lmeval_tasks["HF dataset downloads"] >= min_downloads) | (lmeval_tasks["OpenLLM leaderboard"]))
     ]
 
     # group tasks by dataset and extract the task with shortest name in the group
-    unique_tasks = (
-        filtered_df
-        .loc[filtered_df.groupby('Dataset')['Name'].apply(lambda x: x.str.len().idxmin())]
-    )
+    unique_tasks = filtered_df.loc[filtered_df.groupby("Dataset")["Name"].apply(lambda x: x.str.len().idxmin())]
 
-    unique_tasks = unique_tasks['Name'].tolist()
+    unique_tasks = unique_tasks["Name"].tolist()
 
     LOGGER.info(f"Number of unique LMEval tasks with more than {min_downloads} downloads: {len(unique_tasks)}")