Merge branch 'opendatahub-io:main' into main

Author: dbasunag

.coderabbit.yaml

@@ -0,0 +1,150 @@
+language: en-US
+tone_instructions: ''
+early_access: false
+enable_free_tier: true
+reviews:
+  profile: chill
+  request_changes_workflow: false
+  high_level_summary: true
+  high_level_summary_placeholder: '@coderabbitai summary'
+  high_level_summary_in_walkthrough: true
+  auto_title_placeholder: '@coderabbitai'
+  auto_title_instructions: ''
+  review_status: true
+  commit_status: true
+  fail_commit_status: false
+  collapse_walkthrough: true
+  changed_files_summary: true
+  sequence_diagrams: false
+  assess_linked_issues: true
+  related_issues: true
+  related_prs: true
+  suggested_labels: true
+  auto_apply_labels: false
+  suggested_reviewers: true
+  auto_assign_reviewers: false
+  poem: false
+  labeling_instructions: []
+  path_filters: []
+  path_instructions: []
+  abort_on_close: true
+  disable_cache: false
+  auto_review:
+    enabled: true
+    auto_incremental_review: true
+    ignore_title_keywords: []
+    labels: []
+    drafts: false
+    base_branches: []
+  finishing_touches:
+    docstrings:
+      enabled: true
+    unit_tests:
+      enabled: true
+  tools:
+    ast-grep:
+      rule_dirs: []
+      util_dirs: []
+      essential_rules: true
+      packages: []
+    shellcheck:
+      enabled: true
+    ruff:
+      enabled: true
+    markdownlint:
+      enabled: true
+    github-checks:
+      enabled: true
+      timeout_ms: 90000
+    languagetool:
+      enabled: true
+      enabled_rules: []
+      disabled_rules: []
+      enabled_categories: []
+      disabled_categories: []
+      enabled_only: false
+      level: default
+    biome:
+      enabled: true
+    hadolint:
+      enabled: true
+    swiftlint:
+      enabled: true
+    phpstan:
+      enabled: true
+      level: default
+    golangci-lint:
+      enabled: true
+    yamllint:
+      enabled: true
+    gitleaks:
+      enabled: true
+    checkov:
+      enabled: true
+    detekt:
+      enabled: true
+    eslint:
+      enabled: true
+    rubocop:
+      enabled: true
+    buf:
+      enabled: true
+    regal:
+      enabled: true
+    actionlint:
+      enabled: true
+    pmd:
+      enabled: true
+    cppcheck:
+      enabled: true
+    semgrep:
+      enabled: true
+    circleci:
+      enabled: true
+    clippy:
+      enabled: true
+    sqlfluff:
+      enabled: true
+    prismaLint:
+      enabled: true
+    pylint:
+      enabled: true
+    oxc:
+      enabled: true
+    shopifyThemeCheck:
+      enabled: true
+    luacheck:
+      enabled: true
+    brakeman:
+      enabled: true
+    dotenvLint:
+      enabled: true
+chat:
+  auto_reply: true
+  integrations:
+    jira:
+      usage: auto
+    linear:
+      usage: auto
+knowledge_base:
+  opt_out: false
+  web_search:
+    enabled: true
+  learnings:
+    scope: auto
+  issues:
+    scope: auto
+  jira:
+    usage: auto
+    project_keys: []
+  linear:
+    usage: auto
+    team_keys: []
+  pull_requests:
+    scope: auto
+code_generation:
+  docstrings:
+    language: en-US
+    path_instructions: []
+  unit_tests:
+    path_instructions: []

Dockerfile

@@ -12,7 +12,7 @@ ENV BIN_DIR="$HOME_DIR/.local/bin"
 ENV PATH="$PATH:$BIN_DIR"
 
 RUN apt-get update \
-    && apt-get install -y ssh gnupg software-properties-common curl gpg wget vim \
+    && apt-get install -y ssh gnupg software-properties-common curl gpg wget vim apache2-utils rsync\
     && apt-get clean autoclean \
     && apt-get autoremove --yes \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/
@@ -35,11 +35,6 @@ RUN curl -sSL "https://github.com/fullstorydev/grpcurl/releases/download/v1.9.2/
     && tar xvf /tmp/grpcurl_1.2.tar.gz --no-same-owner \
     && mv grpcurl /usr/bin/grpcurl
 
-# Install htpasswd
-RUN apt-get update && \
-    apt-get install -y apache2-utils && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
 
 RUN useradd -ms /bin/bash $USER
 USER $USER

tests/conftest.py

@@ -42,6 +42,7 @@
 )
 from utilities.infra import update_configmap_data
 from utilities.minio import create_minio_data_connection_secret
+from utilities.operator_utils import get_csv_related_images
 
 LOGGER = get_logger(name=__name__)
 
@@ -517,3 +518,10 @@ def prometheus(admin_client: DynamicClient) -> Prometheus:
         ),  # TODO: Verify SSL with appropriate certs
         bearer_token=get_openshift_token(),
     )
+
+
+@pytest.fixture(scope="session")
+def related_images_refs(admin_client: DynamicClient) -> set[str]:
+    related_images = get_csv_related_images(admin_client=admin_client)
+    related_images_refs = {img["image"] for img in related_images}
+    return related_images_refs

tests/model_explainability/conftest.py

@@ -2,8 +2,12 @@
 
 import pytest
 from kubernetes.dynamic import DynamicClient
+from ocp_resources.config_map import ConfigMap
 from ocp_resources.namespace import Namespace
 from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
+from pytest_testconfig import config as py_config
+
+from tests.model_explainability.trustyai_service.trustyai_service_utils import TRUSTYAI_SERVICE_NAME
 
 
 @pytest.fixture(scope="class")
@@ -19,3 +23,15 @@ def pvc_minio_namespace(
         size="10Gi",
     ) as pvc:
         yield pvc
+
+
+@pytest.fixture(scope="session")
+def trustyai_operator_configmap(
+    admin_client: DynamicClient,
+) -> ConfigMap:
+    return ConfigMap(
+        client=admin_client,
+        namespace=py_config["applications_namespace"],
+        name=f"{TRUSTYAI_SERVICE_NAME}-operator-config",
+        ensure_exists=True,
+    )

tests/model_explainability/lm_eval/conftest.py

@@ -25,42 +25,26 @@
 
 @pytest.fixture(scope="function")
 def lmevaljob_hf(
-    admin_client: DynamicClient, model_namespace: Namespace, patched_trustyai_operator_configmap_allow_online: ConfigMap
+    request: FixtureRequest,
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    patched_trustyai_operator_configmap_allow_online: ConfigMap,
 ) -> Generator[LMEvalJob, None, None]:
     with LMEvalJob(
         client=admin_client,
-        name="test-job",
+        name=LMEVALJOB_NAME,
         namespace=model_namespace.name,
         model="hf",
-        model_args=[{"name": "pretrained", "value": "google/flan-t5-base"}],
-        task_list={
-            "custom": {
-                "systemPrompts": [
-                    {"name": "sp_0", "value": "Be concise. At every point give the shortest acceptable answer."}
-                ],
-                "templates": [
-                    {
-                        "name": "tp_0",
-                        "value": '{ "__type__": "input_output_template", '
-                        '"input_format": "{text_a_type}: {text_a}\\n'
-                        '{text_b_type}: {text_b}", '
-                        '"output_format": "{label}", '
-                        '"target_prefix": '
-                        '"The {type_of_relation} class is ", '
-                        '"instruction": "Given a {text_a_type} and {text_b_type} '
-                        'classify the {type_of_relation} of the {text_b_type} to one of {classes}.",'
-                        ' "postprocessors": [ "processors.take_first_non_empty_line",'
-                        ' "processors.lower_case_till_punc" ] }',
-                    }
-                ],
-            },
-            "taskRecipes": [
-                {"card": {"name": "cards.wnli"}, "systemPrompt": {"ref": "sp_0"}, "template": {"ref": "tp_0"}}
-            ],
-        },
+        model_args=[{"name": "pretrained", "value": "Qwen/Qwen2.5-0.5B"}],
+        task_list=request.param.get("task_list"),
         log_samples=True,
         allow_online=True,
         allow_code_execution=True,
+        system_instruction="Be concise. At every point give the shortest acceptable answer.",
+        chat_template={
+            "enabled": True,
+        },
+        limit="0.01",
     ) as job:
         yield job
 
@@ -80,6 +64,7 @@ def lmevaljob_local_offline(
         model="hf",
         model_args=[{"name": "pretrained", "value": "/opt/app-root/src/hf_home/flan"}],
         task_list=request.param.get("task_list"),
+        limit="0.01",
         log_samples=True,
         offline={"storage": {"pvcName": "lmeval-data"}},
         pod={
@@ -402,6 +387,13 @@ def lmevaljob_hf_pod(admin_client: DynamicClient, lmevaljob_hf: LMEvalJob) -> Ge
     yield get_lmevaljob_pod(client=admin_client, lmevaljob=lmevaljob_hf)
 
 
+@pytest.fixture(scope="function")
+def lmevaljob_local_offline_pod(
+    admin_client: DynamicClient, lmevaljob_local_offline: LMEvalJob
+) -> Generator[Pod, Any, Any]:
+    yield get_lmevaljob_pod(client=admin_client, lmevaljob=lmevaljob_local_offline)
+
+
 @pytest.fixture(scope="function")
 def lmevaljob_vllm_emulator_pod(
     admin_client: DynamicClient, lmevaljob_vllm_emulator: LMEvalJob

tests/model_explainability/lm_eval/test_lm_eval.py

@@ -1,22 +1,65 @@
 import pytest
 
-from tests.model_explainability.lm_eval.utils import verify_lmevaljob_running
 from utilities.constants import Timeout
 
 LMEVALJOB_COMPLETE_STATE: str = "Complete"
 
 
 @pytest.mark.parametrize(
-    "model_namespace",
+    "model_namespace, lmevaljob_hf",
     [
         pytest.param(
-            {"name": "test-lmeval-huggingface"},
-        )
+            {"name": "test-lmeval-hf-arc"}, {"task_list": {"taskNames": ["arc_challenge"]}}, id="arc_challenge"
+        ),
+        pytest.param(
+            {"name": "test-lmeval-hf-mmlu"},
+            {"task_list": {"taskNames": ["mmlu_astronomy_generative"]}},
+            id="mmlu_astronomy_generative",
+        ),
+        pytest.param({"name": "test-lmeval-hf-hellaswag"}, {"task_list": {"taskNames": ["hellaswag"]}}, id="hellaswag"),
+        pytest.param(
+            {"name": "test-lmeval-hf-truthfulqa"}, {"task_list": {"taskNames": ["truthfulqa_gen"]}}, id="truthfulqa_gen"
+        ),
+        pytest.param(
+            {"name": "test-lmeval-hf-winogrande"}, {"task_list": {"taskNames": ["winogrande"]}}, id="winogrande"
+        ),
+        pytest.param(
+            {"name": "test-lmeval-hf-custom-task"},
+            {
+                "task_list": {
+                    "custom": {
+                        "systemPrompts": [
+                            {"name": "sp_0", "value": "Be concise. At every point give the shortest acceptable answer."}
+                        ],
+                        "templates": [
+                            {
+                                "name": "tp_0",
+                                "value": '{ "__type__": "input_output_template", '
+                                '"input_format": "{text_a_type}: {text_a}\\n'
+                                '{text_b_type}: {text_b}", '
+                                '"output_format": "{label}", '
+                                '"target_prefix": '
+                                '"The {type_of_relation} class is ", '
+                                '"instruction": "Given a {text_a_type} and {text_b_type} '
+                                'classify the {type_of_relation} of the {text_b_type} to one of {classes}.",'
+                                ' "postprocessors": [ "processors.take_first_non_empty_line",'
+                                ' "processors.lower_case_till_punc" ] }',
+                            }
+                        ],
+                    },
+                    "taskRecipes": [
+                        {"card": {"name": "cards.wnli"}, "systemPrompt": {"ref": "sp_0"}, "template": {"ref": "tp_0"}}
+                    ],
+                }
+            },
+            id="custom_task",
+        ),
     ],
     indirect=True,
 )
 def test_lmeval_huggingface_model(admin_client, model_namespace, lmevaljob_hf_pod):
-    """Basic test that verifies that LMEval can run successfully pulling a model from HuggingFace."""
+    """Tests that verify running common evaluations (and a custom one) on a model pulled directly from HuggingFace.
+    On each test we run a different evaluation task, limiting it to 1% of the questions on each eval."""
     lmevaljob_hf_pod.wait_for_status(status=lmevaljob_hf_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN)
 
 
@@ -39,10 +82,12 @@ def test_lmeval_local_offline_builtin_tasks_flan_arceasy(
     admin_client,
     model_namespace,
     lmeval_data_downloader_pod,
-    lmevaljob_local_offline,
+    lmevaljob_local_offline_pod,
 ):
     """Test that verifies that LMEval can run successfully in local, offline mode using builtin tasks"""
-    verify_lmevaljob_running(client=admin_client, lmevaljob=lmevaljob_local_offline)
+    lmevaljob_local_offline_pod.wait_for_status(
+        status=lmevaljob_local_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
+    )
 
 
 @pytest.mark.parametrize(
@@ -72,10 +117,12 @@ def test_lmeval_local_offline_unitxt_tasks_flan_20newsgroups(
     admin_client,
     model_namespace,
     lmeval_data_downloader_pod,
-    lmevaljob_local_offline,
+    lmevaljob_local_offline_pod,
 ):
     """Test that verifies that LMEval can run successfully in local, offline mode using unitxt"""
-    verify_lmevaljob_running(client=admin_client, lmevaljob=lmevaljob_local_offline)
+    lmevaljob_local_offline_pod.wait_for_status(
+        status=lmevaljob_local_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_20MIN
+    )
 
 
 @pytest.mark.parametrize(