ci: merge main branch

rnetser · rnetser · commit bffc362d5e6e · 2025-04-08T18:33:14.000+03:00
diff --git a/.github/workflows/add-remove-labels.yml b/.github/workflows/add-remove-labels.yml
@@ -11,12 +11,6 @@ on:
 
   issue_comment:
     types: [created, edited, deleted]
-    # I don't believe the conditional is supported here
-    # if: |
-    #   contains(github.event.comment.body, '/wip') ||
-    #   contains(github.event.comment.body, '/verified') ||
-    #   contains(github.event.comment.body, '/lgtm') ||
-    #   contains(github.event.comment.body, '/hold')
 
 
 permissions:
@@ -26,6 +20,12 @@ permissions:
 
 jobs:
   add-remove-labels:
+    if: |
+      contains(github.event.comment.body, '/wip') ||
+      contains(github.event.comment.body, '/verified') ||
+      contains(github.event.comment.body, '/lgtm') ||
+      contains(github.event.comment.body, '/hold') ||
+      contains(github.event.comment.body, '/cherry-pick')
     runs-on: ubuntu-latest
 
     steps:
@@ -36,27 +36,6 @@ jobs:
           comment-id: ${{ github.event.comment.id }}
           reactions: '+1'
 
-      # This currently fails with either the bot PAT or the standard github token secret
-      # gh: Insufficient scopes for reacting to this Pull Request Review Comment. (HTTP 403)
-      # {"message":"Insufficient scopes for reacting to this Pull Request Review Comment.","documentation_url":"https://docs.github.com/rest/reactions/reactions#create-reaction-for-a-pull-request-review-comment","status":"403"}
-      # It could work if we had a token with the proper permissions.
-      # See https://github.com/peter-evans/create-or-update-comment/issues/392 for why the action above doesn't work.
-      # Confirmed as a bug, see: https://github.com/github/docs/issues/36899
-      # - name: Acknowledge the review with thumbs up reaction
-      #   if: ${{ github.event.review }}
-      #   env:
-      #     GH_TOKEN: ${{ secrets.OPENDATAHUB_TESTS_BOT_PAT }}
-      #     REVIEW_COMMENT_ID: ${{ github.event.review.id }}
-      #     REPO_NAME: ${{ github.event.repository.name }}
-      #     REPO_OWNER: ${{ github.event.repository.owner.login }}
-      #   run: |
-      #     gh api \
-      #     --method POST \
-      #     -H "Accept: application/vnd.github+json" \
-      #     -H "X-GitHub-Api-Version: 2022-11-28" \
-      #     /repos/$REPO_OWNER/$REPO_NAME/pulls/comments/$REVIEW_COMMENT_ID/reactions \
-      #     -f "content=+1"
-
       - uses: actions/checkout@v4
 
       - name: Install uv
diff --git a/.github/workflows/cherry-pick-comment.yml b/.github/workflows/cherry-pick-comment.yml
@@ -0,0 +1,25 @@
+name: Cherry Pick On Comment
+on:
+  issue_comment:
+    types: [created]
+jobs:
+  cherry-pick:
+    name: Cherry Pick
+    if: |
+      github.event.issue.pull_request != '' &&
+      contains(github.event.comment.body, '/cherry-pick') &&
+      ((github.event.pull_request.author_association != 'NONE') &&
+      (github.event.pull_request.author_association != 'MANNEQUIN') &&
+      (github.event.pull_request.author_association != 'FIRST_TIMER') &&
+      (github.event.pull_request.author_association != 'FIRST_TIME_CONTRIBUTOR'))
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the latest code
+        uses: actions/checkout@v2
+        with:
+          token: ${{ secrets.OPENDATAHUB_TESTS_BOT_PAT }}
+          fetch-depth: 0 # otherwise, you will fail to push refs to dest repo
+      - name: Automatic Cherry Pick
+        uses:  dbasunag/cherry-pick-pr@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.OPENDATAHUB_TESTS_BOT_PAT }}
diff --git a/.github/workflows/scripts/constants.py b/.github/workflows/scripts/constants.py
@@ -35,6 +35,8 @@
         `lgtm` label removed on each new commit push.
  * To mark PR as verified comment `/verified` to the PR, to un-verify comment `/verified cancel` to the PR.
         `verified` label removed on each new commit push.
+ * To Cherry-pick a merged PR `/cherry-pick <target_branch_name>` to the PR. If <target_branch_name> is valid,
+         and the current PR is merged, a cherry-picked PR would be created and linked to the current PR.
 
 <details>
 <summary>Supported labels</summary>
diff --git a/tests/model_explainability/conftest.py b/tests/model_explainability/conftest.py
@@ -0,0 +1,21 @@
+from typing import Generator, Any
+
+import pytest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.namespace import Namespace
+from ocp_resources.persistent_volume_claim import PersistentVolumeClaim
+
+
+@pytest.fixture(scope="class")
+def pvc_minio_namespace(
+    admin_client: DynamicClient, minio_namespace: Namespace
+) -> Generator[PersistentVolumeClaim, Any, Any]:
+    with PersistentVolumeClaim(
+        client=admin_client,
+        name="minio-pvc",
+        namespace=minio_namespace.name,
+        accessmodes=PersistentVolumeClaim.AccessMode.RWO,
+        volume_mode=PersistentVolumeClaim.VolumeMode.FILE,
+        size="10Gi",
+    ) as pvc:
+        yield pvc
diff --git a/tests/model_explainability/guardrails/conftest.py b/tests/model_explainability/guardrails/conftest.py
@@ -209,7 +209,7 @@ def vllm_gateway_config(admin_client: DynamicClient, model_namespace: Namespace)
 def minio_llm_deployment(
     admin_client: DynamicClient,
     minio_namespace: Namespace,
-    llm_models_pvc: PersistentVolumeClaim,
+    pvc_minio_namespace: PersistentVolumeClaim,
 ) -> Generator[Deployment, Any, Any]:
     with Deployment(
         client=admin_client,
@@ -229,7 +229,7 @@ def minio_llm_deployment(
                 "volumes": [
                     {
                         "name": "model-volume",
-                        "persistentVolumeClaim": {"claimName": "llm-models-claim"},
+                        "persistentVolumeClaim": {"claimName": pvc_minio_namespace.name},
                     }
                 ],
                 "initContainers": [
@@ -282,18 +282,3 @@ def minio_llm_deployment(
     ) as deployment:
         deployment.wait_for_replicas(timeout=Timeout.TIMEOUT_10MIN)
         yield deployment
-
-
-@pytest.fixture(scope="class")
-def llm_models_pvc(
-    admin_client: DynamicClient, minio_namespace: Namespace
-) -> Generator[PersistentVolumeClaim, Any, Any]:
-    with PersistentVolumeClaim(
-        client=admin_client,
-        name="llm-models-claim",
-        namespace=minio_namespace.name,
-        accessmodes=PersistentVolumeClaim.AccessMode.RWO,
-        volume_mode=PersistentVolumeClaim.VolumeMode.FILE,
-        size="10Gi",
-    ) as pvc:
-        yield pvc
diff --git a/tests/model_explainability/lm_eval/conftest.py b/tests/model_explainability/lm_eval/conftest.py
@@ -2,6 +2,7 @@
 
 import pytest
 from ocp_resources.route import Route
+from ocp_resources.secret import Secret
 from ocp_resources.service import Service
 from pytest import FixtureRequest
 from kubernetes.dynamic import DynamicClient
@@ -14,27 +15,14 @@
 from ocp_resources.resource import ResourceEditor
 from pytest_testconfig import py_config
 
-from utilities.constants import Labels, Timeout, Annotations, Protocols
+from tests.model_explainability.lm_eval.utils import get_lmevaljob_pod
+from utilities.constants import Labels, Timeout, Annotations, Protocols, MinIo
 
 VLLM_EMULATOR: str = "vllm-emulator"
 VLLM_EMULATOR_PORT: int = 8000
 LMEVALJOB_NAME: str = "lmeval-test-job"
 
 
-@pytest.fixture(scope="function")
-def lmevaljob_hf_pod(admin_client: DynamicClient, lmevaljob_hf: LMEvalJob) -> Generator[Pod, Any, Any]:
-    lmeval_pod = Pod(
-        client=admin_client,
-        namespace=lmevaljob_hf.namespace,
-        name=lmevaljob_hf.name,
-    )
-
-    # TODO: Check if we can rely on LMEvalJob instead of pod
-    lmeval_pod.wait(timeout=Timeout.TIMEOUT_2MIN)
-
-    yield lmeval_pod
-
-
 @pytest.fixture(scope="function")
 def lmevaljob_hf(
     admin_client: DynamicClient, model_namespace: Namespace, patched_trustyai_operator_configmap_allow_online: ConfigMap
@@ -107,22 +95,6 @@ def lmevaljob_local_offline(
         yield job
 
 
-@pytest.fixture(scope="function")
-def lmevaljob_vllm_emulator_pod(
-    admin_client: DynamicClient, lmevaljob_vllm_emulator: LMEvalJob
-) -> Generator[Pod, Any, Any]:
-    lmeval_pod = Pod(
-        client=admin_client,
-        namespace=lmevaljob_vllm_emulator.namespace,
-        name=lmevaljob_vllm_emulator.name,
-    )
-
-    # TODO: Check if we can rely on LMEvalJob instead of pod
-    lmeval_pod.wait(timeout=Timeout.TIMEOUT_2MIN)
-
-    yield lmeval_pod
-
-
 @pytest.fixture(scope="function")
 def lmevaljob_vllm_emulator(
     admin_client: DynamicClient,
@@ -307,3 +279,136 @@ def vllm_emulator_route(
         service=vllm_emulator_service.name,
     ) as route:
         yield route
+
+
+@pytest.fixture(scope="function")
+def lmeval_minio_deployment(
+    admin_client: DynamicClient, minio_namespace: Namespace, pvc_minio_namespace: PersistentVolumeClaim
+) -> Generator[Deployment, Any, Any]:
+    minio_app_label = {"app": MinIo.Metadata.NAME}
+    # TODO: Unify with minio_llm_deployment fixture once datasets and models are in new model image
+    with Deployment(
+        client=admin_client,
+        name=MinIo.Metadata.NAME,
+        namespace=minio_namespace.name,
+        replicas=1,
+        selector={"matchLabels": minio_app_label},
+        template={
+            "metadata": {"labels": minio_app_label},
+            "spec": {
+                "volumes": [
+                    {"name": "minio-storage", "persistentVolumeClaim": {"claimName": pvc_minio_namespace.name}}
+                ],
+                "containers": [
+                    {
+                        "name": MinIo.Metadata.NAME,
+                        "image": "quay.io/minio/minio"
+                        "@sha256:46b3009bf7041eefbd90bd0d2b38c6ddc24d20a35d609551a1802c558c1c958f",
+                        "args": ["server", "/data", "--console-address", ":9001"],
+                        "env": [
+                            {"name": "MINIO_ROOT_USER", "value": MinIo.Credentials.ACCESS_KEY_VALUE},
+                            {"name": "MINIO_ROOT_PASSWORD", "value": MinIo.Credentials.SECRET_KEY_VALUE},
+                        ],
+                        "ports": [{"containerPort": MinIo.Metadata.DEFAULT_PORT}, {"containerPort": 9001}],
+                        "volumeMounts": [{"name": "minio-storage", "mountPath": "/data"}],
+                    }
+                ],
+            },
+        },
+        label=minio_app_label,
+        wait_for_resource=True,
+    ) as deployment:
+        deployment.wait_for_replicas(timeout=Timeout.TIMEOUT_10MIN)
+        yield deployment
+
+
+@pytest.fixture(scope="function")
+def lmeval_minio_copy_pod(
+    admin_client: DynamicClient, minio_namespace: Namespace, lmeval_minio_deployment: Deployment, minio_service: Service
+) -> Generator[Pod, Any, Any]:
+    with Pod(
+        client=admin_client,
+        name="copy-to-minio",
+        namespace=minio_namespace.name,
+        restart_policy="Never",
+        volumes=[{"name": "shared-data", "emptyDir": {}}],
+        init_containers=[
+            {
+                "name": "copy-data",
+                "image": "quay.io/trustyai_testing/lmeval-assets-flan-arceasy"
+                "@sha256:11cc9c2f38ac9cc26c4fab1a01a8c02db81c8f4801b5d2b2b90f90f91b97ac98",
+                "command": ["/bin/sh", "-c"],
+                "args": ["cp -r /mnt/data /shared"],
+                "volumeMounts": [{"name": "shared-data", "mountPath": "/shared"}],
+            }
+        ],
+        containers=[
+            {
+                "name": "minio-uploader",
+                "image": "quay.io/minio/mc@sha256:470f5546b596e16c7816b9c3fa7a78ce4076bb73c2c73f7faeec0c8043923123",
+                "command": ["/bin/sh", "-c"],
+                "args": [
+                    f"mc alias set myminio http://{minio_service.name}:{MinIo.Metadata.DEFAULT_PORT} "
+                    f"{MinIo.Credentials.ACCESS_KEY_VALUE} {MinIo.Credentials.SECRET_KEY_VALUE} &&\n"
+                    "mc mb --ignore-existing myminio/models &&\n"
+                    "mc cp --recursive /shared/data/ myminio/models"
+                ],
+                "volumeMounts": [{"name": "shared-data", "mountPath": "/shared"}],
+            }
+        ],
+        wait_for_resource=True,
+    ) as pod:
+        pod.wait_for_status(status=Pod.Status.SUCCEEDED)
+        yield pod
+
+
+@pytest.fixture(scope="function")
+def lmevaljob_s3_offline(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    lmeval_minio_deployment: Deployment,
+    minio_service: Service,
+    lmeval_minio_copy_pod: Pod,
+    minio_data_connection: Secret,
+) -> Generator[LMEvalJob, Any, Any]:
+    with LMEvalJob(
+        client=admin_client,
+        name="evaljob-sample",
+        namespace=model_namespace.name,
+        model="hf",
+        model_args=[{"name": "pretrained", "value": "/opt/app-root/src/hf_home/flan"}],
+        task_list={"taskNames": ["arc_easy"]},
+        log_samples=True,
+        allow_online=False,
+        offline={
+            "storage": {
+                "s3": {
+                    "accessKeyId": {"name": minio_data_connection.name, "key": "AWS_ACCESS_KEY_ID"},
+                    "secretAccessKey": {"name": minio_data_connection.name, "key": "AWS_SECRET_ACCESS_KEY"},
+                    "bucket": {"name": minio_data_connection.name, "key": "AWS_S3_BUCKET"},
+                    "endpoint": {"name": minio_data_connection.name, "key": "AWS_S3_ENDPOINT"},
+                    "region": {"name": minio_data_connection.name, "key": "AWS_DEFAULT_REGION"},
+                    "path": "",
+                    "verifySSL": False,
+                }
+            }
+        },
+    ) as job:
+        yield job
+
+
+@pytest.fixture(scope="function")
+def lmevaljob_hf_pod(admin_client: DynamicClient, lmevaljob_hf: LMEvalJob) -> Generator[Pod, Any, Any]:
+    yield get_lmevaljob_pod(client=admin_client, lmevaljob=lmevaljob_hf)
+
+
+@pytest.fixture(scope="function")
+def lmevaljob_vllm_emulator_pod(
+    admin_client: DynamicClient, lmevaljob_vllm_emulator: LMEvalJob
+) -> Generator[Pod, Any, Any]:
+    yield get_lmevaljob_pod(client=admin_client, lmevaljob=lmevaljob_vllm_emulator)
+
+
+@pytest.fixture(scope="function")
+def lmevaljob_s3_offline_pod(admin_client: DynamicClient, lmevaljob_s3_offline: LMEvalJob) -> Generator[Pod, Any, Any]:
+    yield get_lmevaljob_pod(client=admin_client, lmevaljob=lmevaljob_s3_offline)
diff --git a/tests/model_explainability/lm_eval/test_lm_eval.py b/tests/model_explainability/lm_eval/test_lm_eval.py
@@ -3,6 +3,8 @@
 from tests.model_explainability.lm_eval.utils import verify_lmevaljob_running
 from utilities.constants import Timeout
 
+LMEVALJOB_COMPLETE_STATE: str = "Complete"
+
 
 @pytest.mark.parametrize(
     "model_namespace",
@@ -90,3 +92,24 @@ def test_lmeval_vllm_emulator(admin_client, model_namespace, lmevaljob_vllm_emul
     lmevaljob_vllm_emulator_pod.wait_for_status(
         status=lmevaljob_vllm_emulator_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_10MIN
     )
+
+
+@pytest.mark.parametrize(
+    "model_namespace, minio_data_connection",
+    [
+        pytest.param(
+            {"name": "test-s3-lmeval"},
+            {"bucket": "models"},
+        )
+    ],
+    indirect=True,
+)
+def test_lmeval_s3_storage(
+    admin_client,
+    model_namespace,
+    lmevaljob_s3_offline_pod,
+):
+    """Test to verify that LMEval works with a model stored in a S3 bucket"""
+    lmevaljob_s3_offline_pod.wait_for_status(
+        status=lmevaljob_s3_offline_pod.Status.SUCCEEDED, timeout=Timeout.TIMEOUT_10MIN
+    )
diff --git a/tests/model_explainability/lm_eval/utils.py b/tests/model_explainability/lm_eval/utils.py
diff --git a/tests/model_registry/test_model_registry_creation.py b/tests/model_registry/test_model_registry_creation.py
