Merge branch 'main' into multi_ns

Author: kpunwatk

.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
     rev: v1.5.0
     hooks:
       - id: detect-secrets
-        exclude: .*/__snapshots__/.*
+        exclude: .*/__snapshots__/.*|.*-input\.json$
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.12.3

conftest.py

@@ -5,6 +5,7 @@
 import datetime
 import traceback
 
+import pytest
 import shortuuid
 from _pytest.runner import CallInfo
 from _pytest.reports import TestReport
@@ -35,7 +36,7 @@
 from kubernetes.dynamic import DynamicClient
 from utilities.infra import get_operator_distribution, get_dsci_applications_namespace, get_data_science_cluster
 from ocp_resources.resource import get_client
-
+from ocp_resources.cluster_service_version import ClusterServiceVersion
 
 LOGGER = logging.getLogger(name=__name__)
 BASIC_LOGGER = logging.getLogger(name="basic")
@@ -48,6 +49,9 @@ def pytest_addoption(parser: Parser) -> None:
     upgrade_group = parser.getgroup(name="Upgrade options")
     must_gather_group = parser.getgroup(name="MustGather")
     cluster_sanity_group = parser.getgroup(name="ClusterSanity")
+    ociregistry_group = parser.getgroup(name="OCI Registry")
+    serving_arguments_group = parser.getgroup(name="Serving arguments")
+    model_validation_automation_group = parser.getgroup(name="Model Validation Automation")
     hf_group = parser.getgroup(name="Hugging Face")
 
     # AWS config and credentials options
@@ -118,6 +122,32 @@ def pytest_addoption(parser: Parser) -> None:
         help="Specify the runtime image to use for the tests",
     )
 
+    # OCI Registry options
+    ociregistry_group.addoption(
+        "--registry-pull-secret",
+        default=os.environ.get("OCI_REGISTRY_PULL_SECRET"),
+        help="Registry pull secret to pull oci container images",
+    )
+    ociregistry_group.addoption(
+        "--registry-host",
+        default=os.environ.get("REGISTRY_HOST"),
+        help="Registry host to pull oci container images",
+    )
+
+    # Serving arguments options
+    serving_arguments_group.addoption(
+        "--serving-argument",
+        default=os.environ.get("SERVING_ARGUMENTS"),
+        help="Serving arguments to use for the tests",
+    )
+
+    # Model Validation Automation options
+    model_validation_automation_group.addoption(
+        "--model_car_yaml_path",
+        default=os.environ.get("MODEL_YAML_PATH"),
+        help="Path to the modelcar yaml file",
+    )
+
     # Upgrade options
     upgrade_group.addoption(
         "--pre-upgrade",
@@ -412,3 +442,37 @@ def pytest_exception_interact(node: Item | Collector, call: CallInfo[Any], repor
 
         except Exception as current_exception:
             LOGGER.warning(f"Failed to collect logs: {test_name}: {current_exception} {traceback.format_exc()}")
+
+
+@pytest.fixture(scope="package")
+def fail_if_missing_dependent_operators(admin_client: DynamicClient) -> None:
+    if dependent_operators := py_config.get("dependent_operators"):
+        missing_operators: list[str] = []
+
+        for operator_name in dependent_operators.split(","):
+            csvs = list(
+                ClusterServiceVersion.get(
+                    dyn_client=admin_client,
+                    namespace=py_config["applications_namespace"],
+                )
+            )
+
+            LOGGER.info(f"Verifying if {operator_name} is installed")
+            for csv in csvs:
+                if csv.name.startswith(operator_name):
+                    if csv.status == csv.Status.SUCCEEDED:
+                        break
+
+                    else:
+                        missing_operators.append(
+                            f"Operator {operator_name} is installed but CSV is not in {csv.Status.SUCCEEDED} state"
+                        )
+
+            else:
+                missing_operators.append(f"{operator_name} is not installed")
+
+        if missing_operators:
+            pytest.fail(str(missing_operators))
+
+    else:
+        LOGGER.info("No dependent operators to verify")

tests/conftest.py

@@ -31,6 +31,7 @@
 from ocp_resources.resource import get_client
 from pytest_testconfig import config as py_config
 from simple_logger.logger import get_logger
+import json
 
 from ocp_utilities.operators import uninstall_operator, install_operator
 from utilities.certificates_utils import create_ca_bundle_file
@@ -110,7 +111,7 @@ def model_namespace(
         ns.clean_up()
     else:
         with create_ns(
-            client=admin_client,
+            admin_client=admin_client,
             pytest_request=request,
             teardown=teardown_resources,
         ) as ns:
@@ -139,6 +140,28 @@ def aws_secret_access_key(pytestconfig: Config) -> str:
     return secret_access_key
 
 
+@pytest.fixture(scope="session")
+def registry_pull_secret(pytestconfig: Config) -> str:
+    registry_pull_secret = pytestconfig.option.registry_pull_secret
+    if not registry_pull_secret:
+        raise ValueError(
+            "Registry pull secret is not set. "
+            "Either pass with `--registry_pull_secret` or set `OCI_REGISTRY_PULL_SECRET` environment variable"
+        )
+    return registry_pull_secret
+
+
+@pytest.fixture(scope="session")
+def registry_host(pytestconfig: pytest.Config) -> str | None:
+    registry_host = pytestconfig.option.registry_host
+    if not registry_host:
+        raise ValueError(
+            "Registry host for OCI images is not set. "
+            "Either pass with `--registry_host` or set `REGISTRY_HOST` environment variable"
+        )
+    return registry_host
+
+
 @pytest.fixture(scope="session")
 def valid_aws_config(aws_access_key_id: str, aws_secret_access_key: str) -> tuple[str, str]:
     return aws_access_key_id, aws_secret_access_key
@@ -177,6 +200,40 @@ def ci_s3_bucket_endpoint(pytestconfig: pytest.Config) -> str:
     return ci_bucket_endpoint
 
 
+@pytest.fixture(scope="session")
+def serving_argument(pytestconfig: pytest.Config, modelcar_yaml_config: dict[str, Any] | None) -> list[str]:
+    if modelcar_yaml_config:
+        arg = modelcar_yaml_config.get("serving_argument", [])
+        return arg if isinstance(arg, list) else [arg]
+
+    raw_arg = pytestconfig.option.serving_argument
+    try:
+        return json.loads(raw_arg)
+    except json.JSONDecodeError:
+        raise ValueError(
+            "Serving arguments should be a valid JSON list. "
+            "Either pass with `--serving-argument` or set it correctly in modelcar.yaml"
+        )
+
+
+@pytest.fixture(scope="session")
+def modelcar_yaml_config(pytestconfig: pytest.Config) -> dict[str, Any] | None:
+    """
+    Fixture to get the path to the modelcar.yaml file.
+    """
+    config_path = pytestconfig.option.model_car_yaml_path
+    if not config_path:
+        return None
+    with open(config_path, "r") as file:
+        try:
+            modelcar_yaml = yaml.safe_load(file)
+            if not isinstance(modelcar_yaml, dict):
+                raise ValueError("modelcar.yaml should contain a dictionary.")
+            return modelcar_yaml
+        except yaml.YAMLError as e:
+            raise ValueError(f"Error parsing modelcar.yaml: {e}") from e
+
+
 @pytest.fixture(scope="session")
 def models_s3_bucket_name(pytestconfig: pytest.Config) -> str:
     models_bucket = pytestconfig.option.models_s3_bucket_name
@@ -407,12 +464,12 @@ def cluster_monitoring_config(
 
 @pytest.fixture(scope="class")
 def unprivileged_model_namespace(
-    request: FixtureRequest, unprivileged_client: DynamicClient
+    request: FixtureRequest, admin_client: DynamicClient, unprivileged_client: DynamicClient
 ) -> Generator[Namespace, Any, Any]:
     if request.param.get("modelmesh-enabled"):
         request.getfixturevalue(argname="enabled_modelmesh_in_dsc")
 
-    with create_ns(unprivileged_client=unprivileged_client, pytest_request=request) as ns:
+    with create_ns(admin_client=admin_client, unprivileged_client=unprivileged_client, pytest_request=request) as ns:
         yield ns
 
 
@@ -421,7 +478,7 @@ def unprivileged_model_namespace(
 def minio_namespace(admin_client: DynamicClient) -> Generator[Namespace, Any, Any]:
     with create_ns(
         name=f"{MinIo.Metadata.NAME}-{shortuuid.uuid().lower()}",
-        client=admin_client,
+        admin_client=admin_client,
     ) as ns:
         yield ns
 

tests/model_explainability/trustyai_service/service/conftest.py

@@ -30,6 +30,7 @@
 RmcvTXlNWW9CZUNrUVRWdS9rUkIwK2N2Qy9RMDB4NExvVGpJaQpGdCtKMGc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0t\
 LS0t"  # pragma: allowlist secret
 
+
 @pytest.fixture(scope="class")
 def trustyai_service_with_invalid_db_cert(
     admin_client: DynamicClient,