feat: add test for incorrect DB TLS config in Trusty AI

Author: sheltoncyril

tests/model_explainability/trustyai_service/conftest.py

@@ -14,7 +14,9 @@
 from ocp_resources.secret import Secret
 from ocp_resources.subscription import Subscription
 from ocp_resources.trustyai_service import TrustyAIService
+from ocp_resources.utils.constants import TIMEOUT_2MINUTES
 from ocp_utilities.operators import install_operator, uninstall_operator
+from timeout_sampler import TimeoutExpiredError
 
 from tests.model_explainability.trustyai_service.trustyai_service_utils import TRUSTYAI_SERVICE_NAME
 from tests.model_explainability.trustyai_service.utils import (
@@ -35,6 +37,11 @@
 DB_NAME: str = "trustyai_db"
 DB_USERNAME: str = "trustyai_user"
 DB_PASSWORD: str = "trustyai_password"
+FAKE_TLS_CERTIFICATE: str = (
+    "LS0tLS1CRUdJTiBDRVJUSUZJQ0FUR"
+    "S0tLS0tXG5kMzRkYjMzZlxuLS0tLS"
+    "1FTkQgQ0VSVElGSUNBVEUtLS0tLQ=="
+)  # pragma: allowlist secret
 
 
 @pytest.fixture(scope="class")
@@ -82,6 +89,29 @@ def trustyai_service_with_db_storage(
         yield trustyai_service
 
 
+@pytest.fixture(scope="class")
+def trustyai_service_with_invalid_cert_db_storage(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    cluster_monitoring_config: ConfigMap,
+    user_workload_monitoring_config: ConfigMap,
+    mariadb: MariaDB,
+    trustyai_invalid_db_ca_secret: None,
+) -> None:
+    with TrustyAIService(
+        client=admin_client,
+        name=TRUSTYAI_SERVICE_NAME,
+        namespace=model_namespace.name,
+        storage={"format": "DATABASE", "size": "1Gi", "databaseConfigurations": "db-credentials"},
+        metrics={"schedule": "5s"},
+    ):
+        trustyai_deployment = Deployment(
+            namespace=model_namespace.name, name=TRUSTYAI_SERVICE_NAME, wait_for_resource=True
+        )
+        with pytest.raises(TimeoutExpiredError):
+            trustyai_deployment.wait_for_replicas(timeout=TIMEOUT_2MINUTES)
+
+
 @pytest.fixture(scope="session")
 def user_workload_monitoring_config(admin_client: DynamicClient) -> Generator[ConfigMap, Any, Any]:
     data = {"config.yaml": yaml.dump({"prometheus": {"logLevel": "debug", "retention": "15d"}})}
@@ -238,17 +268,32 @@ def mariadb(
         yield mariadb
 
 
-@pytest.fixture(scope="class")
-def trustyai_db_ca_secret(
-    admin_client: DynamicClient, model_namespace: Namespace, mariadb: MariaDB
+def set_trustyai_db_ca_secret(
+    admin_client: DynamicClient, mariadb_ca_cert: str, model_namespace: Namespace
 ) -> Generator[None, Any, None]:
-    mariadb_ca_secret = Secret(
-        client=admin_client, name=f"{mariadb.name}-ca", namespace=model_namespace.name, ensure_exists=True
-    )
     with Secret(
         client=admin_client,
         name=f"{TRUSTYAI_SERVICE_NAME}-db-ca",
         namespace=model_namespace.name,
-        data_dict={"ca.crt": mariadb_ca_secret.instance.data["ca.crt"]},
+        data_dict={"ca.crt": mariadb_ca_cert},
     ):
         yield
+
+
+@pytest.fixture(scope="class")
+def trustyai_db_ca_secret(
+    admin_client: DynamicClient, model_namespace: Namespace, mariadb: MariaDB
+) -> Generator[None, Any, None]:
+    mariadb_ca_secret = Secret(
+        client=admin_client, name=f"{mariadb.name}-ca", namespace=model_namespace.name, ensure_exists=True
+    )
+    mariadb_ca_cert = mariadb_ca_secret.instance.data["ca.crt"]
+    yield from set_trustyai_db_ca_secret(admin_client, mariadb_ca_cert, model_namespace)
+
+
+@pytest.fixture(scope="class")
+def trustyai_invalid_db_ca_secret(
+    admin_client: DynamicClient, model_namespace: Namespace, mariadb: MariaDB
+) -> Generator[None, Any, None]:
+    mariadb_ca_cert = FAKE_TLS_CERTIFICATE
+    yield from set_trustyai_db_ca_secret(admin_client, mariadb_ca_cert, model_namespace)

tests/model_explainability/trustyai_service/fairness/test_fairness.py

@@ -156,6 +156,16 @@ def test_fairness_send_inference_and_verify_trustyai_service_with_db_storage(
             inference_config=OPENVINO_KSERVE_INFERENCE_CONFIG,
         )
 
+    def test_invalid_db_certificate_trustyai_service_with_db_storage(
+        self,
+        admin_client,
+        current_client_token,
+        model_namespace,
+        trustyai_service_with_invalid_cert_db_storage,
+        minio_data_connection,
+    ):
+        return
+
     def test_name_mappings_with_db_storage(
         self, admin_client, current_client_token, model_namespace, trustyai_service_with_db_storage, onnx_loan_model
     ):

tests/model_explainability/trustyai_service/trustyai_service_utils.py

@@ -634,3 +634,17 @@ def verify_trustyai_service_name_mappings(
     assert response_output_mappings == output_mappings, (
         f"Output mappings mismatch. Expected: {output_mappings}, Got: {response_output_mappings}"
     )
+
+
+def invalid_db_cert_trustyai_service_with_db_storage(
+    client: DynamicClient, token: str, data_path: str, trustyai_service: TrustyAIService
+) -> None:
+    """
+    Creates a MariaDB pod with TLS enabled but sets the wrong certificate as a secret to trustyai_service.
+    Args:
+        client (DynamicClient): The client instance for making API calls.
+        token (str): Authentication token for API access.
+        data_path (str): Directory path containing data batch files.
+        trustyai_service (TrustyAIService): TrustyAIService that will register the model.
+    """
+    return