test: [3.3] New tests to validate postgres password autogeneration (#1239)

* test: [3.3]  New tests to validate postgres password autogeneration

Signed-off-by: Debarati Basu-Nag <dbasunag@redhat.com>

* fix: update calls

Signed-off-by: Debarati Basu-Nag <dbasunag@redhat.com>

---------

Signed-off-by: Debarati Basu-Nag <dbasunag@redhat.com>

Author: dbasunag

tests/model_registry/model_catalog/catalog_config/test_default_source_inclusion_exclusion_cleanup.py

@@ -66,6 +66,7 @@ def test_include_models_by_pattern(
 
         # Ensure baseline model state is restored for subsequent tests
         ensure_baseline_model_state(
+            admin_client=admin_client,
             model_catalog_rest_url=model_catalog_rest_url,
             model_registry_rest_headers=model_registry_rest_headers,
             model_registry_namespace=model_registry_namespace,
@@ -110,6 +111,7 @@ def test_exclude_models_by_pattern(
 
         # Ensure baseline model state is restored for subsequent tests
         ensure_baseline_model_state(
+            admin_client=admin_client,
             model_catalog_rest_url=model_catalog_rest_url,
             model_registry_rest_headers=model_registry_rest_headers,
             model_registry_namespace=model_registry_namespace,
@@ -190,7 +192,7 @@ def test_combined_include_exclude_filtering(
                 )
 
             db_models = get_models_from_database_by_source(
-                source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
+                admin_client=admin_client, source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
             )
 
             is_valid, error_msg = validate_model_filtering_consistency(api_models=api_models, db_models=db_models)
@@ -206,6 +208,7 @@ def test_combined_include_exclude_filtering(
 
         # Ensure baseline model state is restored for subsequent tests
         ensure_baseline_model_state(
+            admin_client=admin_client,
             model_catalog_rest_url=model_catalog_rest_url,
             model_registry_rest_headers=model_registry_rest_headers,
             model_registry_namespace=model_registry_namespace,
@@ -255,7 +258,7 @@ def test_model_cleanup_on_exclusion_change(
                 pytest.fail(f"Phase 1: Timeout waiting for granite models {granite_models}: {e}")
 
             phase1_db_models = get_models_from_database_by_source(
-                source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
+                admin_client=admin_client, source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
             )
 
             assert phase1_api_models == granite_models, (
@@ -292,7 +295,7 @@ def test_model_cleanup_on_exclusion_change(
                 pytest.fail(f"Phase 2: Timeout waiting for prometheus models {prometheus_models}: {e}")
 
             phase2_db_models = get_models_from_database_by_source(
-                source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
+                admin_client=admin_client, source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
             )
 
             # Should only have prometheus models now
@@ -307,6 +310,7 @@ def test_model_cleanup_on_exclusion_change(
 
         # Ensure baseline model state is restored for subsequent tests
         ensure_baseline_model_state(
+            admin_client=admin_client,
             model_catalog_rest_url=model_catalog_rest_url,
             model_registry_rest_headers=model_registry_rest_headers,
             model_registry_namespace=model_registry_namespace,
@@ -351,14 +355,15 @@ def test_source_disabling_removes_models(
 
             # Verify database is also cleaned
             db_models = get_models_from_database_by_source(
-                source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
+                admin_client=admin_client, source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
             )
             assert len(db_models) == 0, f"Database should be clean when source disabled, found: {db_models}"
 
             LOGGER.info("SUCCESS: Source disabling removed all models")
 
         # Ensure baseline model state is restored for subsequent tests
         ensure_baseline_model_state(
+            admin_client=admin_client,
             model_catalog_rest_url=model_catalog_rest_url,
             model_registry_rest_headers=model_registry_rest_headers,
             model_registry_namespace=model_registry_namespace,
@@ -416,6 +421,7 @@ def test_model_removal_logging(
 
         # Ensure baseline model state is restored for subsequent tests
         ensure_baseline_model_state(
+            admin_client=admin_client,
             model_catalog_rest_url=model_catalog_rest_url,
             model_registry_rest_headers=model_registry_rest_headers,
             model_registry_namespace=model_registry_namespace,
@@ -467,6 +473,7 @@ def test_source_disabling_logging(
 
         # Ensure baseline model state is restored for subsequent tests
         ensure_baseline_model_state(
+            admin_client=admin_client,
             model_catalog_rest_url=model_catalog_rest_url,
             model_registry_rest_headers=model_registry_rest_headers,
             model_registry_namespace=model_registry_namespace,

tests/model_registry/model_catalog/catalog_config/utils.py

@@ -132,11 +132,12 @@ def validate_model_catalog_configmap_data(configmap: ConfigMap, num_catalogs: in
         validate_default_catalog(catalogs=catalogs)
 
 
-def get_models_from_database_by_source(source_id: str, namespace: str) -> set[str]:
+def get_models_from_database_by_source(admin_client: DynamicClient, source_id: str, namespace: str) -> set[str]:
     """
     Query database directly to get all model names for a specific source.
 
     Args:
+        admin_client: DynamicClient for Kubernetes API access
         source_id: Catalog source ID to filter by
         namespace: OpenShift namespace for database access
 
@@ -145,7 +146,7 @@ def get_models_from_database_by_source(source_id: str, namespace: str) -> set[st
     """
 
     query = GET_MODELS_BY_SOURCE_ID_DB_QUERY.format(source_id=source_id)
-    result = execute_database_query(query=query, namespace=namespace)
+    result = execute_database_query(admin_client=admin_client, query=query, namespace=namespace)
     parsed = parse_psql_output(psql_output=result)
     return set(parsed.get("values", []))
 
@@ -478,7 +479,7 @@ def execute_inclusion_exclusion_filter_test(
             pytest.fail(f"Timeout waiting for {pattern} models to appear. Expected: {expected_models}, {e}")
 
         db_models = get_models_from_database_by_source(
-            source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
+            admin_client=admin_client, source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
         )
 
         # Validate consistency
@@ -493,6 +494,7 @@ def execute_inclusion_exclusion_filter_test(
 
 @retry(wait_timeout=300, sleep=10, exceptions_dict={Exception: []}, print_log=False)
 def _validate_baseline_models(
+    admin_client: DynamicClient,
     model_catalog_rest_url: list[str],
     model_registry_rest_headers: dict[str, str],
     model_registry_namespace: str,
@@ -513,7 +515,9 @@ def _validate_baseline_models(
     api_models = {model["name"] for model in api_response.get("items", [])}
 
     # Fetch current models from database
-    db_models = get_models_from_database_by_source(source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace)
+    db_models = get_models_from_database_by_source(
+        admin_client=admin_client, source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
+    )
 
     count = len(api_models)
 
@@ -542,6 +546,7 @@ def _validate_baseline_models(
 
 
 def ensure_baseline_model_state(
+    admin_client: DynamicClient,
     model_catalog_rest_url: list[str],
     model_registry_rest_headers: dict[str, str],
     model_registry_namespace: str,
@@ -574,6 +579,7 @@ def ensure_baseline_model_state(
     # Use retry decorator for automatic polling and eventual reconciliation
     try:
         _validate_baseline_models(
+            admin_client=admin_client,
             model_catalog_rest_url=model_catalog_rest_url,
             model_registry_rest_headers=model_registry_rest_headers,
             model_registry_namespace=model_registry_namespace,

tests/model_registry/model_catalog/conftest.py

@@ -447,7 +447,10 @@ def model_catalog_rest_url(model_registry_namespace: str, model_catalog_routes:
 
 @pytest.fixture(scope="function")
 def baseline_redhat_ai_models(
-    model_catalog_rest_url: list[str], model_registry_rest_headers: dict[str, str], model_registry_namespace: str
+    admin_client: DynamicClient,
+    model_catalog_rest_url: list[str],
+    model_registry_rest_headers: dict[str, str],
+    model_registry_namespace: str,
 ) -> dict[str, set[str] | int]:
     """
     fixture providing baseline model data for redhat_ai_models source.
@@ -463,6 +466,8 @@ def baseline_redhat_ai_models(
     )
     api_models = {model["name"] for model in api_response.get("items", [])}
 
-    db_models = get_models_from_database_by_source(source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace)
+    db_models = get_models_from_database_by_source(
+        admin_client=admin_client, source_id=REDHAT_AI_CATALOG_ID, namespace=model_registry_namespace
+    )
 
     return {"api_models": api_models, "db_models": db_models, "count": len(api_models)}

tests/model_registry/model_catalog/db_check/__init__.py

@@ -0,0 +1,70 @@
+import pytest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.network_policy import NetworkPolicy
+from ocp_resources.secret import Secret
+from pytest_testconfig import config as py_config
+from simple_logger.logger import get_logger
+from timeout_sampler import TimeoutSampler
+
+from .utils import extract_secret_values
+
+LOGGER = get_logger(name=__name__)
+
+
+@pytest.fixture(scope="class")
+def model_catalog_postgres_secret(admin_client: DynamicClient, model_registry_namespace: str) -> Secret:
+    """Get the model-catalog-postgres secret from model registry namespace"""
+    return Secret(
+        client=admin_client,
+        name="model-catalog-postgres",
+        namespace=model_registry_namespace,
+        ensure_exists=True,
+    )
+
+
+@pytest.fixture(scope="class")
+def model_catalog_postgres_secret_values(model_catalog_postgres_secret: Secret) -> dict[str, str]:
+    """Capture current values of model-catalog-postgres secret in model registry namespace"""
+    return extract_secret_values(secret=model_catalog_postgres_secret)
+
+
+@pytest.fixture(scope="class")
+def recreated_model_catalog_postgres_secret(
+    admin_client: DynamicClient, model_catalog_postgres_secret: Secret
+) -> dict[str, str]:
+    """Delete model-catalog-postgres secret and wait for it to be recreated"""
+    model_registry_namespace = py_config["model_registry_namespace"]
+    resource_name = "model-catalog-postgres"
+
+    LOGGER.info(f"Deleting secret {resource_name} in namespace {model_registry_namespace}")
+    model_catalog_postgres_secret.delete()
+
+    # Wait for the secret to be recreated by the operator
+    LOGGER.info(f"Waiting for secret {resource_name} to be recreated...")
+
+    recreated_secret = None
+    for secret in TimeoutSampler(
+        wait_timeout=120,
+        sleep=10,
+        func=Secret,
+        client=admin_client,
+        name=resource_name,
+        namespace=model_registry_namespace,
+    ):
+        if secret.exists:
+            LOGGER.info(f"Secret {resource_name} has been recreated")
+            recreated_secret = secret
+            break
+
+    return extract_secret_values(secret=recreated_secret)
+
+
+@pytest.fixture(scope="class")
+def model_catalog_postgres_network_policy(admin_client: DynamicClient, model_registry_namespace: str) -> NetworkPolicy:
+    """Get the model-catalog-postgres NetworkPolicy from model registry namespace"""
+    return NetworkPolicy(
+        client=admin_client,
+        name="model-catalog-postgres",
+        namespace=model_registry_namespace,
+        ensure_exists=True,
+    )