Add upgrade tests for guardrails (#1088)

* feat: add test to migrate trustyai db

* feat: add guardrails upgrade tests

* fix: type annotation

* docs: add docstrings

Author: sheltoncyril

tests/fixtures/guardrails.py

@@ -22,75 +22,105 @@ def guardrails_orchestrator(
     request: FixtureRequest,
     admin_client: DynamicClient,
     model_namespace: Namespace,
+    teardown_resources: bool,
+    pytestconfig: pytest.Config,
 ) -> Generator[GuardrailsOrchestrator, Any, Any]:
     gorch_kwargs = {
         "client": admin_client,
         "name": GUARDRAILS_ORCHESTRATOR_NAME,
         "namespace": model_namespace.name,
-        "log_level": "DEBUG",
-        "replicas": 1,
-        "wait_for_resource": True,
     }
-
-    if request.param.get("auto_config"):
-        gorch_kwargs["auto_config"] = request.param.get("auto_config")
-
-    if request.param.get("orchestrator_config"):
-        orchestrator_config = request.getfixturevalue(argname="orchestrator_config")
-        gorch_kwargs["orchestrator_config"] = orchestrator_config.name
-
-    if request.param.get("enable_guardrails_gateway"):
-        gorch_kwargs["enable_guardrails_gateway"] = True
-
-    if request.param.get("guardrails_gateway_config"):
-        guardrails_gateway_config = request.getfixturevalue(argname="guardrails_gateway_config")
-        gorch_kwargs["guardrails_gateway_config"] = guardrails_gateway_config.name
-
-    if enable_built_in_detectors := request.param.get("enable_built_in_detectors"):
-        gorch_kwargs["enable_built_in_detectors"] = enable_built_in_detectors
-
-    if request.param.get("otel_exporter_config"):
-        metrics_endpoint = request.getfixturevalue(argname="otelcol_metrics_endpoint")
-        traces_endpoint = request.getfixturevalue(argname="tempo_traces_endpoint")
-        gorch_kwargs["otel_exporter"] = {
-            "otlpProtocol": "grpc",
-            "otlpMetricsEndpoint": metrics_endpoint,
-            "otlpTracesEndpoint": traces_endpoint,
-            "enableMetrics": True,
-            "enableTracing": True,
-        }
-
-    with GuardrailsOrchestrator(**gorch_kwargs) as gorch:
-        gorch_deployment = Deployment(name=gorch.name, namespace=gorch.namespace, wait_for_resource=True)
-        gorch_deployment.wait_for_replicas()
+    if pytestconfig.option.post_upgrade:
+        gorch = GuardrailsOrchestrator(**gorch_kwargs, ensure_exists=True)
         yield gorch
+        gorch.clean_up()
+    else:
+        gorch_kwargs["log_level"] = "DEBUG"
+        gorch_kwargs["replicas"] = 1
+        gorch_kwargs["wait_for_resource"] = True
+        if request.param.get("auto_config"):
+            gorch_kwargs["auto_config"] = request.param.get("auto_config")
+
+        if request.param.get("orchestrator_config"):
+            orchestrator_config = request.getfixturevalue(argname="orchestrator_config")
+            gorch_kwargs["orchestrator_config"] = orchestrator_config.name
+
+        if request.param.get("enable_guardrails_gateway"):
+            gorch_kwargs["enable_guardrails_gateway"] = True
+
+        if request.param.get("guardrails_gateway_config"):
+            guardrails_gateway_config = request.getfixturevalue(argname="guardrails_gateway_config")
+            gorch_kwargs["guardrails_gateway_config"] = guardrails_gateway_config.name
+
+        if enable_built_in_detectors := request.param.get("enable_built_in_detectors"):
+            gorch_kwargs["enable_built_in_detectors"] = enable_built_in_detectors
+
+        if request.param.get("otel_exporter_config"):
+            metrics_endpoint = request.getfixturevalue(argname="otelcol_metrics_endpoint")
+            traces_endpoint = request.getfixturevalue(argname="tempo_traces_endpoint")
+            gorch_kwargs["otel_exporter"] = {
+                "otlpProtocol": "grpc",
+                "otlpMetricsEndpoint": metrics_endpoint,
+                "otlpTracesEndpoint": traces_endpoint,
+                "enableMetrics": True,
+                "enableTracing": True,
+            }
+
+        with GuardrailsOrchestrator(**gorch_kwargs, teardown=teardown_resources) as gorch:
+            gorch_deployment = Deployment(name=gorch.name, namespace=gorch.namespace, wait_for_resource=True)
+            gorch_deployment.wait_for_replicas()
+            yield gorch
 
 
 @pytest.fixture(scope="class")
 def orchestrator_config(
-    request: FixtureRequest, admin_client: DynamicClient, model_namespace: Namespace
+    request: FixtureRequest,
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    teardown_resources: bool,
+    pytestconfig: pytest.Config,
 ) -> Generator[ConfigMap, Any, Any]:
-    with ConfigMap(
-        client=admin_client,
-        name="fms-orchestr8-config-nlp",
-        namespace=model_namespace.name,
-        data=request.param["orchestrator_config_data"],
-    ) as cm:
+    if pytestconfig.option.post_upgrade:
+        cm = ConfigMap(
+            client=admin_client, name="fms-orchestr8-config-nlp", namespace=model_namespace.name, ensure_exists=True
+        )
         yield cm
+        cm.clean_up()
+    else:
+        with ConfigMap(
+            client=admin_client,
+            name="fms-orchestr8-config-nlp",
+            namespace=model_namespace.name,
+            data=request.param["orchestrator_config_data"],
+            teardown=teardown_resources,
+        ) as cm:
+            yield cm
 
 
 @pytest.fixture(scope="class")
 def guardrails_gateway_config(
-    request: FixtureRequest, admin_client: DynamicClient, model_namespace: Namespace
+    request: FixtureRequest,
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    teardown_resources: bool,
+    pytestconfig: pytest.Config,
 ) -> Generator[ConfigMap, Any, Any]:
-    with ConfigMap(
-        client=admin_client,
-        name="fms-orchestr8-config-gateway",
-        namespace=model_namespace.name,
-        label={Labels.Openshift.APP: "fmstack-nlp"},
-        data=request.param["guardrails_gateway_config_data"],
-    ) as cm:
+    if pytestconfig.option.post_upgrade:
+        cm = ConfigMap(
+            client=admin_client, name="fms-orchestr8-config-gateway", namespace=model_namespace.name, ensure_exists=True
+        )
         yield cm
+        cm.clean_up()
+    else:
+        with ConfigMap(
+            client=admin_client,
+            name="fms-orchestr8-config-gateway",
+            namespace=model_namespace.name,
+            label={Labels.Openshift.APP: "fmstack-nlp"},
+            data=request.param["guardrails_gateway_config_data"],
+            teardown=teardown_resources,
+        ) as cm:
+            yield cm
 
 
 @pytest.fixture(scope="class")
@@ -142,23 +172,13 @@ def guardrails_orchestrator_health_route(
     admin_client: DynamicClient,
     model_namespace: Namespace,
     guardrails_orchestrator: GuardrailsOrchestrator,
-) -> Generator[Route, Any, Any]:
-    guardrails_orchestrator_health_route = Route(
+) -> Route:
+    return Route(
         name=f"{guardrails_orchestrator.name}-health",
         namespace=guardrails_orchestrator.namespace,
         wait_for_resource=True,
         ensure_exists=True,
     )
-    with ResourceEditor(
-        patches={
-            guardrails_orchestrator_health_route: {
-                "metadata": {
-                    "annotations": {Annotations.HaproxyRouterOpenshiftIo.TIMEOUT: "10m"},
-                }
-            }
-        }
-    ):
-        yield guardrails_orchestrator_health_route
 
 
 @pytest.fixture
@@ -177,20 +197,10 @@ def guardrails_orchestrator_gateway_route(
     admin_client: DynamicClient,
     model_namespace: Namespace,
     guardrails_orchestrator: GuardrailsOrchestrator,
-) -> Generator[Route, Any, Any]:
-    guardrails_orchestrator_gateway_route = Route(
+) -> Route:
+    return Route(
         name=f"{guardrails_orchestrator.name}-gateway",
         namespace=guardrails_orchestrator.namespace,
         wait_for_resource=True,
         ensure_exists=True,
     )
-    with ResourceEditor(
-        patches={
-            guardrails_orchestrator_gateway_route: {
-                "metadata": {
-                    "annotations": {Annotations.HaproxyRouterOpenshiftIo.TIMEOUT: "10m"},
-                }
-            }
-        }
-    ):
-        yield guardrails_orchestrator_gateway_route