[wip] llamastack tests refactor

evals tests

work on lamastack evals tests

improve lmeval provider tests

guardrails tests

[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

wip

guardrails

move rag tests to llama_stack/

fix guardrails tests

add more logging to guardrails tests

Author: adolfo-ab

pyproject.toml

@@ -71,6 +71,7 @@ dependencies = [
     "fire",
     "llama_stack_client==0.2.17",
     "pytest-xdist==3.8.0",
+    "curlify>=3.0.0",
 ]
 
 [project.urls]

tests/conftest.py

@@ -61,6 +61,8 @@
 
 LOGGER = get_logger(name=__name__)
 
+pytest_plugins = ["tests.fixtures.inference", "tests.fixtures.guardrails", "tests.fixtures.evals"]
+
 
 @pytest.fixture(scope="session")
 def admin_client() -> DynamicClient:

tests/fixtures/evals.py

@@ -0,0 +1,42 @@
+from typing import Generator, Any
+
+import pytest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.config_map import ConfigMap
+from ocp_resources.deployment import Deployment
+from ocp_resources.resource import ResourceEditor
+from pytest_testconfig import py_config
+
+from utilities.constants import Annotations
+
+
+@pytest.fixture(scope="function")
+def patched_trustyai_operator_configmap_allow_online(admin_client: DynamicClient) -> Generator[ConfigMap, Any, Any]:
+    namespace: str = py_config["applications_namespace"]
+    trustyai_service_operator: str = "trustyai-service-operator"
+
+    configmap: ConfigMap = ConfigMap(
+        client=admin_client, name=f"{trustyai_service_operator}-config", namespace=namespace, ensure_exists=True
+    )
+    with ResourceEditor(
+        patches={
+            configmap: {
+                "metadata": {"annotations": {Annotations.OpenDataHubIo.MANAGED: "false"}},
+                "data": {
+                    "lmes-allow-online": "true",
+                    "lmes-allow-code-execution": "true",
+                },
+            }
+        }
+    ):
+        deployment: Deployment = Deployment(
+            client=admin_client,
+            name=f"{trustyai_service_operator}-controller-manager",
+            namespace=namespace,
+            ensure_exists=True,
+        )
+        num_replicas: int = deployment.replicas
+        deployment.scale_replicas(replica_count=0)
+        deployment.scale_replicas(replica_count=num_replicas)
+        deployment.wait_for_replicas()
+        yield configmap

tests/fixtures/guardrails.py

@@ -0,0 +1,119 @@
+from typing import Generator, Any
+
+import pytest
+from _pytest.fixtures import FixtureRequest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.config_map import ConfigMap
+from ocp_resources.deployment import Deployment
+from ocp_resources.guardrails_orchestrator import GuardrailsOrchestrator
+from ocp_resources.namespace import Namespace
+from ocp_resources.pod import Pod
+from ocp_resources.route import Route
+
+from utilities.constants import Labels
+
+
+GUARDRAILS_ORCHESTRATOR_NAME: str = "guardrails-orchestrator"
+
+
+@pytest.fixture(scope="class")
+def guardrails_orchestrator(
+    request: FixtureRequest,
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    orchestrator_config: ConfigMap,
+) -> Generator[GuardrailsOrchestrator, Any, Any]:
+    gorch_kwargs = {
+        "client": admin_client,
+        "name": GUARDRAILS_ORCHESTRATOR_NAME,
+        "namespace": model_namespace.name,
+        "orchestrator_config": orchestrator_config.name,
+        "replicas": 1,
+        "wait_for_resource": True,
+    }
+
+    if enable_built_in_detectors := request.param.get("enable_built_in_detectors"):
+        gorch_kwargs["enable_built_in_detectors"] = enable_built_in_detectors
+
+    if request.param.get("enable_guardrails_gateway"):
+        guardrails_gateway_config = request.getfixturevalue(argname="guardrails_gateway_config")
+        gorch_kwargs["enable_guardrails_gateway"] = True
+        gorch_kwargs["guardrails_gateway_config"] = guardrails_gateway_config.name
+
+    with GuardrailsOrchestrator(**gorch_kwargs) as gorch:
+        gorch_deployment = Deployment(name=gorch.name, namespace=gorch.namespace, wait_for_resource=True)
+        gorch_deployment.wait_for_replicas()
+        yield gorch
+
+
+@pytest.fixture(scope="class")
+def orchestrator_config(
+    request: FixtureRequest, admin_client: DynamicClient, model_namespace: Namespace
+) -> Generator[ConfigMap, Any, Any]:
+    with ConfigMap(
+        client=admin_client,
+        name="fms-orchestr8-config-nlp",
+        namespace=model_namespace.name,
+        data=request.param["orchestrator_config_data"],
+    ) as cm:
+        yield cm
+
+
+@pytest.fixture(scope="class")
+def guardrails_gateway_config(
+    request: FixtureRequest, admin_client: DynamicClient, model_namespace: Namespace
+) -> Generator[ConfigMap, Any, Any]:
+    with ConfigMap(
+        client=admin_client,
+        name="fms-orchestr8-config-gateway",
+        namespace=model_namespace.name,
+        label={Labels.Openshift.APP: "fmstack-nlp"},
+        data=request.param["guardrails_gateway_config_data"],
+    ) as cm:
+        yield cm
+
+
+@pytest.fixture(scope="class")
+def guardrails_orchestrator_pod(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    guardrails_orchestrator: GuardrailsOrchestrator,
+) -> Pod:
+    return list(
+        Pod.get(
+            namespace=model_namespace.name, label_selector=f"app.kubernetes.io/instance={GUARDRAILS_ORCHESTRATOR_NAME}"
+        )
+    )[0]
+
+
+@pytest.fixture(scope="class")
+def guardrails_orchestrator_route(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    guardrails_orchestrator: GuardrailsOrchestrator,
+) -> Generator[Route, Any, Any]:
+    yield Route(
+        name=f"{guardrails_orchestrator.name}",
+        namespace=guardrails_orchestrator.namespace,
+        wait_for_resource=True,
+    )
+
+
+@pytest.fixture(scope="class")
+def guardrails_orchestrator_url(
+    guardrails_orchestrator_route: Route,
+) -> str:
+    return f"https://{guardrails_orchestrator_route.host}"
+
+
+@pytest.fixture(scope="class")
+def guardrails_orchestrator_health_route(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    guardrails_orchestrator: GuardrailsOrchestrator,
+) -> Generator[Route, Any, Any]:
+    yield Route(
+        name=f"{guardrails_orchestrator.name}-health",
+        namespace=guardrails_orchestrator.namespace,
+        wait_for_resource=True,
+    )

tests/fixtures/inference.py

@@ -0,0 +1,77 @@
+from typing import Generator, Any
+
+import pytest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.inference_service import InferenceService
+from ocp_resources.namespace import Namespace
+from ocp_resources.pod import Pod
+from ocp_resources.secret import Secret
+from ocp_resources.service import Service
+from ocp_resources.serving_runtime import ServingRuntime
+
+from utilities.constants import RuntimeTemplates, KServeDeploymentType
+from utilities.inference_utils import create_isvc
+from utilities.serving_runtime import ServingRuntimeFromTemplate
+
+
+@pytest.fixture(scope="class")
+def vllm_cpu_runtime(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    minio_pod: Pod,
+    minio_service: Service,
+    minio_data_connection: Secret,
+) -> Generator[ServingRuntime, Any, Any]:
+    with ServingRuntimeFromTemplate(
+        client=admin_client,
+        name="vllm-runtime-cpu-fp16",
+        namespace=model_namespace.name,
+        template_name=RuntimeTemplates.VLLM_CUDA,
+        deployment_type=KServeDeploymentType.RAW_DEPLOYMENT,
+        runtime_image="quay.io/rh-aiservices-bu/vllm-cpu-openai-ubi9"
+        "@sha256:ada6b3ba98829eb81ae4f89364d9b431c0222671eafb9a04aa16f31628536af2",
+        containers={
+            "kserve-container": {
+                "args": [
+                    f"--port={str(8032)}",
+                    "--model=/mnt/models",
+                ],
+                "ports": [{"containerPort": 8032, "protocol": "TCP"}],
+                "volumeMounts": [{"mountPath": "/dev/shm", "name": "shm"}],
+            }
+        },
+        volumes=[{"emptyDir": {"medium": "Memory", "sizeLimit": "2Gi"}, "name": "shm"}],
+    ) as serving_runtime:
+        yield serving_runtime
+
+
+@pytest.fixture(scope="class")
+def qwen_isvc(
+    admin_client: DynamicClient,
+    model_namespace: Namespace,
+    minio_pod: Pod,
+    minio_service: Service,
+    minio_data_connection: Secret,
+    vllm_cpu_runtime: ServingRuntime,
+) -> Generator[InferenceService, Any, Any]:
+    with create_isvc(
+        client=admin_client,
+        name="qwen-isvc",
+        namespace=model_namespace.name,
+        deployment_mode=KServeDeploymentType.RAW_DEPLOYMENT,
+        model_format="vLLM",
+        runtime=vllm_cpu_runtime.name,
+        storage_key=minio_data_connection.name,
+        storage_path="Qwen2.5-0.5B-Instruct",
+        wait_for_predictor_pods=False,
+        resources={
+            "requests": {"cpu": "1", "memory": "8Gi"},
+            "limits": {"cpu": "2", "memory": "10Gi"},
+        },
+    ) as isvc:
+        yield isvc
+
+
+@pytest.fixture(scope="class")
+def qwen_isvc_url(qwen_isvc: InferenceService) -> str:
+    return f"http://{qwen_isvc.name}-predictor.{qwen_isvc.namespace}.svc.cluster.local:8032/v1"