test: add s3 storage provider test fixtures

Author: Bobbins228

tests/conftest.py

@@ -71,6 +71,7 @@
     "tests.fixtures.guardrails",
     "tests.fixtures.trustyai",
     "tests.fixtures.vector_io",
+    "tests.fixtures.files",
 ]
 
 

tests/fixtures/files.py

@@ -0,0 +1,76 @@
+from typing import Callable, Dict
+import pytest
+import os
+from _pytest.fixtures import FixtureRequest
+
+S3_AUTO_CREATE_BUCKET = os.getenv("LLS_FILES_S3_AUTO_CREATE_BUCKET", "true")
+
+
+@pytest.fixture(scope="class")
+def files_provider_config_factory(
+    request: FixtureRequest,
+) -> Callable[[str], list[Dict[str, str]]]:
+    """
+    Factory fixture for configuring external files providers and returning their configuration.
+
+    This fixture returns a factory function that can configure additional files storage providers
+    (such as S3/minio) and return the necessary environment variables
+    for configuring the LlamaStack server to use these providers.
+
+    Args:
+        request: Pytest fixture request object for accessing other fixtures
+
+    Returns:
+        Callable[[str], list[Dict[str, str]]]: Factory function that takes a provider name
+        and returns a list of environment variable dictionaries
+
+    Supported Providers:
+        - "local": defaults to using just local filesystem
+        - "s3": a remote S3/Minio storage provider
+
+    Environment Variables by Provider:
+        - "s3":
+          * ENABLE_S3: Enables S3/Minio storage provider
+          * CI_S3_BUCKET_NAME: Name of the S3/Minio bucket
+          * CI_S3_BUCKET_REGION: Region of the S3/Minio bucket
+          * CI_S3_BUCKET_ENDPOINT: Endpoint URL of the S3/Minio bucket
+          * AWS_ACCESS_KEY_ID: Access key ID for the S3/Minio bucket
+          * AWS_SECRET_ACCESS_KEY: Secret access key for the S3/Minio bucket
+          * S3_AUTO_CREATE_BUCKET: Whether to automatically create the S3/Minio bucket if it doesn't exist
+
+    Example:
+        def test_with_s3(files_provider_config_factory):
+            env_vars = files_provider_config_factory("s3")
+            # env_vars contains S3_BUCKET_NAME, S3_BUCKET_ENDPOINT_URL, etc.
+    """
+
+    def _factory(provider_name: str) -> list[Dict[str, str]]:
+        env_vars: list[dict[str, str]] = []
+
+        if provider_name == "local" or provider_name is None:
+            # Default case - no additional environment variables needed
+            pass
+        elif provider_name == "s3":
+            env_vars.append({"name": "ENABLE_S3", "value": "s3"})
+            env_vars.append({"name": "S3_BUCKET_NAME", "value": request.getfixturevalue(argname="ci_s3_bucket_name")})
+            env_vars.append({
+                "name": "AWS_DEFAULT_REGION",
+                "value": request.getfixturevalue(argname="ci_s3_bucket_region"),
+            })
+            env_vars.append({
+                "name": "S3_ENDPOINT_URL",
+                "value": request.getfixturevalue(argname="ci_s3_bucket_endpoint"),
+            })
+            env_vars.append({
+                "name": "AWS_ACCESS_KEY_ID",
+                "value": request.getfixturevalue(argname="aws_access_key_id"),
+            })
+            env_vars.append({
+                "name": "AWS_SECRET_ACCESS_KEY",
+                "value": request.getfixturevalue(argname="aws_secret_access_key"),
+            })
+            env_vars.append({"name": "S3_AUTO_CREATE_BUCKET", "value": S3_AUTO_CREATE_BUCKET})
+
+        return env_vars
+
+    return _factory

tests/llama_stack/conftest.py

@@ -50,6 +50,7 @@ def enabled_llama_stack_operator(dsc_resource: DataScienceCluster) -> Generator[
 def llama_stack_server_config(
     request: FixtureRequest,
     vector_io_provider_deployment_config_factory: Callable[[str], list[Dict[str, str]]],
+    files_provider_config_factory: Callable[[str], list[Dict[str, str]]],
 ) -> Dict[str, Any]:
     """
     Generate server configuration for LlamaStack distribution deployment and deploy vector I/O provider resources.
@@ -189,6 +190,11 @@ def test_with_remote_milvus(llama_stack_server_config):
         # KUBEFLOW_PIPELINES_TOKEN: Get from current client token
         env_vars.append({"name": "KUBEFLOW_PIPELINES_TOKEN", "value": str(current_client_token)})
 
+    # Depending on parameter files_provider, configure files provider and obtain required env_vars
+    files_provider = params.get("files_provider") or "local"
+    env_vars_files = files_provider_config_factory(provider_name=files_provider)
+    env_vars.extend(env_vars_files)
+
     # Depending on parameter vector_io_provider, deploy vector_io provider and obtain required env_vars
     vector_io_provider = params.get("vector_io_provider") or "milvus"
     env_vars_vector_io = vector_io_provider_deployment_config_factory(provider_name=vector_io_provider)

tests/llama_stack/vector_io/test_vector_stores.py

@@ -20,6 +20,7 @@
             {
                 "llama_stack_storage_size": "2Gi",
                 "vector_io_provider": "milvus",
+                "files_provider": "s3",
             },
             {"vector_io_provider": "milvus"},
             id="vector_io_provider_milvus",
@@ -29,6 +30,7 @@
             {
                 "llama_stack_storage_size": "2Gi",
                 "vector_io_provider": "faiss",
+                "files_provider": "local",
             },
             {"vector_io_provider": "faiss"},
             id="vector_io_provider_faiss",
@@ -38,6 +40,7 @@
             {
                 "llama_stack_storage_size": "2Gi",
                 "vector_io_provider": "milvus-remote",
+                "files_provider": "s3",
             },
             {"vector_io_provider": "milvus-remote"},
             id="vector_io_provider_milvus-remote",