test: add e2e tests

Author: pmannion2

Dockerfile.e2e

@@ -0,0 +1,28 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && \
+  apt-get install -y curl && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
+
+# Install dependencies first (for better caching)
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Install testing dependencies
+RUN pip install --no-cache-dir pytest pytest-cov requests selenium webdriver-manager pytest-selenium
+
+# Copy application code
+COPY . .
+
+# Make Python scripts executable
+RUN chmod +x tests/e2e/*.py
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+
+# Default command runs pytest
+CMD ["pytest", "-xvs", "tests/e2e/"] 

README.md

@@ -145,27 +145,35 @@ cursor.registerCommandEnhancer(async (command, context) => {
 
 ## 🧪 Testing
 
-This project includes comprehensive tests to ensure reliability:
-
-### Running Tests
+### Running Unit Tests
 
+Run the included unit and integration tests:
 ```bash
-# Run all tests with coverage report
-pytest
+./run_tests.sh
+```
 
-# Run just unit tests
-pytest tests/unit
+### End-to-End Testing with Docker
 
-# Run integration tests
-pytest tests/integration
+The project includes end-to-end tests that validate the full application stack using Docker:
 
-# Run with specific marker
-pytest -m unit
-```
+1. Ensure Docker and docker-compose are installed on your system.
 
-### Test Data
+2. Run the E2E test suite:
+   ```bash
+   ./run_e2e_tests.sh
+   ```
+
+This will:
+- Build all necessary Docker containers
+- Start the application stack with test data
+- Run E2E tests against the live services
+- Clean up containers when done
 
-Sample test data is located in the `tests/data` directory.
+For CI/CD environments, add the following to your workflow:
+```yaml
+- name: Run E2E Tests
+  run: ./run_e2e_tests.sh
+```
 
 ## 📊 Performance Monitoring
 

api.py

@@ -51,6 +51,7 @@
 # Check if running in Docker with external Chroma
 CHROMA_HOST = os.getenv("CHROMA_HOST")
 CHROMA_PORT = os.getenv("CHROMA_PORT")
+USE_PERSISTENT_CHROMA = os.getenv("USE_PERSISTENT_CHROMA", "false").lower() == "true"
 
 # Initialize FastAPI app
 app = FastAPI(
@@ -110,7 +111,7 @@ async def global_exception_handler(request: Request, exc: Exception):
 
 # Initialize Chroma client - either local or remote
 try:
-    if CHROMA_HOST and CHROMA_PORT:
+    if CHROMA_HOST and CHROMA_PORT and not USE_PERSISTENT_CHROMA:
         logger.info(f"Connecting to Chroma server at {CHROMA_HOST}:{CHROMA_PORT}")
         chroma_client = chromadb.HttpClient(host=CHROMA_HOST, port=int(CHROMA_PORT))
     else:
@@ -128,7 +129,7 @@ async def global_exception_handler(request: Request, exc: Exception):
     logger.info("Creating a new collection. Please run indexer.py to populate it.")
     try:
         # Try to create the collection if it doesn't exist
-        if CHROMA_HOST and CHROMA_PORT:
+        if CHROMA_HOST and CHROMA_PORT and not USE_PERSISTENT_CHROMA:
             chroma_client = chromadb.HttpClient(host=CHROMA_HOST, port=int(CHROMA_PORT))
         else:
             # Updated client initialization for newer ChromaDB versions

docker-compose.e2e.yml

@@ -0,0 +1,73 @@
+services:
+  # Chroma Vector Database
+  chroma:
+    image: chromadb/chroma:latest
+    volumes:
+      - chroma_data:/chroma/chroma
+    environment:
+      - ALLOW_RESET=true
+      - ANONYMIZED_TELEMETRY=false
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/api/v1/heartbeat"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+    ports:
+      - "8001:8000" # Map to different port to avoid conflict with API
+
+  # API Service
+  api:
+    build: .
+    volumes:
+      - ./:/app # Mount code for development
+      - chroma_index:/app/chroma_index # Vector DB persistence
+    environment:
+      - OPENAI_API_KEY=${OPENAI_API_KEY:-mock_key}
+      - CHROMA_HOST=chroma
+      - CHROMA_PORT=8000
+      - MOCK_EMBEDDINGS=true
+      - USE_PERSISTENT_CHROMA=true # Use persistent local storage instead of HTTP
+    ports:
+      - "8000:8000"
+    depends_on:
+      chroma:
+        condition: service_healthy
+    command: uvicorn api:app --host 0.0.0.0 --port 8000
+
+  # Indexer - set up initial index
+  indexer:
+    build: .
+    volumes:
+      - ./:/app # Mount code for development
+      - chroma_index:/app/chroma_index # Vector DB persistence
+      - ./tests/data:/ignition_project # Use test data for indexing
+    environment:
+      - OPENAI_API_KEY=${OPENAI_API_KEY:-mock_key}
+      - CHROMA_HOST=chroma
+      - CHROMA_PORT=8000
+      - MOCK_EMBEDDINGS=true
+      - USE_PERSISTENT_CHROMA=true # Use persistent local storage instead of HTTP
+    depends_on:
+      chroma:
+        condition: service_healthy
+    command: python indexer.py /ignition_project
+
+  # E2E Test Runner
+  e2e:
+    build:
+      context: .
+      dockerfile: Dockerfile.e2e
+    volumes:
+      - ./:/app # Mount code for development
+    environment:
+      - API_HOST=api
+      - API_PORT=8000
+      - MOCK_EMBEDDINGS=true
+    depends_on:
+      - api
+      - indexer
+    command: pytest -xvs tests/e2e/
+
+volumes:
+  chroma_data: # Persistent volume for Chroma DB
+  chroma_index: # Persistent volume for our index

indexer.py

@@ -51,13 +51,14 @@
 # Check if running in Docker with external Chroma
 CHROMA_HOST = os.getenv("CHROMA_HOST")
 CHROMA_PORT = os.getenv("CHROMA_PORT")
+USE_PERSISTENT_CHROMA = os.getenv("USE_PERSISTENT_CHROMA", "false").lower() == "true"
 
 app = typer.Typer()
 
 
 def setup_chroma_client():
     """Set up and return a Chroma client with persistence."""
-    if CHROMA_HOST and CHROMA_PORT:
+    if CHROMA_HOST and CHROMA_PORT and not USE_PERSISTENT_CHROMA:
         print(f"Connecting to Chroma server at {CHROMA_HOST}:{CHROMA_PORT}")
         return chromadb.HttpClient(host=CHROMA_HOST, port=int(CHROMA_PORT))
     else:

last_index_time.pkl

@@ -0,0 +1,57 @@
+#!/bin/bash
+set -e
+
+# Color output
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+echo -e "${BLUE}=== Ignition RAG Agent E2E Test Suite ===${NC}"
+
+# Check if Docker is available
+if ! command -v docker &> /dev/null || ! command -v docker-compose &> /dev/null; then
+    echo -e "${RED}Error: Docker and docker-compose are required for E2E tests${NC}"
+    exit 1
+fi
+
+# Clean up function
+function cleanup {
+    echo -e "${BLUE}Cleaning up containers...${NC}"
+    docker-compose -f docker-compose.e2e.yml down -v
+}
+
+# Trap to ensure cleanup on exit
+trap cleanup EXIT
+
+# Build and start containers
+echo -e "${BLUE}Building Docker containers for E2E tests...${NC}"
+docker-compose -f docker-compose.e2e.yml build
+
+# First start the chroma and API services
+echo -e "${BLUE}Starting Chroma and API services...${NC}"
+docker-compose -f docker-compose.e2e.yml up -d chroma api
+
+# Wait a moment for the service to initialize
+echo -e "${BLUE}Waiting for services to initialize...${NC}"
+sleep 5
+
+# Run indexer
+echo -e "${BLUE}Running indexer...${NC}"
+docker-compose -f docker-compose.e2e.yml up --exit-code-from indexer indexer
+
+# Run the E2E tests
+echo -e "${BLUE}Running E2E tests...${NC}"
+docker-compose -f docker-compose.e2e.yml up --exit-code-from e2e e2e
+
+# Store the exit code
+EXIT_CODE=$?
+
+if [ "$EXIT_CODE" == "0" ]; then
+    echo -e "${GREEN}E2E tests passed successfully!${NC}"
+    exit 0
+else
+    echo -e "${RED}E2E tests failed with exit code $EXIT_CODE${NC}"
+    exit 1
+fi 

run_e2e_tests.sh

@@ -0,0 +1,48 @@
+import os
+import pytest
+import requests
+import time
+import logging
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+)
+logger = logging.getLogger("e2e-tests")
+
+# Constants
+API_HOST = os.environ.get("API_HOST", "localhost")
+API_PORT = os.environ.get("API_PORT", "8000")
+API_URL = f"http://{API_HOST}:{API_PORT}"
+MAX_RETRIES = 30
+RETRY_INTERVAL = 5
+
+
+@pytest.fixture(scope="session", autouse=True)
+def wait_for_api():
+    """Wait for the API to be available before running tests."""
+    logger.info(f"Waiting for API at {API_URL} to be ready")
+
+    for attempt in range(MAX_RETRIES):
+        try:
+            response = requests.get(f"{API_URL}/health")
+            if response.status_code == 200:
+                logger.info(f"API is ready after {attempt + 1} attempts")
+                # Give the API a moment to fully initialize
+                time.sleep(2)
+                return
+        except requests.RequestException as e:
+            logger.info(f"API connection error: {e}")
+
+        logger.info(
+            f"API not ready yet, retrying in {RETRY_INTERVAL} seconds... (attempt {attempt + 1}/{MAX_RETRIES})"
+        )
+        time.sleep(RETRY_INTERVAL)
+
+    pytest.fail("API service did not become available in time")
+
+
+@pytest.fixture(scope="session")
+def api_url():
+    """Provide API URL to test functions."""
+    return API_URL

tests/e2e/conftest.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+import os
+import requests
+import pytest
+import sys
+import json
+
+# Add parent directory to path
+sys.path.append(
+    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+)
+
+
+class TestE2E:
+    """End-to-end tests for the Ignition RAG system."""
+
+    def test_api_health(self, api_url):
+        """Test API health endpoint."""
+        response = requests.get(f"{api_url}/health")
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "healthy"
+
+    def test_query_endpoint(self, api_url):
+        """Test the query endpoint with a basic question."""
+        query_data = {
+            "query": "Tell me about the tank system",
+            "top_k": 3,
+            "filter_type": None,
+            "filter_path": None,
+        }
+
+        response = requests.post(f"{api_url}/query", json=query_data)
+
+        assert response.status_code == 200
+        data = response.json()
+        assert "results" in data
+        assert "total" in data
+        assert "mock_used" in data
+        assert len(data["results"]) > 0
+
+        # Check first result has the expected structure
+        first_result = data["results"][0]
+        assert "content" in first_result
+        assert "metadata" in first_result
+        assert "similarity" in first_result
+
+    def test_multi_turn_conversation(self, api_url):
+        """Test a multi-turn conversation."""
+        # First query
+        query1_data = {
+            "query": "What is in the tank view?",
+            "top_k": 3,
+            "filter_type": "perspective",
+            "filter_path": None,
+        }
+
+        response1 = requests.post(f"{api_url}/query", json=query1_data)
+
+        assert response1.status_code == 200
+        data1 = response1.json()
+        assert "results" in data1
+        assert "total" in data1
+
+        # Follow-up query
+        query2_data = {
+            "query": "Tell me more about its components",
+            "top_k": 3,
+            "filter_type": None,
+            "filter_path": None,
+        }
+
+        response2 = requests.post(f"{api_url}/query", json=query2_data)
+
+        assert response2.status_code == 200
+        data2 = response2.json()
+        assert "results" in data2
+        assert "total" in data2