Adding milvus testing

.github/workflows/milvus-integration.yml

@@ -0,0 +1,181 @@
+name: Milvus Integration Tests
+
+on:
+  push:
+    branches: [ main, develop ]
+    paths:
+      - 'server/storage/milvus/**'
+      - 'docker-compose.milvus.yml'
+      - '.github/workflows/milvus-integration.yml'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - 'server/storage/milvus/**'
+      - 'docker-compose.milvus.yml'
+      - '.github/workflows/milvus-integration.yml'
+  workflow_dispatch:
+    inputs:
+      run_performance_tests:
+        description: 'Run performance tests'
+        required: false
+        default: 'false'
+        type: boolean
+
+env:
+  MILVUS_INTEGRATION_TESTS: true
+  MILVUS_TEST_HOST: localhost
+  MILVUS_TEST_PORT: 19530
+  MILVUS_TEST_USER: ""
+  MILVUS_TEST_PASSWORD: ""
+
+jobs:
+  milvus-integration:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [3.11, 3.12]
+
+    services:
+      # We'll use docker-compose instead of services for complex setup
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install system dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libavdevice-dev ffmpeg libsox-fmt-all sox
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.8.0
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v3
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+    - name: Install dependencies
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: |
+        poetry install --with test
+        poetry run pip install pymilvus
+
+    - name: Start Milvus services
+      run: |
+        docker-compose -f docker-compose.milvus.yml up -d
+
+        # Wait for services to be ready
+        echo "Waiting for services to start..."
+        sleep 30
+
+        # Wait for etcd
+        timeout 120 bash -c 'until docker exec milvus-etcd etcdctl endpoint health; do sleep 5; done'
+
+        # Wait for minio
+        timeout 120 bash -c 'until curl -f http://localhost:9000/minio/health/live; do sleep 5; done'
+
+        # Wait for milvus
+        timeout 180 bash -c 'until curl -f http://localhost:9091/healthz; do sleep 10; done'
+
+        echo "All services are ready!"
+
+    - name: Verify Milvus connection
+      run: |
+        poetry run python -c "
+        from pymilvus import connections, utility
+        try:
+            connections.connect(host='localhost', port='19530')
+            version = utility.get_server_version()
+            print(f'Successfully connected to Milvus version: {version}')
+            connections.disconnect('default')
+        except Exception as e:
+            print(f'Failed to connect to Milvus: {e}')
+            exit(1)
+        "
+
+    - name: Run Milvus integration tests
+      run: |
+        poetry run pytest server/storage/milvus/test_milvus_integration.py -v -m "not performance" --tb=short
+
+    - name: Run performance tests
+      if: ${{ github.event.inputs.run_performance_tests == 'true' || github.event_name == 'workflow_dispatch' }}
+      run: |
+        poetry run pytest server/storage/milvus/test_milvus_integration.py -v -m "performance" --tb=short
+
+    - name: Show Milvus logs on failure
+      if: failure()
+      run: |
+        echo "=== Milvus logs ==="
+        docker logs milvus-standalone
+        echo "=== etcd logs ==="
+        docker logs milvus-etcd
+        echo "=== MinIO logs ==="
+        docker logs milvus-minio
+
+    - name: Cleanup
+      if: always()
+      run: |
+        docker-compose -f docker-compose.milvus.yml down -v
+        docker system prune -f
+
+  milvus-compatibility:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+
+    strategy:
+      matrix:
+        milvus-version: ["v2.4.15", "v2.4.0", "v2.3.0"]
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.12
+
+    - name: Install Poetry and dependencies
+      run: |
+        pip install poetry==1.8.0
+        poetry install --with test
+        poetry run pip install pymilvus
+
+    - name: Test with Milvus ${{ matrix.milvus-version }}
+      run: |
+        # Modify docker-compose to use specific Milvus version
+        sed -i 's/milvusdb\/milvus:v[0-9.]\+/milvusdb\/milvus:${{ matrix.milvus-version }}/g' docker-compose.milvus.yml
+
+        # Start services
+        docker-compose -f docker-compose.milvus.yml up -d
+
+        # Wait for services
+        sleep 60
+        timeout 180 bash -c 'until curl -f http://localhost:9091/healthz; do sleep 10; done'
+
+        # Set test environment
+        export MILVUS_INTEGRATION_TESTS=true
+        export MILVUS_TEST_HOST=localhost
+        export MILVUS_TEST_PORT=19530
+
+        # Run basic compatibility tests
+        poetry run pytest server/storage/milvus/test_milvus_integration.py::TestMilvusIntegration::test_connection_establishment -v
+        poetry run pytest server/storage/milvus/test_milvus_integration.py::TestMilvusIntegration::test_save_and_retrieve_vcon -v
+
+    - name: Cleanup
+      if: always()
+      run: |
+        docker-compose -f docker-compose.milvus.yml down -v

.github/workflows/milvus.yml

@@ -0,0 +1,102 @@
+name: Milvus Integration Tests
+
+on:
+  push:
+    branches: [main, develop]
+    paths:
+      - 'server/storage/milvus/**'
+      - 'docker-compose.milvus.yml'
+      - 'scripts/test-milvus*'
+  pull_request:
+    branches: [main]
+    paths:
+      - 'server/storage/milvus/**'
+  schedule:
+    - cron: '0 2 * * *'  # Daily at 2 AM
+  workflow_dispatch:
+    inputs:
+      include_performance:
+        description: 'Run performance tests'
+        required: false
+        default: 'false'
+        type: boolean
+
+jobs:
+  milvus-integration:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.12
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.8.0
+
+    - name: Install dependencies
+      run: |
+        poetry install --with test
+        poetry add pymilvus
+
+    - name: Free up disk space
+      run: |
+        docker system prune -af
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+
+    - name: Run Milvus integration tests
+      run: |
+        chmod +x scripts/test-milvus.sh
+        ./scripts/test-milvus.sh
+
+    - name: Run performance tests
+      if: ${{ github.event.inputs.include_performance == 'true' }}
+      run: |
+        chmod +x scripts/test-milvus-comprehensive.sh
+        ./scripts/test-milvus-comprehensive.sh --performance
+
+    - name: Upload test results
+      if: always()
+      uses: actions/upload-artifact@v3
+      with:
+        name: milvus-test-results
+        path: |
+          pytest-results.xml
+          milvus-logs/
+
+  compatibility-matrix:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    strategy:
+      matrix:
+        milvus-version: ["v2.4.15", "v2.4.0", "v2.3.0"]
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.12
+
+    - name: Install dependencies
+      run: |
+        pip install poetry
+        poetry install --with test
+
+    - name: Update Milvus version
+      run: |
+        sed -i 's/milvusdb\/milvus:v[0-9.]\+/milvusdb\/milvus:${{ matrix.milvus-version }}/g' docker-compose.milvus.yml
+
+    - name: Test with Milvus ${{ matrix.milvus-version }}
+      run: |
+        chmod +x scripts/test-milvus.sh
+        ./scripts/test-milvus.sh

Makefile

@@ -0,0 +1,67 @@
+# Makefile for vcon-server
+
+.PHONY: help test milvus-test milvus-test-enhanced milvus-test-perf milvus-up milvus-down milvus-status
+
+# Default target
+help:
+	@echo "Available commands:"
+	@echo "  test                 - Run all tests"
+	@echo "  milvus-test         - Run basic Milvus integration tests"
+	@echo "  milvus-test-enhanced - Run enhanced Milvus tests"
+	@echo "  milvus-test-perf    - Run Milvus performance tests"
+	@echo "  milvus-up           - Start Milvus services"
+	@echo "  milvus-down         - Stop Milvus services"
+	@echo "  milvus-status       - Check Milvus service status"
+
+# Basic test target
+test:
+	@echo "🧪 Running all tests..."
+	@poetry run pytest
+
+# Basic Milvus integration test
+milvus-test:
+	@echo "🧪 Running basic Milvus integration tests..."
+	@./scripts/test-milvus.sh
+
+# Enhanced Milvus tests with vCon functionality
+milvus-test-enhanced:
+	@echo "🚀 Running enhanced Milvus integration tests..."
+	@export MILVUS_INTEGRATION_TESTS=true && \
+	docker-compose -f docker-compose.milvus.yml up -d && \
+	sleep 60 && \
+	poetry run pytest server/storage/milvus/test_milvus_enhanced.py -v -m "not performance" && \
+	docker-compose -f docker-compose.milvus.yml down -v
+
+# Performance tests
+milvus-test-perf:
+	@echo "⚡ Running Milvus performance tests..."
+	@export MILVUS_INTEGRATION_TESTS=true && \
+	docker-compose -f docker-compose.milvus.yml up -d && \
+	sleep 60 && \
+	poetry run pytest server/storage/milvus/test_milvus_enhanced.py -v -m "performance" && \
+	docker-compose -f docker-compose.milvus.yml down -v
+
+# Start Milvus services only
+milvus-up:
+	@echo "🐳 Starting Milvus services..."
+	@docker-compose -f docker-compose.milvus.yml up -d
+	@echo "⏳ Waiting for services to be ready..."
+	@sleep 60
+	@echo "✅ Milvus services ready at:"
+	@echo "   - Milvus API: http://localhost:19530"
+	@echo "   - Health check: http://localhost:9091/healthz"
+	@echo "   - MinIO console: http://localhost:9001"
+
+# Stop Milvus services
+milvus-down:
+	@echo "🛑 Stopping Milvus services..."
+	@docker-compose -f docker-compose.milvus.yml down -v
+
+# Check Milvus service status
+milvus-status:
+	@echo "📊 Milvus service status:"
+	@docker ps --filter "name=milvus" --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" || echo "No Milvus containers running"
+	@echo ""
+	@echo "🔍 Health checks:"
+	@curl -s http://localhost:9091/healthz >/dev/null && echo "✅ Milvus: Healthy" || echo "❌ Milvus: Not available"
+	@curl -s http://localhost:9000/minio/health/live >/dev/null && echo "✅ MinIO: Healthy" || echo "❌ MinIO: Not available"

conftest.py

@@ -1,8 +1,36 @@
 # conftest.py
 import pytest
+from unittest.mock import patch, MagicMock
 from dotenv import load_dotenv
 
-
 @pytest.fixture(scope="session", autouse=True)
 def load_env(pytestconfig):
     load_dotenv(".env.test")
+
+@pytest.fixture(autouse=True)
+def mock_redis_json_improved():
+    """Improved Redis JSON mock that handles the JSON.SET command"""
+    with patch('redis.Redis') as redis_mock:
+        json_mock = MagicMock()
+        redis_mock.return_value.json = MagicMock(return_value=json_mock)
+
+        # Mock the set and get methods
+        json_mock.set = MagicMock(return_value=True)
+        json_mock.get = MagicMock(return_value={'data': 'mocked_data'})
+
+        # Also mock the async version
+        with patch('redis.asyncio.Redis') as async_redis_mock:
+            async_json_mock = MagicMock()
+            async_redis_mock.return_value.json = MagicMock(return_value=async_json_mock)
+
+            # Async methods
+            async def mock_set(*args, **kwargs):
+                return True
+
+            async def mock_get(*args, **kwargs):
+                return {'data': 'mocked_data'}
+
+            async_json_mock.set = mock_set
+            async_json_mock.get = mock_get
+
+            yield