feat(multinode): add multi-node cluster support with worker agents #215
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Rust Tests | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| jobs: | |
| check: | |
| name: Cargo Check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Cache dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y protobuf-compiler | |
| - name: Install wasm-pack and Build WASM | |
| run: | | |
| cargo install wasm-pack | |
| cd crates/temps-captcha-wasm | |
| bun run build | |
| - name: Check workspace | |
| run: cargo check --workspace --all-targets | |
| fmt: | |
| name: Formatting | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt | |
| - name: Check formatting | |
| run: cargo fmt --all -- --check | |
| clippy: | |
| name: Clippy (advisory) | |
| runs-on: ubuntu-latest | |
| # Non-blocking: clippy warnings are reported but do not fail the build | |
| continue-on-error: true | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: clippy | |
| - name: Install Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Cache dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y protobuf-compiler | |
| - name: Install wasm-pack and Build WASM | |
| run: | | |
| cargo install wasm-pack | |
| cd crates/temps-captcha-wasm | |
| bun run build | |
| - name: Run clippy | |
| run: cargo clippy --workspace --all-targets --all-features -- -D warnings | |
| # --------------------------------------------------------------------------- | |
| # Build all test binaries once, then cache for parallel test jobs | |
| # --------------------------------------------------------------------------- | |
| build-tests: | |
| name: Build Test Binaries | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Free up disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force | |
| sudo apt-get autoremove -y && sudo apt-get clean | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Cache dependencies | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| # Use a shared cache key so test jobs can restore | |
| shared-key: test-build | |
| save-if: true | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y protobuf-compiler | |
| - name: Install wasm-pack and Build WASM | |
| run: | | |
| cargo install wasm-pack | |
| cd crates/temps-captcha-wasm | |
| bun run build | |
| - name: Build all test binaries | |
| run: cargo test --no-run --workspace --all-features | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| # --------------------------------------------------------------------------- | |
| # Phase 1: Unit tests (fast, no Docker containers needed) | |
| # --------------------------------------------------------------------------- | |
| # Runs all workspace tests WITHOUT the docker-tests feature. | |
| # This executes all pure unit tests across every crate quickly. | |
| # For temps-providers, Docker-heavy tests are gated behind the | |
| # `docker-tests` feature flag and will NOT run here. | |
| unit-tests: | |
| name: "Unit Tests (${{ matrix.group }})" | |
| runs-on: ubuntu-latest | |
| needs: build-tests | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - group: unit-a | |
| crates: >- | |
| -p temps-providers | |
| -p temps-presets | |
| -p temps-auth | |
| -p temps-core | |
| -p temps-monitoring | |
| -p temps-kv | |
| -p temps-analytics-funnels | |
| -p temps-webhooks | |
| -p temps-infra | |
| -p temps-captcha-wasm | |
| -p temps-geo | |
| -p temps-environments | |
| -p temps-audit | |
| -p temps-analytics-performance | |
| - group: unit-b | |
| crates: >- | |
| -p temps-proxy | |
| -p temps-error-tracking | |
| -p temps-git | |
| -p temps-screenshots | |
| -p temps-routes | |
| -p temps-entities | |
| -p temps-cli | |
| -p temps-status-page | |
| -p temps-import | |
| -p temps-queue | |
| -p temps-query | |
| -p temps-analytics-session-replay | |
| -p temps-dns | |
| -p temps-email | |
| -p temps-analytics | |
| -p temps-analytics-events | |
| -p temps-otel | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Restore build cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: test-build | |
| save-if: false | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y protobuf-compiler | |
| - name: Install wasm-pack and Build WASM | |
| run: | | |
| cargo install wasm-pack | |
| cd crates/temps-captcha-wasm | |
| bun run build | |
| - name: Run unit tests | |
| env: | |
| RUST_BACKTRACE: 1 | |
| CARGO_INCREMENTAL: 0 | |
| run: | | |
| # Run tests, capturing the names of any failures for targeted retry | |
| if cargo test --no-fail-fast --lib ${{ matrix.crates }} 2>&1 | tee /tmp/test-output.txt; then | |
| echo "All tests passed on first attempt" | |
| exit 0 | |
| fi | |
| # Extract failed test names from output | |
| failed_tests=$(grep '^test .* FAILED$' /tmp/test-output.txt | sed 's/^test \(.*\) \.\.\..*FAILED$/\1/' | tr '\n' ' ') | |
| if [ -z "$failed_tests" ]; then | |
| echo "Could not parse failed test names, re-running all tests" | |
| cargo test --no-fail-fast --lib ${{ matrix.crates }} | |
| exit $? | |
| fi | |
| echo "" | |
| echo "==========================================" | |
| echo "Retrying only failed tests: $failed_tests" | |
| echo "==========================================" | |
| echo "" | |
| sleep 5 | |
| # Retry only the failed tests (up to 2 more attempts) | |
| attempt=2 | |
| max_attempts=3 | |
| for test_name in $failed_tests; do | |
| echo "--- Retrying: $test_name (attempt $attempt/$max_attempts) ---" | |
| if ! cargo test --no-fail-fast --lib ${{ matrix.crates }} -- "$test_name" --exact; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| attempt=$((attempt + 1)) | |
| echo "--- Retrying: $test_name (attempt $attempt/$max_attempts) ---" | |
| sleep 5 | |
| cargo test --no-fail-fast --lib ${{ matrix.crates }} -- "$test_name" --exact || exit 1 | |
| else | |
| exit 1 | |
| fi | |
| fi | |
| done | |
| # --------------------------------------------------------------------------- | |
| # Phase 2: Integration tests (Docker + Database required) | |
| # --------------------------------------------------------------------------- | |
| # Runs tests that need Docker daemon and/or a real database. | |
| # - temps-providers runs with --features docker-tests to include | |
| # Docker-heavy tests (backup/restore, container lifecycle, etc.) | |
| # - Other crates that use TestDatabase (testcontainers) run here too. | |
| integration-tests: | |
| name: "Integration Tests (${{ matrix.group }})" | |
| runs-on: ubuntu-latest | |
| needs: [build-tests, unit-tests] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - group: docker-providers | |
| # temps-providers with Docker tests enabled | |
| command: >- | |
| cargo test --no-fail-fast --lib | |
| -p temps-providers --features docker-tests | |
| - group: docker-deployments | |
| # Crates that use TestDatabase and/or Docker | |
| command: >- | |
| cargo test --no-fail-fast | |
| -p temps-deployments | |
| -p temps-backup | |
| -p temps-vulnerability-scanner | |
| -p temps-import-docker | |
| -p temps-migrations | |
| -p temps-database | |
| -p temps-blob | |
| -p temps-query-s3 | |
| -p temps-query-redis | |
| -p temps-query-mongodb | |
| -p temps-query-postgres | |
| -p temps-embeddings | |
| -p temps-config | |
| -p temps-notifications | |
| -p temps-deployer | |
| -p temps-logs | |
| -p temps-domains | |
| -p temps-projects | |
| services: | |
| timescaledb: | |
| image: timescale/timescaledb-ha:pg18 | |
| env: | |
| POSTGRES_DB: test_db | |
| POSTGRES_USER: test_user | |
| POSTGRES_PASSWORD: test_password | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U test_user -d test_db" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - name: Free up disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force | |
| sudo apt-get autoremove -y && sudo apt-get clean | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| - name: Install Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version: latest | |
| - name: Restore build cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: test-build | |
| save-if: false | |
| - name: Install system dependencies | |
| run: sudo apt-get update && sudo apt-get install -y protobuf-compiler | |
| - name: Install wasm-pack and Build WASM | |
| run: | | |
| cargo install wasm-pack | |
| cd crates/temps-captcha-wasm | |
| bun run build | |
| - name: Verify TimescaleDB is ready | |
| run: | | |
| timeout 60 bash -c 'until nc -z localhost 5432; do sleep 1; done' | |
| echo "TimescaleDB is ready" | |
| - name: Run integration tests | |
| env: | |
| RUST_BACKTRACE: 1 | |
| CARGO_INCREMENTAL: 0 | |
| TEMPS_TEST_DATABASE_URL: postgresql://test_user:test_password@localhost:5432/test_db | |
| run: | | |
| # Run tests, capturing the names of any failures for targeted retry | |
| if ${{ matrix.command }} 2>&1 | tee /tmp/test-output.txt; then | |
| echo "All tests passed on first attempt" | |
| exit 0 | |
| fi | |
| # Extract failed test names from output | |
| failed_tests=$(grep '^test .* FAILED$' /tmp/test-output.txt | sed 's/^test \(.*\) \.\.\..*FAILED$/\1/' | tr '\n' ' ') | |
| if [ -z "$failed_tests" ]; then | |
| echo "Could not parse failed test names, re-running all tests" | |
| ${{ matrix.command }} | |
| exit $? | |
| fi | |
| echo "" | |
| echo "==========================================" | |
| echo "Retrying only failed tests: $failed_tests" | |
| echo "==========================================" | |
| echo "" | |
| sleep 10 | |
| # Retry only the failed tests (up to 2 more attempts) | |
| attempt=2 | |
| max_attempts=3 | |
| for test_name in $failed_tests; do | |
| echo "--- Retrying: $test_name (attempt $attempt/$max_attempts) ---" | |
| if ! ${{ matrix.command }} -- "$test_name" --exact; then | |
| if [ $attempt -lt $max_attempts ]; then | |
| attempt=$((attempt + 1)) | |
| echo "--- Retrying: $test_name (attempt $attempt/$max_attempts) ---" | |
| sleep 10 | |
| ${{ matrix.command }} -- "$test_name" --exact || exit 1 | |
| else | |
| exit 1 | |
| fi | |
| fi | |
| done |