ci.yml

# =============================================================================
# CI — Lint, Format, Test on every push and PR
# =============================================================================
name: CI

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

env:
  CARGO_TERM_COLOR: always
  RUSTFLAGS: "-D warnings"

jobs:
  fmt:
    name: Format
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
        with:
          components: rustfmt
      - run: cargo fmt --all --check

  # Validate every YAML file under benchmarks/attacks/ against the e2e
  # scenario JSON Schema (Loop E2E-L2 of the framework tracked in #91).
  # Fast, no Rust toolchain, no caching needed.
  e2e-validate-scenarios:
    name: E2E Scenario Schema
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6
        with:
          python-version: "3.12"
      - name: Install Python deps
        run: python3 -m pip install -r requirements-e2e.txt
      - name: Validate scenarios
        run: python3 scripts/e2e/validate_scenarios.py --verbose

  security-audit:
    name: Security Audit
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
      
      # Audit Node.js dependencies
      - name: Set up Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e  # v6
        with:
          node-version: 24
      - name: NPM Audit (Dashboard)
        run: cd dashboard && npm audit
      - name: NPM Audit (Node bindings)
        run: cd crates/llmtrace-nodejs && npm audit

      # Audit Rust dependencies
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
      - name: Install cargo-audit
        uses: taiki-e/install-action@cca35edeb1d01366c2843b68fc3ca441446d73d3  # v2
        with:
          tool: cargo-audit
      - name: Cargo Audit
        run: cargo audit

  clippy:
    name: Clippy
    runs-on: ubuntu-latest
    needs: fmt
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
        with:
          components: clippy
      - name: Install protoc
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-clippy-
      - run: cargo clippy --workspace -- -D warnings

  test:
    name: Test
    runs-on: ubuntu-latest
    needs: fmt
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
      - name: Install protoc
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-test-
      - run: cargo test --workspace

  # ---------------------------------------------------------------------------
  # Coverage — line coverage report with cargo-llvm-cov
  # ---------------------------------------------------------------------------
  coverage:
    name: Coverage (llvm-cov)
    runs-on: ubuntu-latest
    needs: test
    env:
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
        with:
          components: llvm-tools-preview
      - name: Install protoc
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@cca35edeb1d01366c2843b68fc3ca441446d73d3  # v2
        with:
          tool: cargo-llvm-cov
      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-coverage-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-coverage-
      - name: Generate LCOV report
        run: cargo llvm-cov --workspace --lcov --output-path lcov.info
      - name: Generate HTML report
        run: cargo llvm-cov --workspace --html
      - name: Write Coverage Summary
        if: always()
        run: |
          python3 - <<'PY'
          import os
          import re
          from pathlib import Path

          summary_path = Path(os.environ["GITHUB_STEP_SUMMARY"])
          html_index = Path("target/llvm-cov/html/index.html")

          def write(msg: str) -> None:
              # Append (don't clobber other summaries in the same job).
              with summary_path.open("a", encoding="utf-8") as f:
                  f.write(msg)

          def fmt_row(label: str, pct: str, frac: str) -> str:
              return f"| {label} | {pct} | {frac} |"

          if not html_index.exists():
              write("## Coverage (llvm-cov)\n\nNo HTML report found.\n")
              raise SystemExit(0)

          try:
              html = html_index.read_text(encoding="utf-8", errors="ignore")
              m = re.search(
                  r"<tr class='light-row-bold'>.*?<pre>Totals</pre>.*?</tr>",
                  html,
                  flags=re.S,
              )
              if not m:
                  write("## Coverage (llvm-cov)\n\nFailed to locate totals row in HTML report.\n")
                  raise SystemExit(0)

              row = m.group(0)
              cells = re.findall(r"<pre>\\s*([^<]+?)\\s*</pre>", row)
              # Typically: ["Totals", func, line, region, branch]. Some llvm-cov HTML
              # builds omit branch totals, so handle 3-metric totals rows too.
              metrics = (cells[1:] + ["?"] * 4)[:4]
              func, line, region, branch = metrics

              created = ""
              m2 = re.search(r"Created:\\s*([^<]+)</h4>", html)
              if m2:
                  created = m2.group(1).strip()
          except SystemExit:
              raise
          except Exception as e:
              # Never fail the coverage job because the summary parser broke.
              write(f"## Coverage (llvm-cov)\n\nFailed to parse coverage HTML summary: `{e}`\n")
              write(
                  "\nArtifacts: `coverage-report` contains `lcov.info` and `target/llvm-cov/html/index.html`.\n"
              )
              raise SystemExit(0)

          def split_metric(s: str):
              s = s.strip()
              m = re.match(r"([0-9]+\\.[0-9]+%)\\s*\\(([^)]+)\\)", s)
              if not m:
                  return s, ""
              return m.group(1), m.group(2)
          
          func_pct, func_frac = split_metric(func)
          line_pct, line_frac = split_metric(line)
          region_pct, region_frac = split_metric(region)
          branch_pct, branch_frac = split_metric(branch)
          
          parts = []
          parts.append("## Coverage (llvm-cov)\n")
          if created:
            parts.append(f"_Report created: {created}_\n\n")
          parts.append("| Metric | % | Covered/Total |\n")
          parts.append("|---|---:|---:|\n")
          parts.append(fmt_row("Function", func_pct, func_frac) + "\n")
          parts.append(fmt_row("Line", line_pct, line_frac) + "\n")
          parts.append(fmt_row("Region", region_pct, region_frac) + "\n")
          parts.append(fmt_row("Branch", branch_pct, branch_frac if branch_frac else "") + "\n")
          parts.append(
              "\nArtifacts: `coverage-report` contains `lcov.info` and `target/llvm-cov/html/index.html`.\n"
          )

          write("".join(parts))
          PY
      - name: Upload coverage to Codecov
        # If CODECOV_TOKEN isn't configured, keep CI green and rely on artifacts.
        if: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && env.CODECOV_TOKEN != '' }}
        uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe  # v5
        with:
          files: lcov.info
          token: ${{ env.CODECOV_TOKEN }}
          fail_ci_if_error: true
          verbose: true
      - name: Skip Codecov upload (no token)
        if: ${{ env.CODECOV_TOKEN == '' }}
        run: echo "Skipping Codecov upload because secrets.CODECOV_TOKEN is not set. Coverage artifacts will still be uploaded."
      - name: Upload coverage artifacts
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7
        with:
          name: coverage-report
          path: |
            lcov.info
            target/llvm-cov/html/

  # ---------------------------------------------------------------------------
  # Integration tests — run ignored tests against real Docker Compose services
  # ---------------------------------------------------------------------------
  integration:
    name: Integration Tests
    runs-on: ubuntu-latest
    needs: test
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
      - name: Install protoc
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-integration-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-integration-

      - name: Start infrastructure services
        run: docker compose up -d clickhouse postgres redis
        env:
          CLICKHOUSE_PORT: "8123"
          CLICKHOUSE_NATIVE_PORT: "9000"
          CLICKHOUSE_DATABASE: llmtrace
          POSTGRES_USER: llmtrace
          POSTGRES_PASSWORD: llmtrace
          POSTGRES_DB: llmtrace
          REDIS_PORT: "6379"

      - name: Wait for services to be ready
        run: |
          echo "Waiting for ClickHouse..."
          for i in $(seq 1 60); do
            if curl -sf http://localhost:8123/ping >/dev/null 2>&1; then
              echo "ClickHouse ready after ${i}s"
              break
            fi
            if [ "$i" -eq 60 ]; then
              echo "ClickHouse failed after 60 attempts"
              docker compose logs clickhouse
              exit 1
            fi
            sleep 2
          done
          echo "Waiting for PostgreSQL..."
          for i in $(seq 1 30); do
            if docker compose exec -T postgres pg_isready -U llmtrace >/dev/null 2>&1; then
              echo "PostgreSQL ready after ${i}s"
              break
            fi
            if [ "$i" -eq 30 ]; then
              echo "PostgreSQL failed after 30 attempts"
              docker compose logs postgres
              exit 1
            fi
            sleep 2
          done
          echo "Waiting for Redis..."
          for i in $(seq 1 30); do
            if docker compose exec -T redis redis-cli ping >/dev/null 2>&1; then
              echo "Redis ready after ${i}s"
              break
            fi
            if [ "$i" -eq 30 ]; then
              echo "Redis failed after 30 attempts"
              docker compose logs redis
              exit 1
            fi
            sleep 2
          done
          echo "All services ready."

      - name: Run integration tests
        env:
          LLMTRACE_CLICKHOUSE_URL: http://localhost:8123
          LLMTRACE_CLICKHOUSE_DATABASE: llmtrace
          LLMTRACE_POSTGRES_URL: postgres://llmtrace:llmtrace@localhost:5432/llmtrace
          LLMTRACE_REDIS_URL: redis://127.0.0.1:6379
        run: cargo test --workspace --features "clickhouse,postgres,redis_backend" -- --ignored

      - name: Stop services
        if: always()
        run: docker compose down -v

  e2e:
    name: E2E Tests
    runs-on: ubuntu-latest
    needs: integration
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6

      - name: Set up Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e  # v6
        with:
          node-version: 24
          cache: 'npm'
          cache-dependency-path: dashboard/package-lock.json

      - name: Install Playwright
        run: |
          cd dashboard
          npm ci
          npx playwright install --with-deps

      - name: Build and Start Stack
        env:
          NEXT_PUBLIC_API_URL: http://127.0.0.1:8080
        run: |
          # `compose.yaml` binds `./config.yaml` into the proxy container. Ensure the
          # file exists in CI to avoid Docker interpreting it as a directory.
          cat > config.yaml <<'YAML'
          listen_addr: "0.0.0.0:8080"
          upstream_url: "http://example.com"
          timeout_ms: 30000
          connection_timeout_ms: 5000
          max_connections: 1000
          enable_tls: false
          enable_security_analysis: true
          enable_trace_storage: true
          enable_streaming: true
          max_request_size_bytes: 52428800
          security_analysis_timeout_ms: 5000
          trace_storage_timeout_ms: 10000

          rate_limiting:
            enabled: true
            requests_per_second: 100
            burst_size: 200
            window_seconds: 60

          circuit_breaker:
            enabled: true
            failure_threshold: 10
            recovery_timeout_ms: 30000
            half_open_max_calls: 3

          health_check:
            enabled: true
            path: "/health"
            interval_seconds: 10
            timeout_ms: 5000
            retries: 3

          storage:
            profile: "production"
            auto_migrate: true
            clickhouse_url: "http://clickhouse:8123"
            clickhouse_database: "llmtrace"
            postgres_url: "postgres://llmtrace:llmtrace@postgres:5432/llmtrace"
            redis_url: "redis://redis:6379"

          logging:
            level: "info"
            format: "text"
          YAML

          # Use local build for proxy and dashboard
          docker compose build
          docker compose up -d

      - name: Wait for services
        run: |
          echo "Waiting for Proxy..."
          proxy_ready=0
          for i in $(seq 1 60); do
            if curl -sf http://localhost:8080/health >/dev/null 2>&1; then
              echo "Proxy ready after ${i}s"
              proxy_ready=1
              break
            fi
            sleep 2
          done
          if [ "$proxy_ready" -ne 1 ]; then
            echo "Proxy did not become ready in time"
            docker compose ps
            docker compose logs llmtrace-proxy || true
            exit 1
          fi
          echo "Waiting for Dashboard..."
          dashboard_ready=0
          for i in $(seq 1 30); do
            if curl -sf http://localhost:3000 >/dev/null 2>&1; then
              echo "Dashboard ready after ${i}s"
              dashboard_ready=1
              break
            fi
            sleep 2
          done
          if [ "$dashboard_ready" -ne 1 ]; then
            echo "Dashboard did not become ready in time"
            docker compose ps
            docker compose logs dashboard || true
            exit 1
          fi
          echo "Stack is up."

      - name: Run E2E Tests
        env:
          LLMTRACE_PROXY_URL: http://127.0.0.1:8080
          PLAYWRIGHT_BASE_URL: http://127.0.0.1:3000
        run: |
          cd dashboard
          # Ensure `playwright-results.json` is valid JSON (don't mix reporter output).
          npx playwright test --reporter=json > ../playwright-results.json

      - name: Write E2E Summary
        if: always()
        run: |
          python3 - <<'PY'
          import json
          import os
          from pathlib import Path

          summary_path = Path(os.environ["GITHUB_STEP_SUMMARY"])
          report = Path("playwright-results.json")

          def write(msg: str) -> None:
              with summary_path.open("a", encoding="utf-8") as f:
                  f.write(msg)

          if not report.exists():
              write("## E2E (Playwright)\n\nNo `playwright-results.json` found.\n")
              raise SystemExit(0)

          try:
              data = json.loads(report.read_text(encoding="utf-8", errors="ignore") or "{}")
          except Exception as e:
              # Never fail the E2E job due to summary parsing.
              write("## E2E (Playwright)\n\n")
              write(f"Failed to parse `playwright-results.json`: `{e}`\n")
              write("\nArtifacts: `playwright-report` contains raw Playwright outputs.\n")
              raise SystemExit(0)
          
          def walk(obj):
            if isinstance(obj, dict):
              yield obj
              for v in obj.values():
                yield from walk(v)
            elif isinstance(obj, list):
              for it in obj:
                yield from walk(it)
          
          # Count test outcomes by scanning all dicts with a `status` key.
          counts = {"passed": 0, "failed": 0, "skipped": 0, "timedOut": 0, "interrupted": 0, "flaky": 0}
          for o in walk(data):
            status = o.get("status")
            if status in counts:
              counts[status] += 1
          
          total = sum(counts.values())
          duration_ms = data.get("stats", {}).get("duration")
          duration = f"{duration_ms/1000:.1f}s" if isinstance(duration_ms, (int, float)) else ""
          
          lines = []
          lines.append("## E2E (Playwright)\\n\\n")
          if duration:
            lines.append(f"_Duration: {duration}_\\n\\n")
          lines.append("| Outcome | Count |\\n")
          lines.append("|---|---:|\\n")
          for k in ["passed", "failed", "skipped", "flaky", "timedOut", "interrupted"]:
            lines.append(f"| {k} | {counts[k]} |\\n")
          lines.append(f"| total | {total} |\\n")
          lines.append("\\nArtifacts: `playwright-report`.\\n")
          
          with summary_path.open("a", encoding="utf-8") as f:
            f.write("".join(lines))
          PY

      - name: Upload Playwright Report
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7
        with:
          name: playwright-report
          path: |
            playwright-results.json
            dashboard/test-results/
            dashboard/playwright-report/
          retention-days: 30

      - name: Stop stack
        if: always()
        run: docker compose down -v

  benchmarks:
    name: Benchmarks
    runs-on: ubuntu-latest
    needs: test
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
      - name: Install protoc
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-bench-
      - name: Run benchmark regression gate (regex only)
        env:
          RUST_LOG: warn
        run: cargo run --bin benchmarks -- --output-dir benchmarks/results --analyzer regex
      - name: Upload benchmark results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7
        if: always()
        with:
          name: benchmark-results
          path: benchmarks/results/

  build:
    name: Build
    runs-on: ubuntu-latest
    needs: [clippy, test, coverage]
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6
      - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8  # stable
      - name: Install protoc
        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
      - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-build-
      - run: cargo build --workspace --release

  # ---------------------------------------------------------------------------
  # Container scan (advisory) — scan Docker image on PRs, don't fail
  # ---------------------------------------------------------------------------
  trivy-scan:
    name: Trivy Container Scan
    runs-on: ubuntu-latest
    needs: build
    permissions:
      security-events: write
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f  # v3

      - name: Build Docker image for scanning
        uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8  # v6
        with:
          context: .
          push: false
          load: true
          tags: llmtrace-proxy:scan
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1  # 0.35.0
        with:
          image-ref: llmtrace-proxy:scan
          format: sarif
          output: trivy-results.sarif
          severity: CRITICAL,HIGH
          exit-code: "0"

      - name: Upload Trivy SARIF to GitHub Security
        uses: github/codeql-action/upload-sarif@68bde559dea0fdcac2102bfdf6230c5f70eb485e  # v4
        if: always()
        with:
          sarif_file: trivy-results.sarif