ops(e2e): enable IS-060 datamarking in shadow mode for nightly valida… #489

Workflow file for this run

	# =============================================================================
	# CI — Lint, Format, Test on every push and PR
	# =============================================================================
	name: CI

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]

	env:
	CARGO_TERM_COLOR: always
	RUSTFLAGS: "-D warnings"

	jobs:
	fmt:
	name: Format
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
	- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
	with:
	components: rustfmt
	- run: cargo fmt --all --check

	# Validate every YAML file under benchmarks/attacks/ against the e2e
	# scenario JSON Schema (Loop E2E-L2 of the framework tracked in #91).
	# Fast, no Rust toolchain, no caching needed.
	e2e-validate-scenarios:
	name: E2E Scenario Schema
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
	- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
	with:
	python-version: "3.12"
	- name: Install Python deps
	run: python3 -m pip install -r requirements-e2e.txt
	- name: Validate scenarios
	run: python3 scripts/e2e/validate_scenarios.py --verbose

	security-audit:
	name: Security Audit
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

	# Audit Node.js dependencies
	- name: Set up Node.js
	uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
	with:
	node-version: 24
	- name: NPM Audit (Dashboard)
	run: cd dashboard && npm audit
	- name: NPM Audit (Node bindings)
	run: cd crates/llmtrace-nodejs && npm audit

	# Audit Rust dependencies
	- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
	- name: Install cargo-audit
	uses: taiki-e/install-action@cca35edeb1d01366c2843b68fc3ca441446d73d3 # v2
	with:
	tool: cargo-audit
	- name: Cargo Audit
	run: cargo audit

	clippy:
	name: Clippy
	runs-on: ubuntu-latest
	needs: fmt
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
	- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
	with:
	components: clippy
	- name: Install protoc
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
	- uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
	with:
	path: \|
	~/.cargo/bin/
	~/.cargo/registry/index/
	~/.cargo/registry/cache/
	~/.cargo/git/db/
	target/
	key: ${{ runner.os }}-cargo-clippy-${{ hashFiles('**/Cargo.lock') }}
	restore-keys: \|
	${{ runner.os }}-cargo-clippy-
	- run: cargo clippy --workspace -- -D warnings

	test:
	name: Test
	runs-on: ubuntu-latest
	needs: fmt
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
	- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
	- name: Install protoc
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
	- uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
	with:
	path: \|
	~/.cargo/bin/
	~/.cargo/registry/index/
	~/.cargo/registry/cache/
	~/.cargo/git/db/
	target/
	key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
	restore-keys: \|
	${{ runner.os }}-cargo-test-
	- run: cargo test --workspace

	# ---------------------------------------------------------------------------
	# Coverage — line coverage report with cargo-llvm-cov
	# ---------------------------------------------------------------------------
	coverage:
	name: Coverage (llvm-cov)
	runs-on: ubuntu-latest
	needs: test
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
	- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
	with:
	components: llvm-tools-preview
	- name: Install protoc
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
	- name: Install cargo-llvm-cov
	uses: taiki-e/install-action@cca35edeb1d01366c2843b68fc3ca441446d73d3 # v2
	with:
	tool: cargo-llvm-cov
	- uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
	with:
	path: \|
	~/.cargo/bin/
	~/.cargo/registry/index/
	~/.cargo/registry/cache/
	~/.cargo/git/db/
	target/
	key: ${{ runner.os }}-cargo-coverage-${{ hashFiles('**/Cargo.lock') }}
	restore-keys: \|
	${{ runner.os }}-cargo-coverage-
	- name: Generate LCOV report
	run: cargo llvm-cov --workspace --lcov --output-path lcov.info
	- name: Generate HTML report
	run: cargo llvm-cov --workspace --html
	- name: Write Coverage Summary
	if: always()
	run: \|
	python3 - <<'PY'
	import os
	import re
	from pathlib import Path

	summary_path = Path(os.environ["GITHUB_STEP_SUMMARY"])
	html_index = Path("target/llvm-cov/html/index.html")

	def write(msg: str) -> None:
	# Append (don't clobber other summaries in the same job).
	with summary_path.open("a", encoding="utf-8") as f:
	f.write(msg)

	def fmt_row(label: str, pct: str, frac: str) -> str:
	return f"\| {label} \| {pct} \| {frac} \|"

	if not html_index.exists():
	write("## Coverage (llvm-cov)\n\nNo HTML report found.\n")
	raise SystemExit(0)

	try:
	html = html_index.read_text(encoding="utf-8", errors="ignore")
	m = re.search(
	r"<tr class='light-row-bold'>.?<pre>Totals</pre>.?</tr>",
	html,
	flags=re.S,
	)
	if not m:
	write("## Coverage (llvm-cov)\n\nFailed to locate totals row in HTML report.\n")
	raise SystemExit(0)

	row = m.group(0)
	cells = re.findall(r"<pre>\\s([^<]+?)\\s</pre>", row)
	# Typically: ["Totals", func, line, region, branch]. Some llvm-cov HTML
	# builds omit branch totals, so handle 3-metric totals rows too.
	metrics = (cells[1:] + ["?"] * 4)[:4]
	func, line, region, branch = metrics

	created = ""
	m2 = re.search(r"Created:\\s*([^<]+)</h4>", html)
	if m2:
	created = m2.group(1).strip()
	except SystemExit:
	raise
	except Exception as e:
	# Never fail the coverage job because the summary parser broke.
	write(f"## Coverage (llvm-cov)\n\nFailed to parse coverage HTML summary: `{e}`\n")
	write(
	"\nArtifacts: `coverage-report` contains `lcov.info` and `target/llvm-cov/html/index.html`.\n"
	)
	raise SystemExit(0)

	def split_metric(s: str):
	s = s.strip()
	m = re.match(r"([0-9]+\\.[0-9]+%)\\s*\$([^)]+)\$", s)
	if not m:
	return s, ""
	return m.group(1), m.group(2)

	func_pct, func_frac = split_metric(func)
	line_pct, line_frac = split_metric(line)
	region_pct, region_frac = split_metric(region)
	branch_pct, branch_frac = split_metric(branch)

	parts = []
	parts.append("## Coverage (llvm-cov)\n")
	if created:
	parts.append(f"_Report created: {created}_\n\n")
	parts.append("\| Metric \| % \| Covered/Total \|\n")
	parts.append("\|---\|---:\|---:\|\n")
	parts.append(fmt_row("Function", func_pct, func_frac) + "\n")
	parts.append(fmt_row("Line", line_pct, line_frac) + "\n")
	parts.append(fmt_row("Region", region_pct, region_frac) + "\n")
	parts.append(fmt_row("Branch", branch_pct, branch_frac if branch_frac else "") + "\n")
	parts.append(
	"\nArtifacts: `coverage-report` contains `lcov.info` and `target/llvm-cov/html/index.html`.\n"
	)

	write("".join(parts))
	PY
	- name: Upload coverage to Codecov
	# If CODECOV_TOKEN isn't configured, keep CI green and rely on artifacts.
	if: ${{ (github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository) && env.CODECOV_TOKEN != '' }}
	uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5
	with:
	files: lcov.info
	token: ${{ env.CODECOV_TOKEN }}
	fail_ci_if_error: true
	verbose: true
	- name: Skip Codecov upload (no token)
	if: ${{ env.CODECOV_TOKEN == '' }}
	run: echo "Skipping Codecov upload because secrets.CODECOV_TOKEN is not set. Coverage artifacts will still be uploaded."
	- name: Upload coverage artifacts
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
	with:
	name: coverage-report
	path: \|
	lcov.info
	target/llvm-cov/html/

	# ---------------------------------------------------------------------------
	# Integration tests — run ignored tests against real Docker Compose services
	# ---------------------------------------------------------------------------
	integration:
	name: Integration Tests
	runs-on: ubuntu-latest
	needs: test
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
	- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
	- name: Install protoc
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
	- uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
	with:
	path: \|
	~/.cargo/bin/
	~/.cargo/registry/index/
	~/.cargo/registry/cache/
	~/.cargo/git/db/
	target/
	key: ${{ runner.os }}-cargo-integration-${{ hashFiles('**/Cargo.lock') }}
	restore-keys: \|
	${{ runner.os }}-cargo-integration-

	- name: Start infrastructure services
	run: docker compose up -d clickhouse postgres redis
	env:
	CLICKHOUSE_PORT: "8123"
	CLICKHOUSE_NATIVE_PORT: "9000"
	CLICKHOUSE_DATABASE: llmtrace
	POSTGRES_USER: llmtrace
	POSTGRES_PASSWORD: llmtrace
	POSTGRES_DB: llmtrace
	REDIS_PORT: "6379"

	- name: Wait for services to be ready
	run: \|
	echo "Waiting for ClickHouse..."
	for i in $(seq 1 60); do
	if curl -sf http://localhost:8123/ping >/dev/null 2>&1; then
	echo "ClickHouse ready after ${i}s"
	break
	fi
	if [ "$i" -eq 60 ]; then
	echo "ClickHouse failed after 60 attempts"
	docker compose logs clickhouse
	exit 1
	fi
	sleep 2
	done
	echo "Waiting for PostgreSQL..."
	for i in $(seq 1 30); do
	if docker compose exec -T postgres pg_isready -U llmtrace >/dev/null 2>&1; then
	echo "PostgreSQL ready after ${i}s"
	break
	fi
	if [ "$i" -eq 30 ]; then
	echo "PostgreSQL failed after 30 attempts"
	docker compose logs postgres
	exit 1
	fi
	sleep 2
	done
	echo "Waiting for Redis..."
	for i in $(seq 1 30); do
	if docker compose exec -T redis redis-cli ping >/dev/null 2>&1; then
	echo "Redis ready after ${i}s"
	break
	fi
	if [ "$i" -eq 30 ]; then
	echo "Redis failed after 30 attempts"
	docker compose logs redis
	exit 1
	fi
	sleep 2
	done
	echo "All services ready."

	- name: Run integration tests
	env:
	LLMTRACE_CLICKHOUSE_URL: http://localhost:8123
	LLMTRACE_CLICKHOUSE_DATABASE: llmtrace
	LLMTRACE_POSTGRES_URL: postgres://llmtrace:llmtrace@localhost:5432/llmtrace
	LLMTRACE_REDIS_URL: redis://127.0.0.1:6379
	run: cargo test --workspace --features "clickhouse,postgres,redis_backend" -- --ignored

	- name: Stop services
	if: always()
	run: docker compose down -v

	e2e:
	name: E2E Tests
	runs-on: ubuntu-latest
	needs: integration
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

	- name: Set up Node.js
	uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
	with:
	node-version: 24
	cache: 'npm'
	cache-dependency-path: dashboard/package-lock.json

	- name: Install Playwright
	run: \|
	cd dashboard
	npm ci
	npx playwright install --with-deps

	- name: Build and Start Stack
	env:
	NEXT_PUBLIC_API_URL: http://127.0.0.1:8080
	run: \|
	# `compose.yaml` binds `./config.yaml` into the proxy container. Ensure the
	# file exists in CI to avoid Docker interpreting it as a directory.
	cat > config.yaml <<'YAML'
	listen_addr: "0.0.0.0:8080"
	upstream_url: "http://example.com"
	timeout_ms: 30000
	connection_timeout_ms: 5000
	max_connections: 1000
	enable_tls: false
	enable_security_analysis: true
	enable_trace_storage: true
	enable_streaming: true
	max_request_size_bytes: 52428800
	security_analysis_timeout_ms: 5000
	trace_storage_timeout_ms: 10000

	rate_limiting:
	enabled: true
	requests_per_second: 100
	burst_size: 200
	window_seconds: 60

	circuit_breaker:
	enabled: true
	failure_threshold: 10
	recovery_timeout_ms: 30000
	half_open_max_calls: 3

	health_check:
	enabled: true
	path: "/health"
	interval_seconds: 10
	timeout_ms: 5000
	retries: 3

	storage:
	profile: "production"
	auto_migrate: true
	clickhouse_url: "http://clickhouse:8123"
	clickhouse_database: "llmtrace"
	postgres_url: "postgres://llmtrace:llmtrace@postgres:5432/llmtrace"
	redis_url: "redis://redis:6379"

	logging:
	level: "info"
	format: "text"
	YAML

	# Use local build for proxy and dashboard
	docker compose build
	docker compose up -d

	- name: Wait for services
	run: \|
	echo "Waiting for Proxy..."
	proxy_ready=0
	for i in $(seq 1 60); do
	if curl -sf http://localhost:8080/health >/dev/null 2>&1; then
	echo "Proxy ready after ${i}s"
	proxy_ready=1
	break
	fi
	sleep 2
	done
	if [ "$proxy_ready" -ne 1 ]; then
	echo "Proxy did not become ready in time"
	docker compose ps
	docker compose logs llmtrace-proxy \|\| true
	exit 1
	fi
	echo "Waiting for Dashboard..."
	dashboard_ready=0
	for i in $(seq 1 30); do
	if curl -sf http://localhost:3000 >/dev/null 2>&1; then
	echo "Dashboard ready after ${i}s"
	dashboard_ready=1
	break
	fi
	sleep 2
	done
	if [ "$dashboard_ready" -ne 1 ]; then
	echo "Dashboard did not become ready in time"
	docker compose ps
	docker compose logs dashboard \|\| true
	exit 1
	fi
	echo "Stack is up."

	- name: Run E2E Tests
	env:
	LLMTRACE_PROXY_URL: http://127.0.0.1:8080
	PLAYWRIGHT_BASE_URL: http://127.0.0.1:3000
	run: \|
	cd dashboard
	# Ensure `playwright-results.json` is valid JSON (don't mix reporter output).
	npx playwright test --reporter=json > ../playwright-results.json

	- name: Write E2E Summary
	if: always()
	run: \|
	python3 - <<'PY'
	import json
	import os
	from pathlib import Path

	summary_path = Path(os.environ["GITHUB_STEP_SUMMARY"])
	report = Path("playwright-results.json")

	def write(msg: str) -> None:
	with summary_path.open("a", encoding="utf-8") as f:
	f.write(msg)

	if not report.exists():
	write("## E2E (Playwright)\n\nNo `playwright-results.json` found.\n")
	raise SystemExit(0)

	try:
	data = json.loads(report.read_text(encoding="utf-8", errors="ignore") or "{}")
	except Exception as e:
	# Never fail the E2E job due to summary parsing.
	write("## E2E (Playwright)\n\n")
	write(f"Failed to parse `playwright-results.json`: `{e}`\n")
	write("\nArtifacts: `playwright-report` contains raw Playwright outputs.\n")
	raise SystemExit(0)

	def walk(obj):
	if isinstance(obj, dict):
	yield obj
	for v in obj.values():
	yield from walk(v)
	elif isinstance(obj, list):
	for it in obj:
	yield from walk(it)

	# Count test outcomes by scanning all dicts with a `status` key.
	counts = {"passed": 0, "failed": 0, "skipped": 0, "timedOut": 0, "interrupted": 0, "flaky": 0}
	for o in walk(data):
	status = o.get("status")
	if status in counts:
	counts[status] += 1

	total = sum(counts.values())
	duration_ms = data.get("stats", {}).get("duration")
	duration = f"{duration_ms/1000:.1f}s" if isinstance(duration_ms, (int, float)) else ""

	lines = []
	lines.append("## E2E (Playwright)\\n\\n")
	if duration:
	lines.append(f"_Duration: {duration}_\\n\\n")
	lines.append("\| Outcome \| Count \|\\n")
	lines.append("\|---\|---:\|\\n")
	for k in ["passed", "failed", "skipped", "flaky", "timedOut", "interrupted"]:
	lines.append(f"\| {k} \| {counts[k]} \|\\n")
	lines.append(f"\| total \| {total} \|\\n")
	lines.append("\\nArtifacts: `playwright-report`.\\n")

	with summary_path.open("a", encoding="utf-8") as f:
	f.write("".join(lines))
	PY

	- name: Upload Playwright Report
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
	with:
	name: playwright-report
	path: \|
	playwright-results.json
	dashboard/test-results/
	dashboard/playwright-report/
	retention-days: 30

	- name: Stop stack
	if: always()
	run: docker compose down -v

	benchmarks:
	name: Benchmarks
	runs-on: ubuntu-latest
	needs: test
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
	- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
	- name: Install protoc
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
	- uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
	with:
	path: \|
	~/.cargo/bin/
	~/.cargo/registry/index/
	~/.cargo/registry/cache/
	~/.cargo/git/db/
	target/
	key: ${{ runner.os }}-cargo-bench-${{ hashFiles('**/Cargo.lock') }}
	restore-keys: \|
	${{ runner.os }}-cargo-bench-
	- name: Run benchmark regression gate (regex only)
	env:
	RUST_LOG: warn
	run: cargo run --bin benchmarks -- --output-dir benchmarks/results --analyzer regex
	- name: Upload benchmark results
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
	if: always()
	with:
	name: benchmark-results
	path: benchmarks/results/

	build:
	name: Build
	runs-on: ubuntu-latest
	needs: [clippy, test, coverage]
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
	- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
	- name: Install protoc
	run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
	- uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
	with:
	path: \|
	~/.cargo/bin/
	~/.cargo/registry/index/
	~/.cargo/registry/cache/
	~/.cargo/git/db/
	target/
	key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }}
	restore-keys: \|
	${{ runner.os }}-cargo-build-
	- run: cargo build --workspace --release

	# ---------------------------------------------------------------------------
	# Container scan (advisory) — scan Docker image on PRs, don't fail
	# ---------------------------------------------------------------------------
	trivy-scan:
	name: Trivy Container Scan
	runs-on: ubuntu-latest
	needs: build
	permissions:
	security-events: write
	steps:
	- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3

	- name: Build Docker image for scanning
	uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
	with:
	context: .
	push: false
	load: true
	tags: llmtrace-proxy:scan
	cache-from: type=gha
	cache-to: type=gha,mode=max

	- name: Run Trivy vulnerability scanner
	uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # 0.35.0
	with:
	image-ref: llmtrace-proxy:scan
	format: sarif
	output: trivy-results.sarif
	severity: CRITICAL,HIGH
	exit-code: "0"

	- name: Upload Trivy SARIF to GitHub Security
	uses: github/codeql-action/upload-sarif@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4
	if: always()
	with:
	sarif_file: trivy-results.sarif

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

ops(e2e): enable IS-060 datamarking in shadow mode for nightly valida… #489

Workflow file

ops(e2e): enable IS-060 datamarking in shadow mode for nightly valida… #489

Uh oh!

Workflow file for this run