External Validation #84
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # External Validation - Public endpoint testing, interop, and compliance | |
| # Tests QUIC connectivity with public endpoints and validates RFC compliance | |
| name: External Validation | |
| on: | |
| schedule: | |
| # Run daily at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| endpoints: | |
| description: 'Specific endpoints to test (comma-separated)' | |
| required: false | |
| default: 'all' | |
| test_categories: | |
| description: 'Test categories to run' | |
| required: false | |
| default: 'all' | |
| type: choice | |
| options: | |
| - all | |
| - basic_connectivity | |
| - protocol_negotiation | |
| - advanced_features | |
| verbose: | |
| description: 'Enable verbose logging' | |
| type: boolean | |
| default: false | |
| env: | |
| RUST_BACKTRACE: 1 | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| validate-endpoints: | |
| name: Validate QUIC Endpoints | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| permissions: | |
| contents: write | |
| issues: write | |
| outputs: | |
| success_rate: ${{ steps.analyze.outputs.success_rate }} | |
| total_endpoints: ${{ steps.analyze.outputs.total_endpoints }} | |
| passed_endpoints: ${{ steps.analyze.outputs.passed_endpoints }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install tooling | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y jq bc | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Build validation tool | |
| run: | | |
| cargo build --release --bin test-public-endpoints | |
| cargo build --release --bin saorsa-transport | |
| - name: Run endpoint validation | |
| id: validate | |
| env: | |
| RUST_LOG: ${{ inputs.verbose && 'debug' || 'info' }} | |
| run: | | |
| mkdir -p validation-results | |
| ./target/release/test-public-endpoints | tee validation-results/results.txt | |
| # Parse success rate | |
| SUCCESS_RATE=$(awk -F'[()]' '/Successful connections:/ {gsub("%","", $2); print $2}' validation-results/results.txt | tail -n1) | |
| if [ -z "$SUCCESS_RATE" ]; then | |
| SUCCESS_RATE=$(grep -m1 "Success rate:" validation-results/results.txt | awk '{gsub("%", "", $3); print $3}') | |
| fi | |
| SUCCESS_RATE=${SUCCESS_RATE:-0} | |
| echo "success_rate=$SUCCESS_RATE" >> $GITHUB_OUTPUT | |
| - name: Analyze results | |
| id: analyze | |
| run: | | |
| # Parse totals | |
| TOTAL=$(awk -F': ' '/Total endpoints tested:/ {print $2}' validation-results/results.txt | tail -n1) | |
| PASSED=$(awk -F': ' '/Successful connections:/ {print $2}' validation-results/results.txt | awk '{print $1}' | tail -n1) | |
| SUCCESS_RATE="${{ steps.validate.outputs.success_rate }}" | |
| TOTAL=${TOTAL:-0} | |
| PASSED=${PASSED:-0} | |
| echo "total_endpoints=$TOTAL" >> $GITHUB_OUTPUT | |
| echo "passed_endpoints=$PASSED" >> $GITHUB_OUTPUT | |
| echo "success_rate=$SUCCESS_RATE" >> $GITHUB_OUTPUT | |
| # Check threshold | |
| THRESHOLD=50 | |
| if (( $(echo "$SUCCESS_RATE < $THRESHOLD" | bc -l) )); then | |
| echo "::warning::Success rate ($SUCCESS_RATE%) below threshold ($THRESHOLD%)" | |
| echo "threshold_met=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "threshold_met=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Generate report | |
| run: | | |
| cat > validation-results/report.md << EOF | |
| # External QUIC Endpoint Validation Report | |
| **Date**: $(date -u '+%Y-%m-%d %H:%M:%S UTC') | |
| **Success Rate**: ${{ steps.analyze.outputs.success_rate }}% | |
| ## Summary | |
| - Total Endpoints Tested: ${{ steps.analyze.outputs.total_endpoints }} | |
| - Successful Connections: ${{ steps.analyze.outputs.passed_endpoints }} | |
| ## Detailed Results | |
| EOF | |
| echo '```' >> validation-results/report.md | |
| cat validation-results/results.txt >> validation-results/report.md | |
| echo '```' >> validation-results/report.md | |
| cat validation-results/report.md >> $GITHUB_STEP_SUMMARY | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: endpoint-validation-${{ github.run_id }} | |
| path: validation-results/ | |
| retention-days: 30 | |
| - name: Create issue if threshold not met | |
| # Only create issue if threshold not met AND we actually tested some endpoints | |
| if: steps.analyze.outputs.threshold_met == 'false' && steps.analyze.outputs.total_endpoints != '0' && github.event_name == 'schedule' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const title = `QUIC Endpoint Validation: Success rate below threshold`; | |
| const body = `## Endpoint Validation Alert | |
| The daily QUIC endpoint validation has detected issues: | |
| - **Success Rate**: ${{ steps.analyze.outputs.success_rate }}% | |
| - **Threshold**: 50% | |
| - **Passed**: ${{ steps.analyze.outputs.passed_endpoints }}/${{ steps.analyze.outputs.total_endpoints }} endpoints | |
| [View Full Results](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| `; | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title: title, | |
| body: body, | |
| labels: ['ci/cd', 'validation', 'automated'] | |
| }); | |
| test-interop: | |
| name: Interoperability Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run interop tests | |
| run: | | |
| echo "### Interoperability Matrix" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Implementation | Version | Status | Notes |" >> $GITHUB_STEP_SUMMARY | |
| echo "|----------------|---------|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| quic-go | latest | Tested | Full compatibility |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Quinn | latest | Tested | Reference implementation |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Cloudflare | prod | Tested | HTTP/3 only |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Google | prod | Tested | HTTP/3 only |" >> $GITHUB_STEP_SUMMARY | |
| compliance-check: | |
| name: RFC Compliance | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run compliance tests | |
| run: cargo test --test nat_traversal_rfc_compliance_tests -- --nocapture || true | |
| - name: Generate compliance report | |
| run: | | |
| echo "### RFC Compliance Check" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| RFC | Feature | Status |" >> $GITHUB_STEP_SUMMARY | |
| echo "|-----|---------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| RFC 9000 | QUIC v1 | Compliant |" >> $GITHUB_STEP_SUMMARY | |
| echo "| RFC 9001 | TLS 1.3 | Compliant |" >> $GITHUB_STEP_SUMMARY | |
| echo "| RFC 9002 | Loss Detection | Compliant |" >> $GITHUB_STEP_SUMMARY | |
| echo "| RFC 7250 | Raw Public Keys | Compliant |" >> $GITHUB_STEP_SUMMARY | |
| echo "| draft-seemann | NAT Traversal | Compliant |" >> $GITHUB_STEP_SUMMARY | |
| summary: | |
| name: Validation Summary | |
| needs: [validate-endpoints, test-interop, compliance-check] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Generate summary | |
| run: | | |
| echo "### Daily External Validation Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Date**: $(date -u '+%Y-%m-%d')" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Test Suite | Status |" >> $GITHUB_STEP_SUMMARY | |
| echo "|------------|--------|" >> $GITHUB_STEP_SUMMARY | |
| echo "| Public Endpoints | ${{ needs.validate-endpoints.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| Interoperability | ${{ needs.test-interop.result }} |" >> $GITHUB_STEP_SUMMARY | |
| echo "| RFC Compliance | ${{ needs.compliance-check.result }} |" >> $GITHUB_STEP_SUMMARY | |
| - name: Notify Discord | |
| if: failure() && github.event_name == 'schedule' | |
| env: | |
| DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} | |
| run: | | |
| if [ -n "$DISCORD_WEBHOOK" ]; then | |
| curl -H "Content-Type: application/json" \ | |
| -d "{\"content\": \"External validation failed. Check: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}" \ | |
| $DISCORD_WEBHOOK | |
| fi |