[WIP] Add daily egress connectivity allow list generation #994
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Infrastructure Verification | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| capture-egress: | |
| type: boolean | |
| default: false | |
| description: 'Install OpenSnitch, capture egress traffic, and upload allow list YAML (smoke test)' | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| merge_group: | |
| types: [checks_requested] | |
| # Allow parallel execution with unique cluster names per run | |
| # Each job gets isolated VMs, networks, and resources | |
| concurrency: | |
| group: enclave-ci-${{ github.run_id }} | |
| cancel-in-progress: false | |
| jobs: | |
| infra-verify: | |
| name: Infrastructure Verification | |
| runs-on: [self-hosted, enclave-small] | |
| timeout-minutes: 120 | |
| env: | |
| DEV_SCRIPTS_PATH: ${{ vars.DEV_SCRIPTS_PATH }} | |
| BASE_WORKING_DIR: ${{ vars.BASE_WORKING_DIR }} | |
| PULL_SECRET: ${{ secrets.PULL_SECRET }} | |
| # Bypass CI_TOKEN requirement (we only use dev-scripts for infra, not cluster install) | |
| OPENSHIFT_CI: "true" | |
| ENCLAVE_ENABLE_GPU_PASSTHROUGH: "false" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| # TEMPORARY: enable egress capture via trigger file (remove before merging) | |
| - name: '[TEMP] Check egress capture trigger' | |
| id: egress_trigger | |
| run: | | |
| if [ -f ".github/TRIGGER_CAPTURE_INFRA" ]; then | |
| echo "enabled=true" >> $GITHUB_OUTPUT | |
| echo "Egress capture enabled via trigger file" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "enabled=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Generate unique cluster name | |
| id: cluster_name | |
| uses: ./.github/actions/setup-cluster-name | |
| with: | |
| naming-strategy: hash | |
| prefix: eci | |
| run-id: ${{ github.run_id }} | |
| - name: Export cluster name to environment | |
| run: | | |
| echo "ENCLAVE_CLUSTER_NAME=${{ steps.cluster_name.outputs.cluster-name }}" >> $GITHUB_ENV | |
| echo "Cluster name for this workflow: ${{ steps.cluster_name.outputs.cluster-name }}" | |
| - name: Setup cluster-specific working directory | |
| env: | |
| BASE_WORKING_DIR: ${{ env.BASE_WORKING_DIR }} | |
| run: | | |
| echo "Using cluster name: ${ENCLAVE_CLUSTER_NAME}" | |
| ./scripts/setup/setup_working_dir.sh | |
| echo "WORKING_DIR=$(cat /tmp/working_dir)" >> $GITHUB_ENV | |
| # Create directory for step logs | |
| mkdir -p step-logs | |
| echo "STEP_LOGS_DIR=$PWD/step-logs" >> $GITHUB_ENV | |
| - name: Workflow information | |
| env: | |
| PR_TITLE: ${{ github.event.pull_request.title }} | |
| run: | | |
| echo "## Workflow Information" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Trigger**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Cleanup Strategy**: $CLEANUP_STRATEGY" >> $GITHUB_STEP_SUMMARY | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| echo "- **PR Number**: #${{ github.event.pull_request.number }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **PR Title**: $PR_TITLE" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| - name: Pre-flight checks | |
| uses: ./.github/actions/preflight-checks | |
| with: | |
| title: Infrastructure Verification Pre-flight Checks | |
| check-system-resources: 'true' | |
| check-libvirt: 'true' | |
| - name: Allocate unique subnet for cluster | |
| id: allocate_subnet | |
| uses: ./.github/actions/allocate-subnet | |
| - name: Save allocation log | |
| if: always() && steps.allocate_subnet.outcome != 'skipped' | |
| run: | | |
| echo "Subnet allocation completed with status: ${{ steps.allocate_subnet.outcome }}" > step-logs/01-allocate-subnet.log | |
| - name: Create infrastructure | |
| id: create_infra | |
| env: | |
| WORKING_DIR: ${{ env.WORKING_DIR }} | |
| run: | | |
| echo "## Creating Infrastructure" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Creating VMs, networks, and BMC emulation..." >> $GITHUB_STEP_SUMMARY | |
| echo "Working Directory: ${WORKING_DIR:-not-set}" >> $GITHUB_STEP_SUMMARY | |
| # Initialize log file with header | |
| echo "=== Infrastructure Setup Log ===" > step-logs/02-setup-infrastructure.log | |
| echo "Started: $(date)" >> step-logs/02-setup-infrastructure.log | |
| echo "" >> step-logs/02-setup-infrastructure.log | |
| # Run command and capture all output | |
| set +e | |
| make -f Makefile.ci environment 2>&1 | tee -a step-logs/02-setup-infrastructure.log | |
| EXIT_CODE=${PIPESTATUS[0]} | |
| set -e | |
| # Log completion | |
| echo "" >> step-logs/02-setup-infrastructure.log | |
| echo "Completed: $(date)" >> step-logs/02-setup-infrastructure.log | |
| echo "Exit code: $EXIT_CODE" >> step-logs/02-setup-infrastructure.log | |
| # Exit with the original code | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| echo "❌ Infrastructure creation failed (exit code: $EXIT_CODE)" >> $GITHUB_STEP_SUMMARY | |
| exit $EXIT_CODE | |
| fi | |
| echo "✅ Infrastructure created" >> $GITHUB_STEP_SUMMARY | |
| - name: Provision Landing Zone | |
| id: provision_lz | |
| env: | |
| WORKING_DIR: ${{ env.WORKING_DIR }} | |
| run: | | |
| echo "## Provisioning Landing Zone" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Installing CentOS Stream and configuring Landing Zone VM..." >> $GITHUB_STEP_SUMMARY | |
| make -f Makefile.ci provision-landing-zone | |
| echo "✅ Landing Zone provisioned" >> $GITHUB_STEP_SUMMARY | |
| - name: Setup OpenSnitch for egress capture | |
| if: inputs.capture-egress == true || steps.egress_trigger.outputs.enabled == 'true' | |
| run: | | |
| echo "## Setting up OpenSnitch" >> $GITHUB_STEP_SUMMARY | |
| make -f Makefile.ci setup-opensnitch | |
| echo "OpenSnitch running on Landing Zone" >> $GITHUB_STEP_SUMMARY | |
| - name: Install Enclave Lab (Connected Mode) | |
| id: install_enclave | |
| run: | | |
| echo "## Installing Enclave Lab" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "Running in connected mode for faster testing..." >> $GITHUB_STEP_SUMMARY | |
| ENCLAVE_DEPLOYMENT_MODE=connected make -f Makefile.ci install-enclave 2>&1 | tee step-logs/03-install-enclave.log | |
| - name: Collect egress logs | |
| if: (inputs.capture-egress == true || steps.egress_trigger.outputs.enabled == 'true') && !cancelled() | |
| run: make -f Makefile.ci collect-egress-logs | |
| - name: Analyze egress and generate allow list | |
| if: (inputs.capture-egress == true || steps.egress_trigger.outputs.enabled == 'true') && !cancelled() | |
| run: | | |
| python3 scripts/egress/analyze_egress.py \ | |
| artifacts/egress/opensnitch-connections.json.gz infra \ | |
| > egress-allowlist-infra.yaml | |
| cat egress-allowlist-infra.yaml | |
| echo "## Egress Allow List (smoke test — infra only)" >> $GITHUB_STEP_SUMMARY | |
| echo '```yaml' >> $GITHUB_STEP_SUMMARY | |
| cat egress-allowlist-infra.yaml >> $GITHUB_STEP_SUMMARY | |
| echo '```' >> $GITHUB_STEP_SUMMARY | |
| - name: Collect step logs | |
| if: always() | |
| env: | |
| WORKING_DIR: ${{ env.WORKING_DIR }} | |
| run: make -f Makefile.ci collect-step-logs | |
| - name: Collect artifacts | |
| if: always() | |
| uses: ./.github/actions/collect-artifacts | |
| with: | |
| artifact-type: infra | |
| output-directory: ci-artifacts | |
| - name: Upload artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: infra-verify-artifacts | |
| path: | | |
| ci-artifacts/ | |
| step-logs/ | |
| egress-allowlist-infra.yaml | |
| artifacts/egress/ | |
| retention-days: 7 | |
| - name: Cleanup infrastructure | |
| if: always() | |
| run: | | |
| echo "## Cleanup Infrastructure" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Run cleanup and capture exit code | |
| set +e | |
| make -f Makefile.ci clean 2>&1 | tee step-logs/03-cleanup.log | |
| CLEANUP_EXIT_CODE=$? | |
| set -e | |
| # Report results | |
| if [ $CLEANUP_EXIT_CODE -eq 0 ]; then | |
| echo "✅ Cleanup completed successfully" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "⚠️ Cleanup completed with errors (exit code: $CLEANUP_EXIT_CODE)" >> $GITHUB_STEP_SUMMARY | |
| echo "Check step-logs/03-cleanup.log for details" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Collect post-cleanup state | |
| if: always() | |
| run: | | |
| mkdir -p cleanup-state/libvirt cleanup-state/system cleanup-logs | |
| # Copy cleanup log if it exists | |
| [ -f step-logs/03-cleanup.log ] && cp step-logs/03-cleanup.log cleanup-logs/ || true | |
| # Capture post-cleanup system state | |
| sudo virsh pool-list --all --details > cleanup-state/libvirt/storage-pools.txt 2>&1 || true | |
| sudo virsh net-list --all > cleanup-state/libvirt/networks.txt 2>&1 || true | |
| sudo virsh list --all > cleanup-state/libvirt/vms.txt 2>&1 || true | |
| df -h > cleanup-state/system/disk-usage.txt 2>&1 || true | |
| - name: Verify cleanup completion | |
| if: always() | |
| run: | | |
| echo "## Cleanup Verification" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Check for leftover resources | |
| LEFTOVER_POOLS=$(sudo virsh pool-list --all --name | grep "^eci-" || true) | |
| LEFTOVER_NETWORKS=$(sudo virsh net-list --all --name | grep "^eci-.*-[pe]$" || true) | |
| LEFTOVER_VMS=$(sudo virsh list --all --name | grep "^eci-" || true) | |
| # Save verification results to file (for artifacts) | |
| { | |
| echo "=== Cleanup Verification Results ===" | |
| echo "Timestamp: $(date -u +"%Y%m%d-%H%M%S")" | |
| echo "" | |
| if [ -z "$LEFTOVER_POOLS" ] && [ -z "$LEFTOVER_NETWORKS" ] && [ -z "$LEFTOVER_VMS" ]; then | |
| echo "✅ No leftover resources detected" | |
| else | |
| echo "⚠️ Leftover resources detected:" | |
| [ -n "$LEFTOVER_POOLS" ] && echo "- Pools: $LEFTOVER_POOLS" | |
| [ -n "$LEFTOVER_NETWORKS" ] && echo "- Networks: $LEFTOVER_NETWORKS" | |
| [ -n "$LEFTOVER_VMS" ] && echo "- VMs: $LEFTOVER_VMS" | |
| fi | |
| } | tee cleanup-logs/verification.txt | |
| # Also report to step summary | |
| if [ -z "$LEFTOVER_POOLS" ] && [ -z "$LEFTOVER_NETWORKS" ] && [ -z "$LEFTOVER_VMS" ]; then | |
| echo "✅ No leftover resources detected" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "⚠️ **Leftover resources detected:**" >> $GITHUB_STEP_SUMMARY | |
| [ -n "$LEFTOVER_POOLS" ] && echo "- **Pools**: $(echo $LEFTOVER_POOLS | tr '\n' ' ')" >> $GITHUB_STEP_SUMMARY | |
| [ -n "$LEFTOVER_NETWORKS" ] && echo "- **Networks**: $(echo $LEFTOVER_NETWORKS | tr '\n' ' ')" >> $GITHUB_STEP_SUMMARY | |
| [ -n "$LEFTOVER_VMS" ] && echo "- **VMs**: $(echo $LEFTOVER_VMS | tr '\n' ' ')" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "These resources should be cleaned up manually or via the cleanup workflow." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload post-cleanup artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: post-cleanup-state | |
| path: | | |
| cleanup-logs/ | |
| cleanup-state/ | |
| retention-days: 7 | |
| - name: Workflow summary | |
| if: always() | |
| run: | | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "---" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "## Workflow Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Trigger**: ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Result**: ${{ job.status }}" >> $GITHUB_STEP_SUMMARY |