Repository navigation
E2E Framework Tests #278
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: E2E Framework Tests | |
| # Checkpoint/restore of the inference framework guide workloads (vLLM, SGLang, | |
| # TensorRT-LLM), one vCluster per framework. The cluster setup mirrors | |
| # e2e.yaml; the differences are the 64Gi checkpoint PVC (framework artifacts | |
| # are multi-GB), the framework image resolution, and the diagnostics upload. | |
| # | |
| # Pull requests are tested by manually dispatching this workflow on their | |
| # copy-pr-bot mirror branch (pull-request/<number>, see | |
| # .github/copy-pr-bot.yaml): the GPU runners admit push, schedule, and | |
| # workflow_dispatch events but reject pull_request. There is no automatic | |
| # per-push trigger -- mirror branches usually have no operator/agent/pagebroker | |
| # images of their own, so an automatic run would silently fall back to | |
| # testing main's Snapshot instead of the PR's own changes (see | |
| # hack/resolve-snapshot-image-tag.py). Dispatch push-artifacts.yaml on the PR | |
| # branch first to build its own images, then dispatch this workflow with the | |
| # resulting snapshot_tag against the pull-request/<number> mirror branch (not | |
| # the source branch) so the benchmark preview keys to that PR. | |
| # | |
| # Known-red on this gate, kept red on purpose (not this workflow's bug): | |
| # - vLLM: intermittent CRIU restore segfault ("criu swrk failed: signal: | |
| # segmentation fault"), a product defect in criu itself, not this repo's | |
| # restore wiring -- see PR #179's discussion for the current investigation. | |
| # - SGLang: occasional post-restore hang in resume_memory_occupation(); | |
| # largely resolved by switching to PauseGenerationReqInput(mode="retract") | |
| # (this PR), but not eliminated. | |
| # - TensorRT-LLM has been consistently green. | |
| # Do not re-diagnose these from scratch on a failing run; check whether the | |
| # failure matches one of the above first. | |
| on: | |
| schedule: | |
| # 01:00 Pacific Standard Time, after the nightly e2e.yaml run. | |
| - cron: "0 9 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| framework: | |
| description: Framework to test (all, vllm, sglang, tensorrt-llm) | |
| required: false | |
| default: all | |
| type: choice | |
| options: [all, vllm, sglang, tensorrt-llm] | |
| snapshot_tag: | |
| description: >- | |
| Operator/agent image tag to install. pull-request/<number> mirror | |
| branches require an explicit tag. Leave empty on main/release/* | |
| branches to resolve from the commit under test; other non-PR | |
| branches fall back to the newest main build when the commit has no | |
| published images. | |
| required: false | |
| type: string | |
| datadog_gpu_monitoring: | |
| description: >- | |
| Datadog GPU monitoring on the host cluster during the run. "disable" | |
| turns it off before the tests and back on afterwards if it was on. | |
| required: false | |
| default: disable | |
| type: choice | |
| options: [disable, leave] | |
| permissions: | |
| contents: read | |
| packages: read | |
| jobs: | |
| check: | |
| name: Validate repository | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| persist-credentials: false | |
| - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 | |
| with: | |
| go-version: "1.27.2" | |
| cache: true | |
| cache-dependency-path: | | |
| **/go.sum | |
| go.work.sum | |
| - run: make check | |
| # Datadog's GPU monitoring (system-probe + NVML) is a suspected interferer | |
| # for CRIU/CUDA checkpoint and restore. Switch it off for the whole run in | |
| # one gate, restore it in a separate always-run job -- a per-job toggle | |
| # would re-enable it under a sibling still restoring (see | |
| # hack/datadog-gpu-monitoring.py, which tracks state on the resources so | |
| # overlapping runs hand off correctly). | |
| datadog-gate: | |
| name: datadog gpu monitoring off | |
| needs: check | |
| runs-on: prod-deploy-tester-v1 | |
| timeout-minutes: 5 | |
| concurrency: | |
| group: snapshot-e2e-frameworks-datadog-gate | |
| cancel-in-progress: false | |
| outputs: | |
| mode: ${{ steps.mode.outputs.mode }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - name: Resolve mode | |
| id: mode | |
| env: | |
| MODE: ${{ inputs.datadog_gpu_monitoring || 'disable' }} | |
| run: echo "mode=${MODE}" >> "$GITHUB_OUTPUT" | |
| - name: Report Datadog GPU monitoring state | |
| run: python3 hack/datadog-gpu-monitoring.py status | |
| - name: Disable Datadog GPU monitoring for this run | |
| if: ${{ steps.mode.outputs.mode == 'disable' }} | |
| run: python3 hack/datadog-gpu-monitoring.py disable --holder "${{ github.run_id }}-${{ github.run_attempt }}" | |
| e2e: | |
| name: ${{ matrix.framework }} | |
| needs: [check, datadog-gate] | |
| runs-on: prod-deploy-tester-v1 | |
| # Above the sum of this job's own step timeout-minutes (117, dominated by | |
| # the 50-min test step) plus margin: a job-level timeout kills the whole | |
| # job, including the failure() diagnostics upload and always() vCluster | |
| # cleanup, which is exactly the safety net a slow run needs most. | |
| timeout-minutes: 140 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| framework: [vllm, sglang, tensorrt-llm] | |
| concurrency: | |
| # PR mirror branches get one group per branch, so a new push cancels | |
| # the stale run. Scheduled/dispatched runs share a fixed 'nightly' | |
| # group per framework so they queue instead of racing on this | |
| # GPU-scarce cluster -- cancel-in-progress stays false there so a | |
| # queued run executes instead of being dropped. | |
| group: snapshot-e2e-frameworks-${{ startsWith(github.ref, 'refs/heads/pull-request/') && github.ref_name || 'nightly' }}-${{ matrix.framework }} | |
| cancel-in-progress: ${{ startsWith(github.ref, 'refs/heads/pull-request/') }} | |
| env: | |
| HOST_NAMESPACE: snapshot-e2e-fw-${{ matrix.framework }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| VCLUSTER_NAME: snapshot-e2e-fw-${{ matrix.framework }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| TEST_NAMESPACE: snapshot-e2e | |
| SNAPSHOT_E2E_HOST_NAMESPACE: snapshot-e2e-fw-${{ matrix.framework }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| SNAPSHOT_E2E_VCLUSTER_NAME: snapshot-e2e-fw-${{ matrix.framework }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| SNAPSHOT_E2E_TEST_NAMESPACE: snapshot-e2e | |
| SNAPSHOT_E2E_TARGET_KUBECONFIG: ${{ github.workspace }}/.kubeconfig-snapshot-e2e | |
| SNAPSHOT_E2E_SNAPSHOT_RELEASE: snapshot | |
| SNAPSHOT_E2E_PVC_NAME: snapshot-pvc | |
| SNAPSHOT_E2E_PVC_SIZE: 64Gi | |
| SNAPSHOT_E2E_STORAGE_CLASS: azurefile-csi | |
| SNAPSHOT_E2E_MODE: vcluster | |
| SNAPSHOT_E2E_AVOID_MIG: "true" | |
| SNAPSHOT_E2E_VCLUSTER_K8S_VERSION: v1.32.13 | |
| SNAPSHOT_E2E_VCLUSTER_LOCAL_PORT: "8443" | |
| SNAPSHOT_E2E_HELM_TIMEOUT: 6m | |
| SNAPSHOT_E2E_READY_TIMEOUT_SECONDS: "480" | |
| SNAPSHOT_E2E_RESULT_FILE: ${{ github.workspace }}/.snapshot-e2e-setup-result.json | |
| SNAPSHOT_E2E_FRAMEWORK: ${{ matrix.framework }} | |
| SNAPSHOT_E2E_BENCHMARK_DIR: ${{ github.workspace }}/.e2e-benchmark-results | |
| SNAPSHOT_E2E_DATADOG_GPU_MONITORING: ${{ needs.datadog-gate.outputs.mode }} | |
| # Shared CI model cache (HF cache layout on an NFS export): pods here | |
| # can't resolve huggingface.co, and per-run downloads cost minutes | |
| # anyway. Repository variables override the literals below, which are | |
| # only a fallback until those variables exist. | |
| SNAPSHOT_E2E_MODEL_CACHE_SERVER: ${{ vars.SNAPSHOT_E2E_MODEL_CACHE_SERVER || 'dynamoinfracimodelcache.file.core.windows.net' }} | |
| SNAPSHOT_E2E_MODEL_CACHE_PATH: ${{ vars.SNAPSHOT_E2E_MODEL_CACHE_PATH || '/dynamoinfracimodelcache/models' }} | |
| SNAPSHOT_E2E_MODEL_CACHE_FROM_VARS: ${{ vars.SNAPSHOT_E2E_MODEL_CACHE_SERVER != '' && vars.SNAPSHOT_E2E_MODEL_CACHE_PATH != '' }} | |
| PYTHONUNBUFFERED: "1" | |
| UV_LINK_MODE: copy | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - name: Skip frameworks not selected by dispatch | |
| id: selected | |
| env: | |
| SELECTED: ${{ inputs.framework || 'all' }} | |
| run: | | |
| set -euo pipefail | |
| if [[ "${SELECTED}" == "all" || "${SELECTED}" == "${{ matrix.framework }}" ]]; then | |
| echo "run=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "run=false" >> "$GITHUB_OUTPUT" | |
| echo "Skipping ${{ matrix.framework }}: dispatch selected ${SELECTED}" | |
| fi | |
| - name: Capture host kubeconfig | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| id: host | |
| run: | | |
| echo "kubeconfig=${KUBECONFIG:-}" >> "${GITHUB_OUTPUT}" | |
| - name: Install vCluster CLI | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 3 | |
| uses: ./.github/actions/install-vcluster-cli | |
| - name: Cleanup stale e2e namespaces | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 5 | |
| run: | | |
| set -euo pipefail | |
| cutoff_epoch="$(date -u -d '12 hours ago' +%s)" | |
| kubectl get ns -l snapshot-e2e=true \ | |
| -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.creationTimestamp}{"\n"}{end}' | | |
| while IFS=$'\t' read -r namespace created_at; do | |
| if [[ -z "${namespace}" ]]; then | |
| continue | |
| fi | |
| if [[ "${namespace}" == "${HOST_NAMESPACE}" ]]; then | |
| echo "Keeping current namespace ${namespace}" | |
| continue | |
| fi | |
| created_epoch="$(date -u -d "${created_at}" +%s)" | |
| if (( created_epoch >= cutoff_epoch )); then | |
| echo "Keeping recent namespace ${namespace} created at ${created_at}" | |
| continue | |
| fi | |
| echo "Deleting stale Snapshot e2e namespace ${namespace} created at ${created_at}" | |
| helm uninstall vcluster-hpm -n "${namespace}" || true | |
| vcluster delete "${namespace}" --namespace "${namespace}" || true | |
| kubectl delete namespace "${namespace}" --ignore-not-found --wait=false || true | |
| done | |
| - name: Install Python tooling (ARM64) | |
| if: ${{ steps.selected.outputs.run == 'true' && runner.arch == 'ARM64' }} | |
| timeout-minutes: 3 | |
| uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 | |
| with: | |
| version: "0.10.11" | |
| checksum: "23003df007937dd607409c8ddf010baa82bad2673e60e254632ca5b04edcce13" | |
| enable-cache: "false" | |
| download-from-astral-mirror: "false" | |
| - name: Install Python tooling (X64) | |
| if: ${{ steps.selected.outputs.run == 'true' && runner.arch == 'X64' }} | |
| timeout-minutes: 3 | |
| uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 | |
| with: | |
| version: "0.10.11" | |
| checksum: "5a360b0de092ddf4131f5313d0411b48c4e95e8107e40c3f8f2e9fcb636b3583" | |
| enable-cache: "false" | |
| download-from-astral-mirror: "false" | |
| - name: Resolve Snapshot image tag | |
| if: ${{ steps.selected.outputs.run == 'true' && (inputs.snapshot_tag || '') == '' }} | |
| timeout-minutes: 3 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| # A PR mirror dispatch with no snapshot_tag would silently fall back | |
| # to main's images (see --fallback-main below) and still publish a | |
| # benchmark preview keyed to that PR, looking like it tested the | |
| # PR's own changes when it didn't. Require snapshot_tag there. | |
| if [[ "${GITHUB_REF_NAME}" =~ ^pull-request/[0-9]+$ ]]; then | |
| echo "::error::Dispatching against PR mirror branch ${GITHUB_REF_NAME} requires an explicit snapshot_tag, otherwise the run silently falls back to testing main's images. Dispatch push-artifacts.yaml on the PR's source branch first, then re-dispatch this workflow with the resulting tag." >&2 | |
| exit 1 | |
| fi | |
| # push-artifacts publishes only main/release, so any other ref (a | |
| # non-PR feature branch dispatched by hand) normally has no images | |
| # of its own; fall back to main's, loudly. main and release refs | |
| # stay strict: a nightly must test the commit it checked out. | |
| case "${GITHUB_REF_NAME}" in | |
| main|release/*) python3 hack/resolve-snapshot-image-tag.py ;; | |
| *) python3 hack/resolve-snapshot-image-tag.py --fallback-main ;; | |
| esac | |
| - name: Use provided Snapshot image tag | |
| if: ${{ steps.selected.outputs.run == 'true' && (inputs.snapshot_tag || '') != '' }} | |
| timeout-minutes: 1 | |
| run: | | |
| set -euo pipefail | |
| echo "SNAPSHOT_E2E_SNAPSHOT_TAG=${{ inputs.snapshot_tag }}" >> "$GITHUB_ENV" | |
| echo "Using provided Snapshot image tag: ${{ inputs.snapshot_tag }}" | |
| - name: Report model cache | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| run: | | |
| if [[ "${SNAPSHOT_E2E_MODEL_CACHE_FROM_VARS}" == "true" ]]; then | |
| echo "Shared model cache from repository variables: ${SNAPSHOT_E2E_MODEL_CACHE_SERVER}:${SNAPSHOT_E2E_MODEL_CACHE_PATH}" | |
| else | |
| echo "::notice::Shared model cache uses the workflow default ${SNAPSHOT_E2E_MODEL_CACHE_SERVER}:${SNAPSHOT_E2E_MODEL_CACHE_PATH}; set repository variables SNAPSHOT_E2E_MODEL_CACHE_SERVER and SNAPSHOT_E2E_MODEL_CACHE_PATH to override." | |
| fi | |
| - name: Validate host GPU cluster | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 3 | |
| run: | | |
| set -euo pipefail | |
| uv run --locked --project e2e python -m snapshot_e2e.infra.setup \ | |
| --phase host-preflight | |
| - name: Create vCluster target | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 12 | |
| run: | | |
| set -euo pipefail | |
| uv run --locked --project e2e python -m snapshot_e2e.infra.setup \ | |
| --phase vcluster | |
| - name: Install Snapshot in vCluster | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 8 | |
| run: | | |
| set -euo pipefail | |
| uv run --locked --project e2e python -m snapshot_e2e.infra.setup \ | |
| --phase snapshot-install | |
| - name: Wait for Snapshot components | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 10 | |
| run: | | |
| set -euo pipefail | |
| uv run --locked --project e2e python -m snapshot_e2e.infra.setup \ | |
| --phase snapshot-ready | |
| - name: Export vCluster kubeconfig | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 3 | |
| run: | | |
| set -euo pipefail | |
| uv run --locked --project e2e python - <<'PY' | |
| import json | |
| import os | |
| from pathlib import Path | |
| result = json.loads(Path(os.environ["SNAPSHOT_E2E_RESULT_FILE"]).read_text()) | |
| with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as env: | |
| env.write(f"KUBECONFIG={result['target_kubeconfig']}\n") | |
| env.write(f"SNAPSHOT_E2E_HOST_NAMESPACE={result.get('host_namespace', '')}\n") | |
| env.write(f"SNAPSHOT_E2E_VCLUSTER_NAME={result.get('vcluster_name', '')}\n") | |
| env.write(f"SNAPSHOT_E2E_TEST_NAMESPACE={result['test_namespace']}\n") | |
| env.write(f"SNAPSHOT_E2E_TARGET_KUBECONFIG={result['target_kubeconfig']}\n") | |
| env.write(f"SNAPSHOT_E2E_SNAPSHOT_RELEASE={result['snapshot_release']}\n") | |
| env.write(f"SNAPSHOT_E2E_PVC_NAME={result['pvc_name']}\n") | |
| print(f"Target kubeconfig: {result['target_kubeconfig']}") | |
| PY | |
| - name: Inspect vCluster target | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 3 | |
| run: | | |
| set -euo pipefail | |
| kubectl get nodes -o wide | |
| kubectl get runtimeclass | |
| kubectl get pods -n "${TEST_NAMESPACE}" -o wide | |
| kubectl get pvc -n "${TEST_NAMESPACE}" -o wide | |
| - name: Run environment validation | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 5 | |
| run: | | |
| set -euo pipefail | |
| uv run --locked --project e2e pytest e2e/tests -m environment -vv | |
| # The inner timeout must exceed the phase budgets in | |
| # e2e/snapshot_e2e/frameworks.py plus the request timeout (45 min with | |
| # sglang's 600s restore_timeout_seconds override). SIGINT reaches the | |
| # whole process group, so pytest raises KeyboardInterrupt, runs its | |
| # fixture finalizers (benchmark result, cluster cleanup), and exits | |
| # before the KILL backstop; GNU timeout still returns 124. | |
| - name: Run ${{ matrix.framework }} checkpoint/restore test | |
| if: ${{ steps.selected.outputs.run == 'true' }} | |
| id: framework_test | |
| timeout-minutes: 50 | |
| run: | | |
| set -uo pipefail | |
| timeout --signal=INT --kill-after=150s 46m \ | |
| uv run --locked --project e2e pytest e2e/tests/test_frameworks.py -vv -s | |
| status=$? | |
| if [[ "${status}" -eq 124 ]]; then | |
| PYTHONPATH=e2e python3 -m snapshot_e2e.benchmark fallback \ | |
| --suite framework-checkpoint-restore \ | |
| --case "${{ matrix.framework }}" \ | |
| --test test_framework_checkpoint_restore_serves_inference \ | |
| --outcome timed_out \ | |
| --message "pytest exceeded the 46 minute benchmark timeout" | |
| fi | |
| exit "${status}" | |
| - name: Ensure benchmark result exists | |
| if: ${{ always() && steps.selected.outputs.run == 'true' }} | |
| env: | |
| TEST_STEP_OUTCOME: ${{ steps.framework_test.outcome }} | |
| run: | | |
| set -euo pipefail | |
| PYTHONPATH=e2e python3 -m snapshot_e2e.benchmark fallback \ | |
| --suite framework-checkpoint-restore \ | |
| --case "${{ matrix.framework }}" \ | |
| --test test_framework_checkpoint_restore_serves_inference \ | |
| --outcome infrastructure_failed \ | |
| --message "pytest produced no benchmark result; step outcome=${TEST_STEP_OUTCOME}" | |
| # Pytest can succeed with skipped cases. A selected backend only qualifies | |
| # when its own result passed in this run attempt; stale workspace files or | |
| # synthesized failure records must never stand in for restored inference. | |
| - name: Require passing ${{ matrix.framework }} result | |
| if: ${{ always() && steps.selected.outputs.run == 'true' }} | |
| run: | | |
| set -euo pipefail | |
| PYTHONPATH=e2e python3 -m snapshot_e2e.benchmark_history require-pass \ | |
| --artifacts-dir "${SNAPSHOT_E2E_BENCHMARK_DIR}" \ | |
| --case "${{ matrix.framework }}" | |
| - name: Upload benchmark result | |
| if: ${{ always() && steps.selected.outputs.run == 'true' }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-framework-benchmark-${{ matrix.framework }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ env.SNAPSHOT_E2E_BENCHMARK_DIR }}/*.json | |
| if-no-files-found: error | |
| include-hidden-files: true | |
| retention-days: 30 | |
| # The test already prints its own dump; this preserves the raw cluster | |
| # state and full agent logs, which do not fit in a job log. | |
| - name: Collect diagnostics | |
| if: ${{ failure() && steps.selected.outputs.run == 'true' }} | |
| timeout-minutes: 5 | |
| run: | | |
| set +e | |
| out="${RUNNER_TEMP}/diagnostics" | |
| mkdir -p "${out}" | |
| kubectl get pods -n "${TEST_NAMESPACE}" -o yaml > "${out}/pods.yaml" | |
| kubectl describe pods -n "${TEST_NAMESPACE}" > "${out}/pods-describe.txt" | |
| kubectl get events -n "${TEST_NAMESPACE}" --sort-by=.lastTimestamp > "${out}/events.txt" | |
| kubectl get podsnapshots -n "${TEST_NAMESPACE}" -o yaml > "${out}/podsnapshots.yaml" | |
| kubectl get podsnapshotcontents -o yaml > "${out}/podsnapshotcontents.yaml" | |
| kubectl logs -n "${TEST_NAMESPACE}" -l app.kubernetes.io/component=snapshot-agent \ | |
| --all-containers --tail=-1 --prefix > "${out}/snapshot-agent.log" | |
| kubectl logs -n "${TEST_NAMESPACE}" -l app.kubernetes.io/component=snapshot-operator \ | |
| --all-containers --tail=-1 --prefix > "${out}/snapshot-operator.log" | |
| for pod in $(kubectl get pods -n "${TEST_NAMESPACE}" -l snapshot-e2e-test -o name); do | |
| kubectl logs -n "${TEST_NAMESPACE}" "${pod}" --all-containers --tail=-1 \ | |
| > "${out}/${pod#pod/}.log" | |
| done | |
| # Kernel trap lines are the only trace a crashed CRIU leaves; the | |
| # agent is privileged with hostPID, so its dmesg is the node's. | |
| for pod in $(kubectl get pods -n "${TEST_NAMESPACE}" -l app.kubernetes.io/component=snapshot-agent -o name); do | |
| kubectl exec -n "${TEST_NAMESPACE}" "${pod}" -c agent -- \ | |
| sh -c 'dmesg -T 2>/dev/null | tail -300' > "${out}/dmesg-${pod#pod/}.txt" 2>&1 || true | |
| done | |
| echo "DIAGNOSTICS_DIR=${out}" >> "$GITHUB_ENV" | |
| - name: Upload diagnostics | |
| if: ${{ failure() && steps.selected.outputs.run == 'true' }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-frameworks-${{ matrix.framework }}-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ env.DIAGNOSTICS_DIR }} | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Cleanup e2e vCluster | |
| if: ${{ always() && steps.selected.outputs.run == 'true' }} | |
| env: | |
| KUBECONFIG: ${{ steps.host.outputs.kubeconfig }} | |
| run: | | |
| set +e | |
| helm uninstall vcluster-hpm -n "${HOST_NAMESPACE}" | |
| if command -v vcluster >/dev/null 2>&1; then | |
| vcluster delete "${VCLUSTER_NAME}" --namespace "${HOST_NAMESPACE}" | |
| fi | |
| kubectl delete namespace "${HOST_NAMESPACE}" --ignore-not-found --wait=false | |
| datadog-restore: | |
| name: datadog gpu monitoring back on | |
| needs: [datadog-gate, e2e] | |
| # Run whether the tests passed, failed, or were cancelled; only skip when | |
| # the gate never ran (then nothing was disabled and nothing is held). | |
| if: ${{ always() && needs.datadog-gate.result == 'success' && needs.datadog-gate.outputs.mode == 'disable' }} | |
| runs-on: prod-deploy-tester-v1 | |
| timeout-minutes: 5 | |
| concurrency: | |
| group: snapshot-e2e-frameworks-datadog-gate | |
| cancel-in-progress: false | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| - name: Restore Datadog GPU monitoring | |
| run: python3 hack/datadog-gpu-monitoring.py restore --holder "${{ github.run_id }}-${{ github.run_attempt }}" | |
| # PR mirrors and ordinary dispatches compare against history with a | |
| # read-only token. Keep this separate from the scheduled publisher: mirror | |
| # branches contain PR code but run as push events, so merely guarding a | |
| # write step inside one write-capable job would still expose that token. | |
| benchmark-history-read-only: | |
| name: compare framework benchmark history | |
| needs: e2e | |
| if: ${{ !cancelled() && !(github.event_name == 'schedule' && github.ref == 'refs/heads/main') }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| actions: read | |
| contents: read | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| # All attempts of this run: "Re-run failed jobs" re-executes only the | |
| # failed matrix jobs, so the passing frameworks' results still live in | |
| # the earlier attempt's artifacts. | |
| # No continue-on-error: a pattern matching nothing is not a failure for | |
| # this action, so the flag would only turn a real API error into a set of | |
| # fabricated infrastructure_failed results. | |
| - name: Download framework benchmark results | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| pattern: e2e-framework-benchmark-*-${{ github.run_id }}-* | |
| path: ${{ runner.temp }}/e2e-framework-benchmark-results | |
| merge-multiple: true | |
| - name: Fetch benchmark history | |
| id: history | |
| uses: ./.github/actions/fetch-benchmark-history | |
| with: | |
| token: ${{ github.token }} | |
| - name: Compare benchmark results | |
| env: | |
| SELECTED_FRAMEWORK: ${{ inputs.framework || 'all' }} | |
| SNAPSHOT_E2E_HISTORY_DIR: ${{ steps.history.outputs.history_dir }} | |
| SNAPSHOT_E2E_SNAPSHOT_TAG: ${{ inputs.snapshot_tag || '' }} | |
| run: | | |
| set -euo pipefail | |
| cases=() | |
| if [[ "${SELECTED_FRAMEWORK}" == "all" ]]; then | |
| cases=(vllm sglang tensorrt-llm) | |
| else | |
| cases=("${SELECTED_FRAMEWORK}") | |
| fi | |
| case_args=() | |
| for framework in "${cases[@]}"; do | |
| case_args+=(--expected-case "${framework}") | |
| done | |
| PYTHONPATH=e2e python3 -m snapshot_e2e.benchmark_history aggregate \ | |
| --artifacts-dir "${RUNNER_TEMP}/e2e-framework-benchmark-results" \ | |
| --history-dir "${SNAPSHOT_E2E_HISTORY_DIR}" \ | |
| --output-dir "${RUNNER_TEMP}/e2e-framework-benchmark-comparison" \ | |
| --summary-file "${GITHUB_STEP_SUMMARY}" \ | |
| "${case_args[@]}" | |
| - name: Upload benchmark comparison | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-benchmark-comparison-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ runner.temp }}/e2e-framework-benchmark-comparison | |
| if-no-files-found: error | |
| retention-days: 30 | |
| # Exactly one scheduled default-branch job receives contents:write. It | |
| # aggregates all matrix artifacts and pushes one serialized history commit. | |
| benchmark-history-publish: | |
| name: publish framework benchmark history | |
| needs: e2e | |
| # !cancelled(), not always(): an operator-cancelled nightly must not | |
| # synthesize and permanently publish infrastructure_failed records that | |
| # identity dedup then stops a re-run from replacing. | |
| if: ${{ !cancelled() && github.event_name == 'schedule' && github.ref == 'refs/heads/main' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| concurrency: | |
| group: snapshot-e2e-benchmark-history | |
| cancel-in-progress: false | |
| permissions: | |
| actions: read | |
| contents: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| persist-credentials: false | |
| # No continue-on-error: a pattern matching nothing is not a failure for | |
| # this action, so the flag would only turn a real API error into a set of | |
| # fabricated infrastructure_failed results. | |
| - name: Download framework benchmark results | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| pattern: e2e-framework-benchmark-*-${{ github.run_id }}-* | |
| path: ${{ runner.temp }}/e2e-framework-benchmark-results | |
| merge-multiple: true | |
| - name: Fetch or initialize benchmark history | |
| id: history | |
| uses: ./.github/actions/fetch-benchmark-history | |
| with: | |
| token: ${{ github.token }} | |
| bootstrap: "true" | |
| persist-credentials: "true" | |
| - name: Aggregate and index benchmark results | |
| env: | |
| SNAPSHOT_E2E_HISTORY_DIR: ${{ steps.history.outputs.history_dir }} | |
| run: | | |
| set -euo pipefail | |
| echo "SNAPSHOT_E2E_HISTORY_DIR=${SNAPSHOT_E2E_HISTORY_DIR}" >> "${GITHUB_ENV}" | |
| PYTHONPATH=e2e python3 -m snapshot_e2e.benchmark_history aggregate \ | |
| --artifacts-dir "${RUNNER_TEMP}/e2e-framework-benchmark-results" \ | |
| --history-dir "${SNAPSHOT_E2E_HISTORY_DIR}" \ | |
| --output-dir "${RUNNER_TEMP}/e2e-framework-benchmark-comparison" \ | |
| --summary-file "${GITHUB_STEP_SUMMARY}" \ | |
| --expected-case vllm \ | |
| --expected-case sglang \ | |
| --expected-case tensorrt-llm \ | |
| --publish | |
| - name: Commit and push benchmark history | |
| run: | | |
| set -euo pipefail | |
| git -C "${SNAPSHOT_E2E_HISTORY_DIR}" config user.name "github-actions[bot]" | |
| git -C "${SNAPSHOT_E2E_HISTORY_DIR}" config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git -C "${SNAPSHOT_E2E_HISTORY_DIR}" add results index | |
| if git -C "${SNAPSHOT_E2E_HISTORY_DIR}" diff --cached --quiet; then | |
| echo "History already contains this workflow attempt; nothing to push." | |
| exit 0 | |
| fi | |
| git -C "${SNAPSHOT_E2E_HISTORY_DIR}" commit --signoff \ | |
| --message "data(e2e): record benchmark run ${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| git -C "${SNAPSHOT_E2E_HISTORY_DIR}" push origin HEAD:refs/heads/e2e-benchmark-history | |
| - name: Upload benchmark comparison | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: e2e-benchmark-comparison-${{ github.run_id }}-${{ github.run_attempt }} | |
| path: ${{ runner.temp }}/e2e-framework-benchmark-comparison | |
| if-no-files-found: error | |
| retention-days: 30 |