nightly-e2e.yaml

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Nightly E2E tests:
#
#   cloud-e2e                Cloud inference (NVIDIA Endpoint API) on ubuntu-latest.
#   messaging-providers-e2e  Validates messaging credential provider/placeholder/L7-proxy chain
#                            for Telegram + Discord + Slack. Uses fake tokens. Slack additionally
#                            exercises OpenShell provider-shaped alias resolution (#2085 follow-up).
#   openclaw-slack-pairing-e2e
#                            Validates hermetic Slack Socket Mode pairing request approval across
#                            gateway and connect-shell OpenClaw state roots (#3730/#3737).
#   openclaw-discord-pairing-e2e
#                            Validates hermetic Discord pairing request approval across
#                            gateway and connect-shell OpenClaw state roots (#4061).
#   issue-4462-scope-upgrade-approval-e2e
#                            Validates real CLI scope-upgrade approval and confirms
#                            the approved agent run stays on gateway mode (#4462).
#   issue-4462-gateway-pinned-approval-characterization-e2e
#                            Characterizes legacy gateway-pinned scope approval
#                            against a real sandbox, then recovers with the fix.
#   messaging-compatible-endpoint-e2e
#                            Validates Telegram + OpenAI-compatible endpoint inference routing
#                            through inference.local with a hermetic local mock (#2766).
#   kimi-inference-compat-e2e
#                            Validates Kimi K2.6 safe exec splitting through OpenClaw trajectories
#                            with a hermetic OpenAI-compatible mock (#2620).
#   bedrock-runtime-compatible-anthropic-e2e
#                            Validates the silent Bedrock Runtime custom Anthropic endpoint path
#                            through a hermetic fake Bedrock Runtime host for OpenClaw and Hermes.
#   token-rotation-e2e       Validates that rotating a messaging token and re-running onboard
#                            propagates the new credential to the sandbox. Combined Telegram +
#                            Discord + Slack coverage with cross-talk assertions. See issue #1903.
#   sandbox-survival-e2e     Sandbox survival across gateway restarts (onboard, inference,
#                            gateway stop/start, verify sandbox + workspace + inference).
#   openshell-gateway-upgrade-e2e
#                            Validates real v0.0.36 curl install upgrade into
#                            the current supported OpenShell with pre-upgrade backup, restored
#                            agent state, and the same agent type running.
#   hermes-e2e               Hermes Agent E2E — install → onboard --agent hermes → health
#                            probe → live inference. Validates the multi-agent architecture.
#   hermes-dashboard-e2e     Hermes Agent E2E with optional web dashboard enabled,
#                            validating API/dashboard forwards and host reachability.
#   hermes-root-entrypoint-smoke-e2e
#                            Builds the real Hermes image and verifies root entrypoint startup,
#                            gateway-user execution, v0.14 layout repair, and PID migration.
#   openclaw-onboard-security-posture-e2e
#                            Full OpenClaw onboard on a non-root host user
#                            with trusted rc-file and runtime guard assertions.
#   hermes-onboard-security-posture-e2e
#                            Full Hermes onboard on a non-root host user
#                            with trusted rc-file and runtime guard assertions.
#   hermes-inference-switch-e2e
#                            Switches a running Hermes sandbox with `nemohermes inference set`
#                            and verifies route, config.yaml, hashes, and live requests.
#   hermes-discord-e2e       Hermes Discord onboarding — validates the top-level Hermes
#                            Discord schema plus OpenShell placeholder/token isolation.
#   hermes-slack-e2e         Hermes Slack onboarding — validates the Hermes Slack policy,
#                            Slack providers, and OpenShell credential rewrite path.
#   openclaw-inference-switch-e2e
#                            Switches a running OpenClaw sandbox with `nemoclaw inference set`
#                            and verifies route, openclaw.json, hashes, and live requests.
#   credential-migration-e2e Validates legacy ~/.nemoclaw/credentials.json migration to the
#                            OpenShell gateway, secure zero-fill on unlink, allowlist filter
#                            on non-credential env keys, and symlink-safe deletion.
#   launchable-smoke-e2e     Community install path (brev-launchable-ci-cpu.sh) on ubuntu-latest.
#   gpu-e2e                  Local Ollama inference on an NVKS ephemeral GPU runner.
#   gpu-double-onboard-e2e   Ollama proxy token consistency after re-onboard (#2553).
#   notify-on-failure        Auto-creates a GitHub issue when any E2E job fails.
#
# Runs directly on the runner (not inside Docker) because OpenShell bootstraps
# a K3s cluster inside a privileged Docker container — nesting would break networking.
#
# NVIDIA_API_KEY for cloud-e2e:
#   - Repository secret: Settings → Secrets and variables → Actions → Repository secrets.
#   - Environment secret: only available if the job sets `environment: <that environment name>`.
#     (Storing the key under Environments / NVIDIA_API_KEY without `environment:` here leaves the
#     variable empty in the job — repository secrets and environment secrets are separate.)
# Only runs on schedule and manual dispatch — never on PRs (secret protection).

name: E2E / Nightly
run-name: >-
  ${{ github.event_name == 'workflow_dispatch' && inputs.advisor_dispatch_id != '' && format('E2E / Nightly ({0})', inputs.advisor_dispatch_id) || 'E2E / Nightly' }}

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:
    inputs:
      jobs:
        description: >-
          Comma-separated job names to run (empty = all).
          Valid: cloud-e2e, cloud-onboard-e2e, cloud-inference-e2e,
          skill-agent-e2e, docs-validation-e2e, messaging-providers-e2e,
          openclaw-slack-pairing-e2e,
          openclaw-tui-chat-correlation-e2e,
          issue-3600-gpu-proof-optional-e2e,
          openclaw-discord-pairing-e2e,
          issue-4462-scope-upgrade-approval-e2e,
          issue-4462-gateway-pinned-approval-characterization-e2e,
          messaging-compatible-endpoint-e2e,
          kimi-inference-compat-e2e,
          bedrock-runtime-compatible-anthropic-e2e,
          token-rotation-e2e, sandbox-survival-e2e,
          openshell-gateway-upgrade-e2e,
          issue-2478-crash-loop-recovery-e2e, hermes-e2e,
          hermes-dashboard-e2e,
          hermes-root-entrypoint-smoke-e2e,
          openclaw-onboard-security-posture-e2e,
          hermes-onboard-security-posture-e2e,
          hermes-inference-switch-e2e, hermes-discord-e2e,
          hermes-slack-e2e, sandbox-operations-e2e, inference-routing-e2e,
          openclaw-inference-switch-e2e,
          network-policy-e2e, state-backup-restore-e2e, tunnel-lifecycle-e2e, diagnostics-e2e,
          credential-migration-e2e,
          snapshot-commands-e2e, shields-config-e2e,
          vm-driver-privileged-exec-routing-e2e, rebuild-openclaw-e2e,
          upgrade-stale-sandbox-e2e, rebuild-hermes-e2e,
          rebuild-hermes-stale-base-e2e, double-onboard-e2e,
          onboard-repair-e2e, onboard-resume-e2e, onboard-negative-paths-e2e,
          runtime-overrides-e2e,
          credential-sanitization-e2e, telegram-injection-e2e,
          overlayfs-autofix-e2e, device-auth-health-e2e,
          launchable-smoke-e2e, gpu-e2e, gpu-double-onboard-e2e,
          channels-add-remove-e2e, channels-stop-start-e2e, brave-search-e2e
        required: false
        type: string
        default: ""
      target_ref:
        description: >-
          Optional branch, ref, or SHA to test. When empty, tests run against
          the workflow ref selected for the dispatch. Used by e2e-advisor
          auto-dispatch so the trusted main workflow can test a PR head SHA.
        required: false
        type: string
        default: ""
      pr_number:
        description: Optional PR number for selective-dispatch result comments.
        required: false
        type: string
        default: ""
      advisor_dispatch_id:
        description: Optional correlation ID from e2e-advisor auto-dispatch.
        required: false
        type: string
        default: ""

permissions:
  contents: read

concurrency:
  group: nightly-e2e-${{ github.event_name }}-${{ github.event_name == 'workflow_dispatch' && format('{0}-{1}', github.ref, inputs.pr_number || 'manual') || 'schedule' }}
  cancel-in-progress: true

# Selective-dispatch contract: tools/e2e-advisor/dispatch.mts discovers
# dispatchable jobs by looking for each job's exact predicate shape below:
#   github.event_name != 'workflow_dispatch' || inputs.jobs == '' ||
#   contains(format(',{0},', inputs.jobs), ',<job-id>,')
# Keep this predicate format in sync with test/e2e-advisor-dispatch.test.ts if
# the workflow changes how individual jobs opt in to selective dispatch.
jobs:
  cloud-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',cloud-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-full-e2e.sh
      artifact_name: "install-log"
      artifact_path: "/tmp/nemoclaw-e2e-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-nightly"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  cloud-onboard-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',cloud-onboard-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-cloud-onboard-e2e.sh
      artifact_name: "install-log-cloud-onboard"
      artifact_path: "/tmp/nemoclaw-e2e-cloud-onboard-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_MODE":"custom","NEMOCLAW_POLICY_PRESETS":"npm,pypi","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-cloud-onboard"}'
      checked_out_ref_env: "NEMOCLAW_PUBLIC_INSTALL_REF"
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  cloud-inference-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',cloud-inference-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-cloud-inference-e2e.sh
      timeout_minutes: 30
      artifact_name: "install-log-cloud-inference"
      artifact_path: "/tmp/nemoclaw-e2e-cloud-inference-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-cloud-inference"}'
      nvidia_api_key: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  skill-agent-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',skill-agent-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-skill-agent-e2e.sh
      timeout_minutes: 30
      artifact_name: "install-log-skill-agent"
      artifact_path: "/tmp/nemoclaw-e2e-skill-agent-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-skill-agent"}'
      nvidia_api_key: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  docs-validation-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',docs-validation-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Install NemoClaw
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: bash install.sh --non-interactive --yes-i-accept-third-party-software

      - name: Run docs validation
        env:
          CHECK_DOC_LINKS_REMOTE: "0"
        run: |
          set -euo pipefail
          [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true
          export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
          [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
          [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH"
          bash test/e2e/test-docs-validation.sh

  # ── Messaging Providers E2E ──────────────────────────────────
  # Validates the full provider/placeholder/L7-proxy chain for token-backed
  # messaging credentials, and the QR-only WhatsApp config/policy/no-provider
  # path. Uses fake tokens by default — the L7 proxy rewrites placeholders and
  # the real API returns 401, proving the chain works. See: PR #1081
  messaging-providers-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',messaging-providers-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-messaging-providers.sh
      timeout_minutes: 75
      artifact_name: "install-log-messaging-providers"
      artifact_path: |
        /tmp/nemoclaw-e2e-install.log
        /tmp/nemoclaw-e2e-whatsapp-*.log
      env_json: '{"DISCORD_BOT_TOKEN":"test-fake-discord-token-e2e","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-msg-provider","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-e2e","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-e2e"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
      TELEGRAM_BOT_TOKEN_REAL: ${{ secrets.TELEGRAM_BOT_TOKEN_REAL }}
      TELEGRAM_CHAT_ID_E2E: ${{ secrets.TELEGRAM_CHAT_ID_E2E }}
      DISCORD_BOT_TOKEN_REAL: ${{ secrets.DISCORD_BOT_TOKEN_REAL }}
      DISCORD_CHANNEL_ID_E2E: ${{ secrets.DISCORD_CHANNEL_ID_E2E }}
      SLACK_BOT_TOKEN_REAL: ${{ secrets.SLACK_BOT_TOKEN_REAL }}
      SLACK_APP_TOKEN_REAL: ${{ secrets.SLACK_APP_TOKEN_REAL }}
      SLACK_CHANNEL_ID_E2E: ${{ secrets.SLACK_CHANNEL_ID_E2E }}
  openclaw-slack-pairing-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',openclaw-slack-pairing-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-openclaw-slack-pairing.sh
      artifact_name: "install-log-openclaw-slack-pairing"
      artifact_path: "/tmp/nemoclaw-e2e-openclaw-slack-pairing-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-slack-pairing","SLACK_APP_TOKEN":"xapp-fake-slack-pairing-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-pairing-e2e"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  openclaw-tui-chat-correlation-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',openclaw-tui-chat-correlation-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 75
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Resolve public install ref
        id: public_install_ref
        shell: bash
        run: |
          printf 'ref=%s\n' "$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
        with:
          node-version: "22"

      - name: Install test dependencies
        run: npm ci --include=dev

      - name: Run OpenClaw TUI chat correlation E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_RECREATE_SANDBOX: "1"
          NEMOCLAW_SANDBOX_NAME: "e2e-openclaw-tui-correlation"
          NEMOCLAW_PUBLIC_INSTALL_REF: ${{ steps.public_install_ref.outputs.ref }}
          GITHUB_TOKEN: ${{ github.token }}
        run: bash test/e2e/test-openclaw-tui-chat-correlation.sh

      - name: Upload install log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: install-log-openclaw-tui-chat-correlation
          path: /tmp/nemoclaw-e2e-openclaw-tui-correlation-install.log
          if-no-files-found: ignore

  # ── DGX Station GPU optional proof validation (#3600) ──────────
  # CI cannot emulate GB300, but this guards the release-blocker mitigation:
  # optional direct GPU proofs must not abort onboard before the fatal throw.
  issue-3600-gpu-proof-optional-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',issue-3600-gpu-proof-optional-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
        with:
          node-version: "22"

      - name: Install test dependencies
        run: npm ci --include=dev

      - name: Verify optional GPU proof cannot abort onboard
        run: npx vitest run src/lib/onboard/sandbox-gpu-preflight.test.ts --pool=forks -t "direct sandbox GPU proof"

  # ── OpenClaw Discord Pairing E2E (#4061) ──────────────────────
  # Hermetic Discord Gateway placeholder rewrite proof, then connect-shell
  # `openclaw pairing approve discord <code>` against shared state.
  openclaw-discord-pairing-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',openclaw-discord-pairing-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-openclaw-discord-pairing.sh
      artifact_name: "install-log-openclaw-discord-pairing"
      artifact_path: "/tmp/nemoclaw-e2e-openclaw-discord-pairing-install.log"
      env_json: '{"DISCORD_BOT_TOKEN":"test-fake-discord-pairing-e2e","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-discord-pairing"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  # ── OpenClaw Scope-Upgrade Approval E2E (#4462) ────────────────
  # Positive proof: in a real sandbox, accept either a visible pending CLI
  # scope upgrade or the fixed watcher's immediate approval, then confirm
  # openclaw agent still uses the gateway path rather than embedded fallback.
  issue-4462-scope-upgrade-approval-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',issue-4462-scope-upgrade-approval-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-issue-4462-scope-upgrade-approval.sh
      timeout_minutes: 60
      artifact_name: "issue-4462-scope-upgrade-approval-logs"
      artifact_path: |
        /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-install.log
        /tmp/nemoclaw-issue-4462-scope-upgrade-approval.log
        /tmp/nemoclaw-issue-4462-scope-upgrade-agent.log
        /tmp/nemoclaw-issue-4462-scope-upgrade-state.log
      env_json: '{"NEMOCLAW_4462_MODE":"approval","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AUTO_PAIR_DEADLINE_SECS":"30","NEMOCLAW_AUTO_PAIR_FAST_DEADLINE_SECS":"3","NEMOCLAW_AUTO_PAIR_SLOW_INTERVAL_SECS":"600","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade"}'
      nvidia_api_key: true
      github_token: false
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  # ── OpenClaw Gateway-Pinned Approval Characterization (#4462) ──
  # Diagnostic proof: in a real sandbox, wait for the fixed watcher to exit,
  # force the legacy gateway-pinned approve path, record the observed
  # OpenClaw outcome, and recover through the fixed proxy-env guard if needed.
  issue-4462-gateway-pinned-approval-characterization-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',issue-4462-gateway-pinned-approval-characterization-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-issue-4462-scope-upgrade-approval.sh
      timeout_minutes: 60
      artifact_name: "issue-4462-gateway-pinned-approval-characterization-logs"
      artifact_path: |
        /tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log
        /tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log
        /tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log
        /tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log
      env_json: '{"NEMOCLAW_4462_AGENT_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-agent.log","NEMOCLAW_4462_APPROVAL_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-approval.log","NEMOCLAW_4462_AUTO_PAIR_DEADLINE_SECS":"12","NEMOCLAW_4462_AUTO_PAIR_FAST_DEADLINE_SECS":"1","NEMOCLAW_4462_AUTO_PAIR_RUN_TIMEOUT_SECS":"2","NEMOCLAW_4462_AUTO_PAIR_SLOW_INTERVAL_SECS":"1","NEMOCLAW_4462_INSTALL_LOG":"/tmp/nemoclaw-e2e-issue-4462-scope-upgrade-repro-install.log","NEMOCLAW_4462_MODE":"legacy-repro","NEMOCLAW_4462_STATE_LOG":"/tmp/nemoclaw-issue-4462-scope-upgrade-repro-state.log","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-issue-4462-scope-upgrade-repro"}'
      nvidia_api_key: true
      github_token: false
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  messaging-compatible-endpoint-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',messaging-compatible-endpoint-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-messaging-compatible-endpoint.sh
      artifact_name: "install-log-messaging-compatible-endpoint"
      artifact_path: "/tmp/nemoclaw-e2e-messaging-compatible-endpoint-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-msg-compat","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-e2e"}'
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  # ── Channels add/remove lifecycle E2E (#3462 Test 2) ────────────────
  # Regression coverage for #3437 (channels add must auto-apply the matching
  # network policy preset so the bridge boots with egress to its upstream API)
  # and #3671 (channels remove must detach providers, un-apply the preset,
  # and survive a follow-up rebuild without being silently re-added from
  # shell env). Telegram-only — the other paste-token channels walk the same
  # KNOWN_CHANNELS + preset lookup code path.
  channels-add-remove-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',channels-add-remove-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-channels-add-remove.sh
      timeout_minutes: 75
      artifact_name: "install-log-channels-add-remove"
      artifact_path: |
        /tmp/nemoclaw-e2e-install.log
        /tmp/nc-add.log
        /tmp/nc-remove.log
        /tmp/nc-rebuild-add.log
        /tmp/nc-rebuild-remove.log
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-channels-add-remove","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-add-remove-e2e"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  # ── Channels stop/start lifecycle E2E (#3462) ───────────────────────
  # Regression coverage for #3453 (stop must disable across rebuild), #3381
  # (start must re-attach from cached credentials).
  # Exercises OpenClaw and Hermes across telegram, discord, wechat, slack, and whatsapp.
  channels-stop-start-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',channels-stop-start-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-channels-stop-start.sh
      timeout_minutes: 120
      artifact_name: "install-log-channels-stop-start"
      artifact_path: |
        /tmp/nemoclaw-e2e-install.log
        /tmp/nemoclaw-e2e-channels-*-install.log
        /tmp/nc-channels-*.log
      env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-stop-start-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_ID":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_SANDBOX_NAME":"e2e-channels-stop-start","SLACK_ALLOWED_USERS":"U0123456789,U09ABCDEFGH","SLACK_APP_TOKEN":"xapp-fake-slack-app-token-stop-start-e2e","SLACK_BOT_TOKEN":"xoxb-fake-slack-token-stop-start-e2e","TELEGRAM_ALLOWED_IDS":"123456789","TELEGRAM_BOT_TOKEN":"test-fake-telegram-token-stop-start-e2e","WECHAT_ACCOUNT_ID":"e2e-fake-account-stop-start","WECHAT_ALLOWED_IDS":"wxid_stopstart_operator","WECHAT_BASE_URL":"https://ilinkai-fake-stop-start.wechat.com","WECHAT_BOT_TOKEN":"test-fake-wechat-token-stop-start-e2e","WECHAT_USER_ID":"wxid_stopstart_operator"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  brave-search-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',brave-search-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-brave-search-e2e.sh
      artifact_name: "install-log-brave-search"
      artifact_path: "/tmp/nemoclaw-e2e-brave-search-onboard.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-brave-search"}'
      brave_api_key: true
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  kimi-inference-compat-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',kimi-inference-compat-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Run Kimi inference compatibility E2E test
        env:
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_SANDBOX_NAME: "e2e-kimi-compat"
          GITHUB_TOKEN: ${{ github.token }}
        run: bash test/e2e/test-kimi-inference-compat.sh

      - name: Upload onboard log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: install-log-kimi-inference-compat
          path: /tmp/nemoclaw-e2e-kimi-inference-compat-onboard.log
          if-no-files-found: ignore

      - name: Upload build/setup log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: build-log-kimi-inference-compat
          path: /tmp/nemoclaw-e2e-kimi-inference-compat-build.log
          if-no-files-found: ignore

      - name: Upload agent log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: agent-log-kimi-inference-compat
          path: /tmp/nemoclaw-e2e-kimi-inference-compat-agent.log
          if-no-files-found: ignore

  # ── Bedrock Runtime compatible Anthropic endpoint (#3767) ─────
  # Hermetic fake Bedrock Runtime endpoint path. The sandbox only sees
  # inference.local; the host-side OpenShell provider owns the hidden adapter
  # token and the upstream Bedrock bearer derived from the fake pasted key.
  bedrock-runtime-compatible-anthropic-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',bedrock-runtime-compatible-anthropic-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        agent: [openclaw, hermes]
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Run Bedrock Runtime compatible Anthropic E2E test
        env:
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_RECREATE_SANDBOX: "1"
          NEMOCLAW_AGENT: ${{ matrix.agent }}
          NEMOCLAW_SANDBOX_NAME: e2e-bedrock-${{ matrix.agent }}
          GITHUB_TOKEN: ${{ github.token }}
        run: bash test/e2e/test-bedrock-runtime-compatible-anthropic.sh

      - name: Upload onboard log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: onboard-log-bedrock-runtime-compatible-anthropic-${{ matrix.agent }}
          path: /tmp/nemoclaw-e2e-bedrock-runtime-${{ matrix.agent }}-onboard.log
          if-no-files-found: ignore

      - name: Upload build/setup log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: build-log-bedrock-runtime-compatible-anthropic-${{ matrix.agent }}
          path: /tmp/nemoclaw-e2e-bedrock-runtime-${{ matrix.agent }}-build.log
          if-no-files-found: ignore

      - name: Upload fake Bedrock Runtime log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: mock-log-bedrock-runtime-compatible-anthropic-${{ matrix.agent }}
          path: /tmp/nemoclaw-e2e-bedrock-runtime-${{ matrix.agent }}-mock.log
          if-no-files-found: ignore

      - name: Upload Bedrock Runtime adapter log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: adapter-log-bedrock-runtime-compatible-anthropic-${{ matrix.agent }}
          path: ~/.nemoclaw/bedrock-runtime-adapter.log
          if-no-files-found: ignore

  # ── Token rotation (credential propagation to L7 proxy) ─────
  # Validates that rotating a messaging token and re-running onboard
  # propagates the new credential to the sandbox. Uses two fake tokens
  # per provider (Telegram + Discord) to prove the sandbox is rebuilt on
  # rotation and reused when unchanged.
  # See: issue #1903
  token-rotation-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',token-rotation-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Run token rotation E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_POLICY_TIER: "open"
          GITHUB_TOKEN: ${{ github.token }}
          TELEGRAM_BOT_TOKEN_A: "test-fake-token-A-rotation-e2e"
          TELEGRAM_BOT_TOKEN_B: "test-fake-token-B-rotation-e2e"
          DISCORD_BOT_TOKEN_A: "test-fake-discord-A-rotation-e2e"
          DISCORD_BOT_TOKEN_B: "test-fake-discord-B-rotation-e2e"
          SLACK_BOT_TOKEN_A: "xoxb-fake-A-rotation-e2e"
          SLACK_BOT_TOKEN_B: "xoxb-fake-B-rotation-e2e"
          SLACK_APP_TOKEN_A: "xapp-fake-A-rotation-e2e"
          SLACK_APP_TOKEN_B: "xapp-fake-B-rotation-e2e"
        run: bash test/e2e/test-token-rotation.sh

      - name: Upload install log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: install-log-token-rotation
          path: /tmp/nemoclaw-e2e-install.log
          if-no-files-found: ignore

  # ── Sandbox survival (gateway restart recovery) ──────────────
  sandbox-survival-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',sandbox-survival-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-sandbox-survival.sh
      timeout_minutes: 30
      artifact_name: "sandbox-survival-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-survival"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  issue-2478-crash-loop-recovery-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',issue-2478-crash-loop-recovery-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-issue-2478-crash-loop-recovery.sh
      timeout_minutes: 30
      artifact_name: "issue-2478-crash-loop-recovery-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-2478"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  hermes-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',hermes-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-hermes-e2e.sh
      timeout_minutes: 60
      artifact_name: "hermes-e2e-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-hermes-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  hermes-dashboard-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',hermes-dashboard-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-hermes-e2e.sh
      timeout_minutes: 60
      artifact_name: "hermes-dashboard-e2e-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-hermes-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_E2E_HERMES_DASHBOARD":"1","NEMOCLAW_HERMES_DASHBOARD":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-dashboard"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  hermes-root-entrypoint-smoke-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',hermes-root-entrypoint-smoke-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-hermes-root-entrypoint-smoke.sh
      timeout_minutes: 45
      artifact_name: "hermes-root-entrypoint-smoke-log"
      artifact_path: "/tmp/nemoclaw-hermes-root-entrypoint-smoke.log"
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  openclaw-onboard-security-posture-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',openclaw-onboard-security-posture-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-full-e2e.sh
      timeout_minutes: 60
      artifact_name: "openclaw-onboard-security-posture-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_E2E_EXPECT_NON_ROOT_HOST":"1","NEMOCLAW_E2E_SECURITY_POSTURE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-security-posture"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  hermes-onboard-security-posture-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',hermes-onboard-security-posture-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-hermes-e2e.sh
      timeout_minutes: 60
      artifact_name: "hermes-onboard-security-posture-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-hermes-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_E2E_EXPECT_NON_ROOT_HOST":"1","NEMOCLAW_E2E_SECURITY_POSTURE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-security-posture"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  hermes-inference-switch-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',hermes-inference-switch-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-hermes-inference-switch.sh
      timeout_minutes: 60
      artifact_name: "hermes-inference-switch-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-hermes-inference-switch-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-inference-switch"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  hermes-discord-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',hermes-discord-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-hermes-discord-e2e.sh
      timeout_minutes: 60
      artifact_name: "hermes-discord-e2e-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-hermes-discord-install.log"
      env_json: '{"DISCORD_ALLOWED_IDS":"1005536447329222676","DISCORD_BOT_TOKEN":"test-fake-discord-token-hermes-e2e","DISCORD_REQUIRE_MENTION":"0","DISCORD_SERVER_IDS":"1491590992753590594","NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-discord"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  hermes-slack-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',hermes-slack-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-hermes-slack-e2e.sh
      runner: linux-amd64-cpu4
      timeout_minutes: 60
      artifact_name: "hermes-slack-e2e-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-hermes-slack-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-hermes-slack","SLACK_APP_TOKEN":"xapp-test-hermes-slack-app-token","SLACK_BOT_TOKEN":"xoxb-test-hermes-slack-token"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  sandbox-operations-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',sandbox-operations-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Start gateway log streamer (background)
        run: |
          # Diagnostic for NVIDIA/NemoClaw#2484: container log driver in
          # openshell's k3s setup doesn't allow reading container stdio —
          # only working path to /tmp/gateway.log is via SSH, which
          # `nemoclaw <sandbox> logs` uses internally.
          #
          # Snapshot mode (not follow): every 10s, overwrite per-sandbox
          # log file with the latest gateway log content. Bounded output
          # (~62 lines per snapshot). When a sandbox is destroyed by the
          # test, the file holds the final pre-destroy snapshot.
          mkdir -p docker-logs
          nohup bash -c '
            export PATH="$HOME/.local/bin:$PATH"
            # Strategy: every 5s, snapshot each live sandbox via
            # `docker exec openshell-cluster-nemoclaw kubectl ...`. This
            # bypasses both per-pod networking (which has had connection-
            # refused races for some sandboxes) and the host openshell
            # client (which loses gateway metadata after TC-SBX-06s
            # docker-kill). kubectl talks directly to k3s in the cluster
            # container.
            #
            # Snapshot mode (overwrite per iteration), not live tail-F:
            # the gateway-persistent.log file accumulates everything since
            # boot (mirrored from /tmp/gateway.log by nemoclaw-start.sh),
            # so a single full-cat at any point gives us complete history.
            # Each iteration is short-lived so transient connection issues
            # do not cause us to lose the entire stream.
            #
            # Also snapshot kubectl pod listing per iteration so we have
            # the actual pod naming convention even if the cluster is
            # destroyed by teardown later.
            while sleep 5; do
              if ! docker ps --format "{{.Names}}" 2>/dev/null | grep -q "^openshell-cluster-nemoclaw$"; then
                continue
              fi
              docker exec openshell-cluster-nemoclaw kubectl get pods -A --no-headers >docker-logs/_pods.txt 2>&1
              registry="$HOME/.nemoclaw/sandboxes.json"
              [ -f "$registry" ] || continue
              live=$(jq -r ".sandboxes // {} | keys[]?" "$registry" 2>/dev/null)
              for name in $live; do
                case "$name" in
                  *[!a-z0-9_-]*|"") continue ;;
                esac
                # Find pod by sandbox name. openshell uses the sandbox
                # name as the namespace and "agent" as the pod name.
                # Try a few common patterns.
                pod_match=$(awk -v n="$name" "\$1==n || \$2==n || \$1==\"sandbox-\" n || \$2==\"sandbox-\" n {print \$1\"/\"\$2; exit}" docker-logs/_pods.txt)
                if [ -z "$pod_match" ]; then
                  # Fallback: any pod whose name contains the sandbox name
                  pod_match=$(awk -v n="$name" "index(\$2,n)>0 {print \$1\"/\"\$2; exit}" docker-logs/_pods.txt)
                fi
                if [ -z "$pod_match" ]; then continue; fi
                pod_ns="${pod_match%%/*}"
                pod_name="${pod_match##*/}"
                docker exec openshell-cluster-nemoclaw kubectl exec -n "$pod_ns" "$pod_name" -- bash -c "
                  for f in /sandbox/.openclaw/logs/gateway-persistent.log /tmp/gateway.log /tmp/openclaw-*/openclaw-*.log; do
                    [ -f \"\$f\" ] || continue
                    printf \"\\n----- %s (size=%s) -----\\n\" \"\$f\" \"\$(stat -c%s \"\$f\" 2>/dev/null || echo ?)\"
                    cat -- \"\$f\" 2>/dev/null
                  done
                " > "docker-logs/sandbox-${name}.log" 2>&1
              done
            done
          ' >/dev/null 2>&1 &
          echo $! > /tmp/gateway-log-streamer.pid

      - name: Run sandbox operations E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_POLICY_TIER: "open"
          GITHUB_TOKEN: ${{ github.token }}
          # Override the 1800s default in test/e2e/e2e-timeout.sh. Sandbox
          # creation alone is ~14 min per sandbox in current CI conditions
          # (build+upload to k3s gateway), and the test creates two — leaving
          # the default 30-min budget completely consumed by setup with no
          # room for the actual TC-SBX cases. The job-level timeout (60 min,
          # set in `timeout-minutes` above) is the real upper bound.
          NEMOCLAW_E2E_TIMEOUT_SECONDS: "2700"
        run: bash test/e2e/test-sandbox-operations.sh

      - name: Stop gateway log streamer
        if: always()
        # Diagnostic step: never let `bash -e` kill the snapshot loop on a
        # single command failure (openshell ssh-config, nemoclaw logs, etc.
        # all routinely fail post-test depending on TC-SBX-06's docker-kill
        # state). We log the failures inline and continue.
        shell: bash --noprofile --norc -uo pipefail {0}
        run: |
          [ -f /tmp/gateway-log-streamer.pid ] && kill "$(cat /tmp/gateway-log-streamer.pid)" 2>/dev/null || true
          # Kill any per-sandbox SSH+tail followers spawned by the streamer.
          pkill -f 'tail -n \+1 -F /tmp/gateway.log' 2>/dev/null || true
          pkill -f 'ssh.*openshell-' 2>/dev/null || true
          sleep 2
          # Final snapshot: tail -F glob expands once at start, so log files
          # for openclaw processes that ran as a different UID (creating new
          # /tmp/openclaw-<uid>/ dirs mid-test) get missed. Re-glob now and
          # append every openclaw log file from each live sandbox to the
          # per-sandbox docker-logs file.
          #
          # Use `nemoclaw <name> logs` (not raw openshell ssh-config + ssh)
          # because nemoclaw handles SSH key/host setup and is robust to
          # streamer race conditions. Tested working in TC-SBX-04.
          export PATH="$HOME/.local/bin:$PATH"
          echo "=== final-snapshot: PATH=$PATH"
          echo "=== final-snapshot: nemoclaw=$(command -v nemoclaw)"
          echo "=== final-snapshot: openshell=$(command -v openshell)"
          # TC-SBX-06's docker kill of the gateway pod can leave openshell
          # without an active gateway selected; re-select before the snapshot
          # so `nemoclaw <name> logs` and direct `openshell sandbox exec` both
          # have a target. The select is best-effort — failure (e.g., gateway
          # not yet recovered) just means we fall through to ssh-config-based
          # capture below.
          openshell gateway select nemoclaw 2>&1 | head -5 || true
          openshell gateway list 2>&1 | head -10 || true
          # NEW PATH: bypass the openshell client entirely. The
          # openshell-cluster-nemoclaw docker container runs k3s with
          # kubectl available inside. Even after TC-SBX-06's docker-kill,
          # docker auto-restarts the container and k3s state survives via
          # /var/lib/rancher/k3s. Use `docker exec ... kubectl` to read
          # the persistent log directly from each sandbox pod, with no
          # dependency on the host's openshell metadata.
          echo "=== final-snapshot: docker containers:"
          docker ps --format '{{.Names}}\t{{.Status}}' 2>&1 | head -10
          echo "=== final-snapshot: cluster pods:"
          docker exec openshell-cluster-nemoclaw kubectl get pods -A --no-headers 2>&1 | head -20
          if [ -f "$HOME/.nemoclaw/sandboxes.json" ]; then
            echo "=== final-snapshot: sandboxes.json contents:"
            cat "$HOME/.nemoclaw/sandboxes.json" 2>&1 | head -30
            registry_keys=$(jq -r ".sandboxes // {} | keys[]?" "$HOME/.nemoclaw/sandboxes.json" 2>&1)
            echo "=== final-snapshot: sandbox names from jq: '$registry_keys'"
            for name in $registry_keys; do
              case "$name" in *[!a-z0-9_-]*|"") echo "=== final-snapshot: skipping invalid name '$name'"; continue ;; esac
              echo "=== final-snapshot: capturing logs for '$name'"
              {
                printf '\n\n===== FINAL SNAPSHOT: %s =====\n' "$name"
                # FIRST attempt: docker exec into the cluster container and
                # kubectl-exec into the sandbox pod. This works even when
                # the host openshell client is broken post-TC-SBX-06 because
                # docker (and k3s inside the cluster) survive the gateway
                # docker-kill via auto-restart + persistent k3s state.
                pod_ns_name=$(docker exec openshell-cluster-nemoclaw kubectl get pods -A --no-headers 2>/dev/null | awk -v n="$name" '$2==n {print $1"/"$2; exit}')
                if [ -n "$pod_ns_name" ]; then
                  echo "(found pod $pod_ns_name for $name)"
                  pod_ns="${pod_ns_name%%/*}"
                  pod_name="${pod_ns_name##*/}"
                  k_out=$(mktemp)
                  docker exec openshell-cluster-nemoclaw kubectl exec -n "$pod_ns" "$pod_name" -- bash -c '
                    for f in /sandbox/.openclaw/logs/gateway-persistent.log /tmp/gateway.log /tmp/openclaw-*/openclaw-*.log; do
                      [ -f "$f" ] || continue
                      printf "\n----- %s (size=%s) -----\n" "$f" "$(stat -c%s "$f" 2>/dev/null || echo ?)"
                      cat -- "$f" 2>/dev/null || true
                    done
                  ' >"$k_out" 2>&1
                  k_rc=$?
                  echo "(kubectl exec rc=$k_rc size=$(wc -c <"$k_out"))"
                  tail -c 500000 "$k_out"
                  rm -f "$k_out"
                else
                  echo "(no kubectl pod found matching '$name')"
                fi
                # Existing fallbacks (raw ssh + nemoclaw logs) preserved
                # below in case the docker/kubectl path also fails — they
                # provide complementary coverage during transient states.
                ssh_cfg="/tmp/sshcfg-final-${name}.tmp"
                if openshell sandbox ssh-config "$name" >"$ssh_cfg" 2>&1 && [ -s "$ssh_cfg" ]; then
                  ssh_out=$(mktemp)
                  ssh -F "$ssh_cfg" \
                      -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
                      -o ConnectTimeout=10 -o LogLevel=ERROR \
                      "openshell-${name}" \
                      'for f in /sandbox/.openclaw/logs/gateway-persistent.log \
                               /tmp/gateway.log \
                               /tmp/openclaw-*/openclaw-*.log; do
                         [ -f "$f" ] || continue
                         printf "\n----- %s (size=%s) -----\n" "$f" "$(stat -c%s "$f" 2>/dev/null || echo ?)"
                         cat -- "$f" 2>/dev/null || true
                       done' >"$ssh_out" 2>&1
                  ssh_rc=$?
                  tail -c 500000 "$ssh_out"
                  rm -f "$ssh_out"
                  [ "$ssh_rc" -eq 0 ] || echo "(direct ssh exited rc=$ssh_rc)"
                else
                  echo "(openshell sandbox ssh-config failed for $name)"
                  # Fallback to nemoclaw logs (less reliable, but try anything)
                  if command -v nemoclaw >/dev/null 2>&1; then
                    nm_out=$(mktemp)
                    nemoclaw "$name" logs >"$nm_out" 2>&1
                    echo "(nemoclaw logs rc=$? size=$(wc -c <"$nm_out"))"
                    tail -c 500000 "$nm_out"
                    rm -f "$nm_out"
                  fi
                fi
                rm -f "$ssh_cfg"
              } >> "docker-logs/sandbox-${name}.log"
            done
          else
            echo "=== final-snapshot: sandboxes.json not found at $HOME/.nemoclaw/sandboxes.json"
          fi
          # Cap each log file at 5MB by keeping only the last 5MB — useful
          # content (real gateway events) is mixed throughout, so tail-trim
          # is fine for diagnostic purposes.
          for f in docker-logs/*.log; do
            [ -f "$f" ] || continue
            sz=$(stat -c%s "$f" 2>/dev/null || stat -f%z "$f" 2>/dev/null || echo 0)
            if [ "$sz" -gt 5242880 ]; then
              tail -c 5242880 "$f" > "${f}.tail" && mv "${f}.tail" "$f"
            fi
          done
          ls -la docker-logs/ 2>&1 | head -20 || true
          du -sh docker-logs/ 2>&1 || true

      - name: Upload sandbox gateway logs on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: sandbox-operations-docker-logs
          path: docker-logs/
          if-no-files-found: ignore

      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: sandbox-operations-test-log
          path: test-sandbox-operations-*.log
          if-no-files-found: ignore

  # ── Inference routing (credential isolation + error classification) ──
  # TC-INF-05: real API key absent from sandbox env/process/filesystem
  # TC-INF-06: invalid API key → classified credential error (PR-safe)
  # TC-INF-07: unreachable endpoint → classified transport error (PR-safe)
  inference-routing-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',inference-routing-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-inference-routing.sh
      timeout_minutes: 30
      artifact_name: "inference-routing-test-log"
      artifact_path: "test-inference-routing-*.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"open"}'
      nvidia_api_key: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  openclaw-inference-switch-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',openclaw-inference-switch-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-openclaw-inference-switch.sh
      artifact_name: "openclaw-inference-switch-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-openclaw-inference-switch-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-openclaw-inference-switch"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  network-policy-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',network-policy-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-network-policy.sh
      artifact_name: "network-policy-test-log"
      artifact_path: "test-network-policy-*.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_POLICY_TIER":"restricted","NEMOCLAW_RECREATE_SANDBOX":"1"}'
      nvidia_api_key: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  state-backup-restore-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',state-backup-restore-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-state-backup-restore.sh
      timeout_minutes: 60
      artifact_name: "state-backup-restore-test-log"
      artifact_path: "test-state-backup-restore-*.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1"}'
      nvidia_api_key: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  tunnel-lifecycle-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',tunnel-lifecycle-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-tunnel-lifecycle.sh
      timeout_minutes: 60
      artifact_name: "tunnel-lifecycle-test-log"
      artifact_path: "test-tunnel-lifecycle-*.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1"}'
      nvidia_api_key: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  diagnostics-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',diagnostics-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-diagnostics.sh
      artifact_name: "diagnostics-test-log"
      artifact_path: "test-diagnostics-*.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1"}'
      nvidia_api_key: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  credential-migration-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',credential-migration-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-credential-migration.sh
      timeout_minutes: 30
      artifact_name: "install-log-credential-migration"
      artifact_path: "/tmp/nemoclaw-e2e-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-cred-migration"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  snapshot-commands-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',snapshot-commands-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-snapshot-commands.sh
      timeout_minutes: 30
      artifact_name: "snapshot-commands-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-snapshot"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  shields-config-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',shields-config-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-shields-config.sh
      timeout_minutes: 30
      artifact_name: "shields-config-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-shields-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-shields"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  vm-driver-privileged-exec-routing-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',vm-driver-privileged-exec-routing-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-vm-driver-privileged-exec-routing.sh
      timeout_minutes: 10
      artifact_name: "vm-driver-privileged-exec-routing-log"
      artifact_path: "/tmp/nemoclaw-vm-driver-privileged-exec-routing-build.log"
      env_json: '{"NEMOCLAW_NON_INTERACTIVE":"1"}'
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  rebuild-openclaw-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',rebuild-openclaw-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-rebuild-openclaw.sh
      timeout_minutes: 60
      artifact_name: "rebuild-openclaw-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-rebuild-oc"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  upgrade-stale-sandbox-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-upgrade-stale-sandbox.sh
      timeout_minutes: 60
      artifact_name: "upgrade-stale-sandbox-logs"
      artifact_path: |
        /tmp/nemoclaw-e2e-old-install.log
        /tmp/nemoclaw-e2e-upgrade-install.log
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-upgrade-stale"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  openshell-gateway-upgrade-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',openshell-gateway-upgrade-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Setup Node
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
        with:
          node-version: "22"

      - name: Run OpenShell gateway upgrade E2E test
        env:
          GITHUB_TOKEN: ${{ github.token }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: bash test/e2e/test-openshell-gateway-upgrade.sh

      - name: Upload gateway upgrade logs on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: openshell-gateway-upgrade-logs
          path: |
            /tmp/nemoclaw-e2e-openshell-gateway-upgrade.log
            /tmp/nemoclaw-e2e-openshell-gateway-install.log
            /tmp/nemoclaw-e2e-openshell-gateway-old-install.log
            /tmp/nemoclaw-e2e-openshell-gateway-current-install.log
            /tmp/nemoclaw-e2e-openshell-gateway-start.log
            /tmp/nemoclaw-e2e-openshell-gateway-process.log
            /tmp/nemoclaw-e2e-openshell-gateway-compatible-mock.log
          if-no-files-found: ignore

  # ── Hermes rebuild upgrade E2E ──────────────────────────────
  # Same upgrade scenario as OpenClaw but for Hermes Agent.
  rebuild-hermes-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',rebuild-hermes-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-rebuild-hermes.sh
      timeout_minutes: 60
      artifact_name: "rebuild-hermes-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-rebuild-hm"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  rebuild-hermes-stale-base-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',rebuild-hermes-stale-base-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-rebuild-hermes.sh
      timeout_minutes: 60
      artifact_name: "rebuild-hermes-stale-base-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_AGENT":"hermes","NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-rebuild-hm-base"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  double-onboard-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',double-onboard-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}
          persist-credentials: false
      - name: Install NemoClaw
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: bash install.sh --non-interactive --yes-i-accept-third-party-software
      - name: Run double onboard E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: |
          [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true
          export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
          [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
          [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH"
          bash test/e2e/test-double-onboard.sh
      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: double-onboard-test-log
          path: test-double-onboard-*.log
          if-no-files-found: ignore

  # ── Onboard Repair E2E ─────────────────────────────────────
  onboard-repair-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',onboard-repair-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}
          persist-credentials: false
      - name: Install NemoClaw
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: bash install.sh --non-interactive --yes-i-accept-third-party-software
      - name: Run onboard repair E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: |
          [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true
          export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
          [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
          [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH"
          bash test/e2e/test-onboard-repair.sh
      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: onboard-repair-test-log
          path: test-onboard-repair-*.log
          if-no-files-found: ignore

  # ── Onboard Resume E2E ─────────────────────────────────────
  onboard-resume-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',onboard-resume-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}
          persist-credentials: false
      - name: Install NemoClaw
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: bash install.sh --non-interactive --yes-i-accept-third-party-software
      - name: Run onboard resume E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: |
          [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true
          export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
          [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
          [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH"
          bash test/e2e/test-onboard-resume.sh
      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: onboard-resume-test-log
          path: test-onboard-resume-*.log
          if-no-files-found: ignore

  # -- Onboard Negative Paths E2E -------------------------------
  onboard-negative-paths-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',onboard-negative-paths-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 75
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}
          persist-credentials: false
      - name: Install NemoClaw
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: bash install.sh --non-interactive --yes-i-accept-third-party-software
      - name: Run onboard negative-path E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: |
          set -euo pipefail
          [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true
          export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
          [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
          [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH"
          bash test/e2e/test-onboard-negative-paths.sh
      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: onboard-negative-paths-test-log
          path: /tmp/nemoclaw-e2e-onboard-negative-paths.log
          if-no-files-found: ignore

  # ── Runtime Overrides E2E ──────────────────────────────────
  runtime-overrides-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',runtime-overrides-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}
          persist-credentials: false
      - name: Install NemoClaw
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: bash install.sh --non-interactive --yes-i-accept-third-party-software
      - name: Run runtime overrides E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
        run: |
          [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true
          export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
          [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
          [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH"
          bash test/e2e/test-runtime-overrides.sh
      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: runtime-overrides-test-log
          path: test-runtime-overrides-*.log
          if-no-files-found: ignore

  # ── Credential Sanitization E2E ────────────────────────────
  # Requires a running sandbox. Bootstraps via install.sh then runs tests.
  credential-sanitization-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',credential-sanitization-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}
      - name: Install NemoClaw and onboard sandbox
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_SANDBOX_NAME: "e2e-test"
        run: bash install.sh --non-interactive --yes-i-accept-third-party-software
      - name: Run credential sanitization E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_SANDBOX_NAME: "e2e-test"
        run: |
          # shellcheck source=/dev/null
          [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true
          export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
          [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
          [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH"
          bash test/e2e/test-credential-sanitization.sh
      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: credential-sanitization-test-log
          path: test-credential-sanitization-*.log
          if-no-files-found: ignore

  # ── Telegram Injection E2E ─────────────────────────────────
  # Requires a running sandbox. Bootstraps via install.sh then runs tests.
  telegram-injection-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',telegram-injection-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}
      - name: Install NemoClaw and onboard sandbox
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_SANDBOX_NAME: "e2e-test"
        run: bash install.sh --non-interactive --yes-i-accept-third-party-software
      - name: Run telegram injection E2E test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_SANDBOX_NAME: "e2e-test"
        run: |
          # shellcheck source=/dev/null
          [ -f "$HOME/.bashrc" ] && source "$HOME/.bashrc" 2>/dev/null || true
          export NVM_DIR="${NVM_DIR:-$HOME/.nvm}"
          [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
          [ -d "$HOME/.local/bin" ] && [[ ":$PATH:" != *":$HOME/.local/bin:"* ]] && export PATH="$HOME/.local/bin:$PATH"
          bash test/e2e/test-telegram-injection.sh
      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: telegram-injection-test-log
          path: test-telegram-injection-*.log
          if-no-files-found: ignore

  # Remove this job — and the matching notify-on-failure entry — in the
  # same PR that deletes cluster-image-patch.ts when the OpenShell
  # roadmap migration off k3s (NVIDIA/OpenShell#873) lands.
  # ── Docker 26+ overlayfs nested-mount auto-fix (#2481) ──────
  # TEMPORARY: validates the auto-fix in src/lib/cluster-image-patch.ts.
  overlayfs-autofix-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',overlayfs-autofix-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-overlayfs-autofix.sh
      artifact_name: "overlayfs-autofix-logs"
      artifact_path: |
        /tmp/nemoclaw-e2e-install.log
        /tmp/nemoclaw-e2e-onboard-positive.log
        /tmp/nemoclaw-e2e-onboard-negative.log
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_SANDBOX_NAME":"e2e-overlayfs"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  device-auth-health-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' && (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',device-auth-health-e2e,'))
    uses: ./.github/workflows/e2e-script.yaml
    with:
      ref: ${{ inputs.target_ref || github.ref }}
      script: test/e2e/test-device-auth-health.sh
      timeout_minutes: 30
      artifact_name: "device-auth-health-install-log"
      artifact_path: "/tmp/nemoclaw-e2e-health-install.log"
      env_json: '{"NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE":"1","NEMOCLAW_NON_INTERACTIVE":"1","NEMOCLAW_RECREATE_SANDBOX":"1","NEMOCLAW_SANDBOX_NAME":"e2e-health-auth"}'
      nvidia_api_key: true
      github_token: true
    secrets:
      NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
      BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
  launchable-smoke-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',launchable-smoke-e2e,'))
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Run launchable install-flow smoke test
        env:
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          NEMOCLAW_NON_INTERACTIVE: "1"
          NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
          NEMOCLAW_SANDBOX_NAME: "e2e-launchable"
          NEMOCLAW_RECREATE_SANDBOX: "1"
          SKIP_DOCKER_PULL: "1"
          GITHUB_TOKEN: ${{ github.token }}
        run: bash test/e2e/test-launchable-smoke.sh

      - name: Upload install log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: launchable-smoke-install-log
          path: /tmp/nemoclaw-launchable-install.log
          if-no-files-found: ignore

      - name: Upload onboard log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: launchable-smoke-onboard-log
          path: /tmp/nemoclaw-launchable-onboard.log
          if-no-files-found: ignore

      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: launchable-smoke-test-log
          path: /tmp/nemoclaw-launchable-test.log
          if-no-files-found: ignore

  # ── GPU E2E (Ollama local inference) ──────────────────────────
  # Runs on an NVKS ephemeral GPU runner (RTX Pro 6000, 36 GB VRAM).
  # Each job gets a fresh VM — no state leakage between runs.
  gpu-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      vars.GPU_E2E_ENABLED == 'true' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',gpu-e2e,'))
    runs-on: linux-amd64-gpu-rtxpro6000-latest-1
    timeout-minutes: 30
    env:
      NEMOCLAW_NON_INTERACTIVE: "1"
      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
      NEMOCLAW_SANDBOX_NAME: "e2e-gpu-ollama"
      NEMOCLAW_RECREATE_SANDBOX: "1"
      NEMOCLAW_PROVIDER: "ollama"
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Verify GPU availability
        run: |
          echo "=== GPU Info ==="
          nvidia-smi
          echo ""
          echo "=== VRAM ==="
          nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
          echo ""
          echo "=== Docker ==="
          docker info --format '{{.ServerVersion}}'

      - name: Run GPU E2E test (Ollama local inference)
        run: bash test/e2e/test-gpu-e2e.sh

      - name: Upload install log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: gpu-e2e-install-log
          path: /tmp/nemoclaw-gpu-e2e-install.log
          if-no-files-found: ignore

      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: gpu-e2e-test-log
          path: /tmp/nemoclaw-gpu-e2e-test.log
          if-no-files-found: ignore

  # ── GPU Double-Onboard E2E (Ollama token consistency) ────────
  # Reproduces issue #2553: re-onboard with Ollama must not leave the
  # proxy running with a different token than what's persisted to disk.
  # Runs on its own ephemeral VM — no dependency on gpu-e2e.
  gpu-double-onboard-e2e:
    if: >-
      github.repository == 'NVIDIA/NemoClaw' &&
      vars.GPU_E2E_ENABLED == 'true' &&
      (github.event_name != 'workflow_dispatch' ||
       inputs.jobs == '' ||
       contains(format(',{0},', inputs.jobs), ',gpu-double-onboard-e2e,'))
    runs-on: linux-amd64-gpu-rtxpro6000-latest-1
    timeout-minutes: 30
    env:
      NEMOCLAW_NON_INTERACTIVE: "1"
      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
      NEMOCLAW_SANDBOX_NAME: "e2e-gpu-double-onboard"
      NEMOCLAW_RECREATE_SANDBOX: "1"
      NEMOCLAW_PROVIDER: "ollama"
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.target_ref || github.ref }}

      - name: Verify GPU availability
        run: |
          echo "=== GPU Info ==="
          nvidia-smi
          echo ""
          echo "=== VRAM ==="
          nvidia-smi --query-gpu=name,memory.total --format=csv,noheader
          echo ""
          echo "=== Docker ==="
          docker info --format '{{.ServerVersion}}'

      - name: Run GPU double-onboard E2E test
        run: bash test/e2e/test-gpu-double-onboard.sh

      - name: Upload install log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: gpu-double-onboard-install-log
          path: /tmp/nemoclaw-gpu-double-onboard-install.log
          if-no-files-found: ignore

      - name: Upload re-onboard log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: gpu-double-onboard-reonboard-log
          path: /tmp/nemoclaw-gpu-double-onboard-reonboard.log
          if-no-files-found: ignore

      - name: Upload test log on failure
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: gpu-double-onboard-test-log
          path: /tmp/nemoclaw-gpu-double-onboard-test.log
          if-no-files-found: ignore

  notify-on-failure:
    runs-on: ubuntu-latest
    needs:
      [
        cloud-e2e,
        cloud-onboard-e2e,
        cloud-inference-e2e,
        skill-agent-e2e,
        docs-validation-e2e,
        messaging-providers-e2e,
        openclaw-slack-pairing-e2e,
        openclaw-tui-chat-correlation-e2e,
        issue-3600-gpu-proof-optional-e2e,
        openclaw-discord-pairing-e2e,
        issue-4462-scope-upgrade-approval-e2e,
        issue-4462-gateway-pinned-approval-characterization-e2e,
        messaging-compatible-endpoint-e2e,
        channels-add-remove-e2e,
        channels-stop-start-e2e,
        brave-search-e2e,
        kimi-inference-compat-e2e,
        bedrock-runtime-compatible-anthropic-e2e,
        token-rotation-e2e,
        sandbox-survival-e2e,
        issue-2478-crash-loop-recovery-e2e,
        hermes-e2e,
        hermes-dashboard-e2e,
        hermes-root-entrypoint-smoke-e2e,
        openclaw-onboard-security-posture-e2e,
        hermes-onboard-security-posture-e2e,
        hermes-inference-switch-e2e,
        hermes-discord-e2e,
        hermes-slack-e2e,
        sandbox-operations-e2e,
        inference-routing-e2e,
        openclaw-inference-switch-e2e,
        network-policy-e2e,
        state-backup-restore-e2e,
        tunnel-lifecycle-e2e,
        diagnostics-e2e,
        credential-migration-e2e,
        snapshot-commands-e2e,
        shields-config-e2e,
        vm-driver-privileged-exec-routing-e2e,
        rebuild-openclaw-e2e,
        upgrade-stale-sandbox-e2e,
        openshell-gateway-upgrade-e2e,
        rebuild-hermes-e2e,
        rebuild-hermes-stale-base-e2e,
        double-onboard-e2e,
        onboard-repair-e2e,
        onboard-resume-e2e,
        onboard-negative-paths-e2e,
        runtime-overrides-e2e,
        credential-sanitization-e2e,
        telegram-injection-e2e,
        overlayfs-autofix-e2e,
        device-auth-health-e2e,
        launchable-smoke-e2e,
        gpu-e2e,
        gpu-double-onboard-e2e,
      ]
    if: ${{ always() && github.event_name == 'schedule' && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }}
    permissions:
      issues: write
    steps:
      - name: Create or update failure issue
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const title = 'Nightly E2E failed';

            const needs = ${{ toJSON(needs) }};
            const failed = Object.entries(needs).filter(([, v]) => v.result === 'failure').map(([k]) => k);
            const cancelled = Object.entries(needs).filter(([, v]) => v.result === 'cancelled').map(([k]) => k);
            const summary = [
              failed.length ? `**Failed:** ${failed.join(', ')}` : '',
              cancelled.length ? `**Cancelled:** ${cancelled.join(', ')}` : '',
            ].filter(Boolean).join('\n');

            const { data: existing } = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: 'open',
              labels: 'CI/CD',
              per_page: 100,
            });
            const match = existing.find(i => !i.pull_request && i.title.startsWith(title));

            if (match) {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: match.number,
                body: `Failed again on ${new Date().toISOString().split('T')[0]}.\n\n**Run:** ${runUrl}\n${summary}\n**Artifacts:** Check the run artifacts for install/test logs (artifact names vary by job).`,
              });
            } else {
              await github.rest.issues.create({
                owner: context.repo.owner,
                repo: context.repo.repo,
                title: `${title} — ${new Date().toISOString().split('T')[0]}`,
                body: `The nightly E2E pipeline failed.\n\n**Run:** ${runUrl}\n${summary}\n**Artifacts:** Check the run artifacts for install/test logs (artifact names vary by job).`,
                labels: ['bug', 'CI/CD'],
              });
            }

  report-to-pr:
    runs-on: ubuntu-latest
    needs:
      [
        cloud-e2e,
        cloud-onboard-e2e,
        cloud-inference-e2e,
        skill-agent-e2e,
        docs-validation-e2e,
        messaging-providers-e2e,
        openclaw-slack-pairing-e2e,
        openclaw-tui-chat-correlation-e2e,
        issue-3600-gpu-proof-optional-e2e,
        openclaw-discord-pairing-e2e,
        issue-4462-scope-upgrade-approval-e2e,
        issue-4462-gateway-pinned-approval-characterization-e2e,
        messaging-compatible-endpoint-e2e,
        channels-add-remove-e2e,
        channels-stop-start-e2e,
        brave-search-e2e,
        kimi-inference-compat-e2e,
        bedrock-runtime-compatible-anthropic-e2e,
        token-rotation-e2e,
        sandbox-survival-e2e,
        issue-2478-crash-loop-recovery-e2e,
        hermes-e2e,
        hermes-dashboard-e2e,
        hermes-root-entrypoint-smoke-e2e,
        openclaw-onboard-security-posture-e2e,
        hermes-onboard-security-posture-e2e,
        hermes-inference-switch-e2e,
        hermes-discord-e2e,
        hermes-slack-e2e,
        sandbox-operations-e2e,
        inference-routing-e2e,
        openclaw-inference-switch-e2e,
        network-policy-e2e,
        state-backup-restore-e2e,
        tunnel-lifecycle-e2e,
        diagnostics-e2e,
        credential-migration-e2e,
        snapshot-commands-e2e,
        shields-config-e2e,
        vm-driver-privileged-exec-routing-e2e,
        rebuild-openclaw-e2e,
        upgrade-stale-sandbox-e2e,
        openshell-gateway-upgrade-e2e,
        rebuild-hermes-e2e,
        rebuild-hermes-stale-base-e2e,
        double-onboard-e2e,
        onboard-repair-e2e,
        onboard-resume-e2e,
        onboard-negative-paths-e2e,
        runtime-overrides-e2e,
        credential-sanitization-e2e,
        telegram-injection-e2e,
        overlayfs-autofix-e2e,
        device-auth-health-e2e,
        launchable-smoke-e2e,
        gpu-e2e,
        gpu-double-onboard-e2e,
      ]
    if: ${{ always() && github.event_name == 'workflow_dispatch' }}
    permissions:
      issues: write
      pull-requests: write
    steps:
      - name: Post E2E results to PR
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const needs = ${{ toJSON(needs) }};
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const workflowBranch = context.ref.replace('refs/heads/', '');
            const targetRef = ${{ toJSON(inputs.target_ref) }} || '';
            const prNumberInput = ${{ toJSON(inputs.pr_number) }} || '';
            const displayRef = targetRef || workflowBranch;
            const requestedJobs = ${{ toJSON(inputs.jobs) }} || "";

            let prNumber = prNumberInput ? Number.parseInt(prNumberInput, 10) : undefined;
            if (!prNumber) {
              // Find open PR for this branch. This is the legacy manual-dispatch
              // path where the workflow itself is dispatched on the PR branch.
              const { data: prs } = await github.rest.pulls.list({
                owner: context.repo.owner,
                repo: context.repo.repo,
                head: `${context.repo.owner}:${workflowBranch}`,
                state: 'open',
              });

              if (prs.length === 0) {
                core.info(`No open PR found for branch ${workflowBranch} — skipping comment.`);
                return;
              }

              prNumber = prs[0].number;
            }

            const requested = requestedJobs
              .split(',')
              .map((job) => job.trim())
              .filter(Boolean);
            const requestedSet = new Set(requested);

            // Build results table. For selective dispatches, report only the
            // requested jobs; otherwise the comment is dominated by expected skips.
            const emoji = { success: '✅', failure: '❌', cancelled: '⚠️', skipped: '⏭️' };
            const allEntries = Object.entries(needs).sort(([a], [b]) => a.localeCompare(b));
            const missingRequested = requested.filter((job) => !(job in needs));
            const reportedEntries = requested.length
              ? allEntries.filter(([name]) => requestedSet.has(name))
              : allEntries;
            const rows = reportedEntries
              .sort(([a], [b]) => a.localeCompare(b))
              .map(([name, { result }]) => `| ${name} | ${emoji[result] || '❓'} ${result} |`);
            for (const name of missingRequested) {
              rows.push(`| ${name} | ❓ not reported |`);
            }

            const ran = reportedEntries.filter(([, v]) => v.result !== 'skipped');
            const passed = ran.filter(([, v]) => v.result === 'success');
            const failed = ran.filter(([, v]) => v.result === 'failure');
            const skipped = reportedEntries.filter(([, v]) => v.result === 'skipped');

            const status =
              failed.length > 0 || missingRequested.length > 0
                ? '❌ Some jobs failed'
                : skipped.length > 0 && passed.length === 0
                  ? '⚠️ No requested jobs ran'
                  : '✅ All requested jobs passed';

            const body = [
              `### Selective E2E Results — ${status}`,
              '',
              `**Run:** [${context.runId}](${runUrl})`,
              `**Target ref:** \`${displayRef}\``,
              targetRef ? `**Workflow ref:** \`${workflowBranch}\`` : undefined,
              requestedJobs ? `**Requested jobs:** \`${requestedJobs}\`` : '**Requested jobs:** all (no filter)',
              `**Summary:** ${passed.length} passed, ${failed.length} failed, ${skipped.length} skipped`,
              '',
              '| Job | Result |',
              '|-----|--------|',
              ...rows,
              '',
              failed.length > 0
                ? `> **Failed jobs:** ${failed.map(([k]) => k).join(', ')}. Check [run artifacts](${runUrl}) for logs.`
                : '',
              missingRequested.length > 0
                ? `> **Missing requested jobs:** ${missingRequested.join(', ')}. The reporting workflow needs to include these jobs.`
                : '',
            ].filter((line) => line !== undefined).join('\n');

            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: prNumber,
              body,
            });

  # ── Nightly Scorecard ──────────────────────────────────────────────────
  # Aggregates overnight results into a scorecard published to
  # $GITHUB_STEP_SUMMARY. Identifies flaky jobs, computes pass/fail/cancel
  # breakdowns, and compares trends against the prior day.
  # Only runs on schedule (not workflow_dispatch — that uses report-to-pr).
  scorecard:
    runs-on: ubuntu-latest
    needs:
      [
        cloud-e2e,
        cloud-onboard-e2e,
        cloud-inference-e2e,
        skill-agent-e2e,
        docs-validation-e2e,
        messaging-providers-e2e,
        openclaw-slack-pairing-e2e,
        openclaw-tui-chat-correlation-e2e,
        issue-3600-gpu-proof-optional-e2e,
        openclaw-discord-pairing-e2e,
        issue-4462-scope-upgrade-approval-e2e,
        issue-4462-gateway-pinned-approval-characterization-e2e,
        messaging-compatible-endpoint-e2e,
        channels-add-remove-e2e,
        channels-stop-start-e2e,
        brave-search-e2e,
        kimi-inference-compat-e2e,
        bedrock-runtime-compatible-anthropic-e2e,
        token-rotation-e2e,
        sandbox-survival-e2e,
        issue-2478-crash-loop-recovery-e2e,
        hermes-e2e,
        hermes-dashboard-e2e,
        hermes-root-entrypoint-smoke-e2e,
        openclaw-onboard-security-posture-e2e,
        hermes-onboard-security-posture-e2e,
        hermes-inference-switch-e2e,
        hermes-discord-e2e,
        hermes-slack-e2e,
        sandbox-operations-e2e,
        inference-routing-e2e,
        openclaw-inference-switch-e2e,
        network-policy-e2e,
        state-backup-restore-e2e,
        tunnel-lifecycle-e2e,
        diagnostics-e2e,
        credential-migration-e2e,
        snapshot-commands-e2e,
        shields-config-e2e,
        vm-driver-privileged-exec-routing-e2e,
        rebuild-openclaw-e2e,
        upgrade-stale-sandbox-e2e,
        openshell-gateway-upgrade-e2e,
        rebuild-hermes-e2e,
        rebuild-hermes-stale-base-e2e,
        double-onboard-e2e,
        onboard-repair-e2e,
        onboard-resume-e2e,
        onboard-negative-paths-e2e,
        runtime-overrides-e2e,
        credential-sanitization-e2e,
        telegram-injection-e2e,
        overlayfs-autofix-e2e,
        device-auth-health-e2e,
        launchable-smoke-e2e,
        gpu-e2e,
        gpu-double-onboard-e2e,
      ]
    if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
    permissions:
      actions: read
    steps:
      - name: Generate nightly scorecard
        id: scorecard
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            // ── Config ──────────────────────────────────────────────
            const EXCLUDED_JOBS = new Set(['gpu-e2e', 'notify-on-failure', 'report-to-pr', 'scorecard']);

            // ── Helpers ─────────────────────────────────────────────
            function formatDate(date) {
              return date.toLocaleDateString('en-US', { month: 'short', day: 'numeric' });
            }

            // ── Gather results from the current run's needs context ─
            const needs = ${{ toJSON(needs) }};
            const today = formatDate(new Date());
            const isDispatch = context.eventName === 'workflow_dispatch';
            const requestedJobsRaw = isDispatch ? '${{ inputs.jobs }}'.trim() : '';
            const requestedJobs = requestedJobsRaw
              ? requestedJobsRaw.split(',').map((name) => name.trim()).filter(Boolean)
              : [];
            const isSelectiveDispatch = isDispatch && requestedJobs.length > 0;
            const runMode = isSelectiveDispatch
              ? 'Selective dispatch'
              : isDispatch
                ? 'Manual full run'
                : 'Scheduled full nightly';

            const entries = Object.entries(needs).filter(([name]) => !EXCLUDED_JOBS.has(name));
            let success = 0;
            let failure = 0;
            let cancelled = 0;
            let skipped = 0;

            for (const [, { result }] of entries) {
              if (result === 'success') success++;
              else if (result === 'failure') failure++;
              else if (result === 'cancelled') cancelled++;
              else if (result === 'skipped') skipped++;
            }

            const total = entries.length;
            const ran = total - skipped;
            const perfect = failure === 0 && cancelled === 0 && ran > 0;

            // ── Identify failed jobs ────────────────────────────────
            const failedJobs = entries
              .filter(([, { result }]) => result === 'failure')
              .map(([name]) => name)
              .sort();

            // ── Fetch prior-day run for trend comparison ────────────
            let trendLine = '';
            if (isSelectiveDispatch) {
              trendLine = 'Trend: ⊘ Not shown for selective dispatches';
            } else {
              try {
                const WORKFLOW_FILE = 'nightly-e2e.yaml';
                const now = new Date();
                const since48h = new Date(now.getTime() - 48 * 60 * 60 * 1000).toISOString();
                const since24h = new Date(now.getTime() - 24 * 60 * 60 * 1000).toISOString();

                const priorRuns = [];
                for (let page = 1; page <= 10 && priorRuns.length === 0; page++) {
                  const { data } = await github.rest.actions.listWorkflowRuns({
                    owner: context.repo.owner,
                    repo: context.repo.repo,
                    workflow_id: WORKFLOW_FILE,
                    created: `>=${since48h}`,
                    per_page: 100,
                    page,
                  });

                  priorRuns.push(
                    ...data.workflow_runs.filter(r =>
                      r.status === 'completed' &&
                      r.event === 'schedule' &&
                      new Date(r.created_at) < new Date(since24h)
                    )
                  );

                  if (data.workflow_runs.length < 100) break;
                }

                if (priorRuns.length > 0) {
                  // Check the most recent prior run
                  const priorRun = priorRuns[0];
                  const priorPerfect = priorRun.conclusion === 'success';
                  if (perfect && priorPerfect) {
                    trendLine = 'Trend: ➡️ Stable (perfect both days)';
                  } else if (perfect && !priorPerfect) {
                    trendLine = 'Trend: ↗️ Improving (yesterday had failures → today perfect)';
                  } else if (!perfect && priorPerfect) {
                    trendLine = 'Trend: ↘️ Degrading (yesterday perfect → today has failures)';
                  } else {
                    trendLine = 'Trend: ➡️ Stable (failures both days)';
                  }
                } else {
                  trendLine = 'Trend: ⊘ No prior-day data for comparison';
                }
              } catch (e) {
                trendLine = `Trend: ⊘ Could not fetch prior-day data (${e.message})`;
              }
            }

            // ── Build scorecard ─────────────────────────────────────
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const lines = [
              `## 🌅 NemoClaw Nightly Scorecard — ${today}`,
              '',
              `**Run mode:** ${runMode}`,
            ];

            if (isSelectiveDispatch) {
              lines.push(`**Requested jobs:** ${requestedJobs.map((name) => `\`${name}\``).join(', ')}`);
            }

            lines.push(
              `**Jobs run:** ${ran} of ${total}`,
              `  ✅ ${success} passed`,
              `  ❌ ${failure} failed`,
              `  ⊘  ${cancelled} cancelled`,
              `  ⏭️  ${skipped} skipped`,
            );

            if (failedJobs.length > 0) {
              lines.push('');
              lines.push('**Failed jobs:**');
              for (const name of failedJobs) {
                lines.push(`  - \`${name}\``);
              }
            }

            if (perfect) {
              lines.push('');
              lines.push('🎉 **All jobs passed!**');
            }

            lines.push('');
            lines.push(trendLine);
            lines.push('');
            lines.push(`🔗 [Full run details](${runUrl})`);

            const scorecard = lines.join('\n');
            core.summary.addRaw(scorecard);
            await core.summary.write();
            core.setOutput('scorecard', scorecard);

      # ── Optional Slack notification ────────────────────────────
      - name: Post scorecard to Slack
        if: ${{ steps.scorecard.outputs.scorecard != '' }}
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
          SCORECARD_TEXT: ${{ steps.scorecard.outputs.scorecard }}
        with:
          script: |
            const webhookUrl = process.env.SLACK_WEBHOOK_URL;
            if (!webhookUrl) {
              core.info('SLACK_WEBHOOK_URL not configured — skipping Slack notification');
              return;
            }

            const scorecard = process.env.SCORECARD_TEXT;

            // Strip markdown formatting for Slack plain-text rendering
            const slackText = scorecard
              .replace(/^## /gm, '')
              .replace(/\*\*/g, '*')
              .replace(/\[([^\]]+)\]\(([^)]+)\)/g, '<$2|$1>');

            const resp = await fetch(webhookUrl, {
              method: 'POST',
              headers: { 'Content-Type': 'application/json' },
              body: JSON.stringify({ text: slackText }),
            });

            if (!resp.ok) {
              core.warning(`Slack webhook returned ${resp.status}: ${await resp.text()}`);
            } else {
              core.info('Scorecard posted to Slack');
            }