model-e2e.yml

name: Model E2E

# Live-model tests are intentionally manual: they consume external API quota,
# require repository secrets, and are non-deterministic. They are observability
# checks, not merge-queue gates.
on:
  workflow_dispatch:
    inputs:
      suite:
        description: 'Live-model suite to run'
        required: true
        type: choice
        default: all
        options:
          - all
          - none-runtime
          - opensandbox
          - docker

concurrency:
  group: model-e2e-${{ github.ref }}
  cancel-in-progress: true

permissions: {}

jobs:
  e2e:
    name: E2E (none runtime, live models)
    if: inputs.suite == 'all' || inputs.suite == 'none-runtime'
    runs-on: [self-hosted, linux, x64, trusted]
    timeout-minutes: 35
    permissions:
      contents: read
    steps:
      - name: Checkout source
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Check secret availability
        id: secrets
        env:
          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          QODER_ACCESS_TOKEN: ${{ secrets.QODER_ACCESS_TOKEN }}
        run: |
          set -euo pipefail
          if [ -z "$DASHSCOPE_API_KEY" ]; then
            echo "::error::DASHSCOPE_API_KEY is required for full e2e."
            exit 1
          fi
          if [ -z "$QODER_ACCESS_TOKEN" ]; then
            echo "::error::QODER_ACCESS_TOKEN is required for full e2e."
            exit 1
          fi
          echo "available=true" >> "$GITHUB_OUTPUT"

      - name: Set up Go
        if: steps.secrets.outputs.available == 'true'
        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
        with:
          go-version: "1.25.x"
          cache: true

      - name: Set up Node.js
        if: steps.secrets.outputs.available == 'true'
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version: "22"

      - name: Install claude-code CLI
        if: steps.secrets.outputs.available == 'true'
        run: |
          set -euo pipefail
          npm install -g --include=optional '@anthropic-ai/claude-code@2.1.217'
          claude --version

      - name: Install codex CLI
        if: steps.secrets.outputs.available == 'true'
        run: |
          set -euo pipefail
          npm install -g --include=optional '@openai/codex@0.80.0'
          codex --version

      - name: Install qwen-code CLI
        if: steps.secrets.outputs.available == 'true'
        run: |
          set -euo pipefail
          npm install -g '@qwen-code/qwen-code@0.20.1'
          qwen --version

      - name: Install qodercli (pinned v1.0.14)
        if: steps.secrets.outputs.available == 'true'
        env:
          QODER_VERSION: "1.0.14"
          QODER_SHA256: "4d83ec3d3a0948b73012cfdd6e2757be0a5da5c89a288425a5607fd42be1de7b"
        run: |
          set -euo pipefail
          tarball=$(mktemp)
          curl -fsSL "https://qoder-ide.oss-accelerate.aliyuncs.com/qodercli/releases/${QODER_VERSION}/qodercli-linux-x64.tar.gz" -o "$tarball"
          echo "$QODER_SHA256  $tarball" | sha256sum -c -
          mkdir -p "$HOME/.qoder/bin"
          tar -xzf "$tarball" -C "$HOME/.qoder/bin"
          rm -f "$tarball"
          echo "$HOME/.qoder/bin" >> "$GITHUB_PATH"

      - name: Verify qodercli on PATH
        if: steps.secrets.outputs.available == 'true'
        run: |
          set -euo pipefail
          command -v qodercli
          if ! qodercli --version; then
            echo "::warning::qodercli does not support a successful --version probe"
          fi

      - name: Run e2e tests (none runtime, full mode)
        if: steps.secrets.outputs.available == 'true'
        run: go test -tags e2e -timeout 1800s -count=1 -v ./e2e
        env:
          SKILL_UP_FULL_E2E: "1"
          SKILL_UP_E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts
          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          DASHSCOPE_BASE_URL: https://dashscope.aliyuncs.com/apps/anthropic
          ANTHROPIC_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          ANTHROPIC_BASE_URL: https://dashscope.aliyuncs.com/apps/anthropic
          ANTHROPIC_MODEL: qwen3.6-plus
          OPENAI_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          OPENAI_BASE_URL: https://dashscope.aliyuncs.com/compatible-mode/v1
          OPENAI_MODEL: qwen3.6-plus
          DASHSCOPE_MODEL: qwen3.6-plus
          QODER_PERSONAL_ACCESS_TOKEN: ${{ secrets.QODER_ACCESS_TOKEN }}

      - name: Upload e2e workspace artifacts
        if: always() && steps.secrets.outputs.available == 'true' && hashFiles('e2e-artifacts/**') != ''
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: model-e2e-none-workspaces
          path: e2e-artifacts/
          if-no-files-found: ignore
          retention-days: 14

  e2e-opensandbox:
    name: E2E (OpenSandbox, live model)
    if: inputs.suite == 'all' || inputs.suite == 'opensandbox'
    runs-on: [self-hosted, linux, x64, docker, trusted]
    timeout-minutes: 35
    permissions:
      contents: read
    steps:
      - name: Checkout source
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Check secret availability
        id: secrets
        env:
          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
        run: |
          set -euo pipefail
          if [ -z "$DASHSCOPE_API_KEY" ]; then
            echo "::error::DASHSCOPE_API_KEY is required for OpenSandbox e2e."
            exit 1
          fi
          echo "available=true" >> "$GITHUB_OUTPUT"

      - name: Set up Go
        if: steps.secrets.outputs.available == 'true'
        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
        with:
          go-version: "1.25.x"
          cache: true

      - name: Install uv
        if: steps.secrets.outputs.available == 'true'
        uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2

      - name: Install OpenSandbox server
        if: steps.secrets.outputs.available == 'true'
        run: |
          set -euo pipefail
          uv tool install 'opensandbox-server==0.1.13'
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"

      - name: Start OpenSandbox server
        if: steps.secrets.outputs.available == 'true'
        env:
          OPENSANDBOX_IMAGE: node:22@sha256:5647be709086c696ff32edaaf1c70cd26d1da6ab2b39c32f3c7b4c4a31957e37
          OPENSANDBOX_EXECD_IMAGE: opensandbox/execd:v1.0.16@sha256:af7b55c861926c1304371c4578007fbaa424538219154a6a49a5d636217d2a3a
        run: |
          set -euo pipefail
          docker version
          api_key="ci-$(openssl rand -hex 16)"
          echo "::add-mask::$api_key"
          config="$RUNNER_TEMP/sandbox.toml"
          cat > "$config" <<EOF
          [server]
          host = "127.0.0.1"
          port = 8080
          api_key = "$api_key"

          [runtime]
          type = "docker"
          execd_image = "$OPENSANDBOX_EXECD_IMAGE"

          [docker]
          network_mode = "host"
          EOF
          log="$RUNNER_TEMP/opensandbox-server.log"
          nohup opensandbox-server --config "$config" > "$log" 2>&1 &
          echo "OPENSANDBOX_SERVER_PID=$!" >> "$GITHUB_ENV"
          for i in $(seq 1 60); do
            if curl -fsS http://127.0.0.1:8080/health 2>/dev/null | grep -q healthy; then
              echo "OpenSandbox server is healthy"
              break
            fi
            if [ "$i" -eq 60 ]; then
              echo "::error::OpenSandbox server did not become healthy within 120s"
              cat "$log"
              exit 1
            fi
            sleep 2
          done
          {
            echo "OPENSANDBOX_API_KEY=$api_key"
            echo "OPENSANDBOX_BASE_URL=http://127.0.0.1:8080"
            echo "OPENSANDBOX_IMAGE=$OPENSANDBOX_IMAGE"
          } >> "$GITHUB_ENV"

      - name: Run e2e tests (OpenSandbox runtime)
        if: steps.secrets.outputs.available == 'true'
        run: go test -tags e2e -timeout 1800s -count=1 -v -run TestAgent_Codex_OpenSandboxRuntime ./e2e
        env:
          OPENAI_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          OPENAI_BASE_URL: https://dashscope.aliyuncs.com/compatible-mode/v1
          OPENAI_MODEL: qwen3.6-plus
          SKILL_UP_E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-opensandbox-artifacts

      - name: Stop OpenSandbox server
        if: always() && steps.secrets.outputs.available == 'true'
        run: |
          set -euo pipefail
          if [ -n "${OPENSANDBOX_SERVER_PID:-}" ]; then
            if kill "$OPENSANDBOX_SERVER_PID" 2>/dev/null; then
              echo "OpenSandbox server stopped"
            else
              echo "::notice::OpenSandbox server was already stopped"
            fi
          fi

      - name: Upload OpenSandbox server log
        if: always() && steps.secrets.outputs.available == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: model-e2e-opensandbox-server-log
          path: ${{ runner.temp }}/opensandbox-server.log
          if-no-files-found: ignore
          retention-days: 14

      - name: Upload OpenSandbox workspace artifacts
        if: always() && steps.secrets.outputs.available == 'true' && hashFiles('e2e-opensandbox-artifacts/**') != ''
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: model-e2e-opensandbox-workspaces
          path: e2e-opensandbox-artifacts/
          if-no-files-found: ignore
          retention-days: 14

  e2e-docker:
    name: E2E (Docker runtime, live model)
    if: inputs.suite == 'all' || inputs.suite == 'docker'
    runs-on: [self-hosted, linux, x64, docker, trusted]
    timeout-minutes: 25
    permissions:
      contents: read
    steps:
      - name: Checkout source
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
        with:
          persist-credentials: false

      - name: Check secret availability
        id: secrets
        env:
          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
        run: |
          set -euo pipefail
          if [ -z "$DASHSCOPE_API_KEY" ]; then
            echo "::error::DASHSCOPE_API_KEY is required for Docker e2e."
            exit 1
          fi
          echo "available=true" >> "$GITHUB_OUTPUT"

      - name: Set up Go
        if: steps.secrets.outputs.available == 'true'
        uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6
        with:
          go-version: "1.25.x"
          cache: true

      - name: Pre-pull Docker image
        if: steps.secrets.outputs.available == 'true'
        run: docker pull node:22@sha256:5647be709086c696ff32edaaf1c70cd26d1da6ab2b39c32f3c7b4c4a31957e37

      - name: Run Docker full e2e
        if: steps.secrets.outputs.available == 'true'
        run: go test -tags e2e -timeout 1800s -count=1 -v -run TestAgent_ClaudeCode_DockerRuntime ./e2e
        env:
          SKILL_UP_FULL_E2E: "1"
          SKILL_UP_E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-docker-full-artifacts
          ANTHROPIC_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          ANTHROPIC_BASE_URL: https://dashscope.aliyuncs.com/apps/anthropic
          ANTHROPIC_MODEL: qwen3.6-plus

      - name: Upload Docker e2e workspace artifacts
        if: always() && steps.secrets.outputs.available == 'true' && hashFiles('e2e-docker-full-artifacts/**') != ''
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: model-e2e-docker-workspaces
          path: e2e-docker-full-artifacts/
          if-no-files-found: ignore
          retention-days: 14

  summary:
    name: Model E2E Summary
    if: always()
    needs: [e2e, e2e-opensandbox, e2e-docker]
    runs-on: ubuntu-24.04
    timeout-minutes: 5
    permissions: {}
    steps:
      - name: Write job summary
        env:
          NONE_RESULT: ${{ needs.e2e.result }}
          OPENSANDBOX_RESULT: ${{ needs.e2e-opensandbox.result }}
          DOCKER_RESULT: ${{ needs.e2e-docker.result }}
          SELECTED_SUITE: ${{ inputs.suite }}
        run: |
          set -euo pipefail
          {
            echo "### Model E2E"
            echo
            echo "- Selected suite: \`${SELECTED_SUITE}\`"
            echo
            echo "| Runtime | Result |"
            echo "|---|---|"
            echo "| none | ${NONE_RESULT} |"
            echo "| OpenSandbox | ${OPENSANDBOX_RESULT} |"
            echo "| Docker | ${DOCKER_RESULT} |"
          } >> "$GITHUB_STEP_SUMMARY"
          for result in "$NONE_RESULT" "$OPENSANDBOX_RESULT" "$DOCKER_RESULT"; do
            if [ "$result" != success ] && [ "$result" != skipped ]; then
              echo "::error::One or more selected live-model suites failed"
              exit 1
            fi
          done