ci-benchmark.yaml

name: CI - Benchmark
run-name: >-
  ${{ github.event_name == 'workflow_dispatch'
    && format('Benchmark: {0} | {1}',
        inputs.platform,
        github.ref_name)
    || format('Benchmark: PR #{0} | {1}',
        github.event.issue.number,
        github.event.comment.body) }}

concurrency:
  group: >-
    ${{
      github.event_name == 'issue_comment' &&
      !contains(github.event.comment.body, '/benchmark kind') &&
      !contains(github.event.comment.body, '/benchmark openshift')
      && format('benchmark-isolated-{0}', github.run_id)
      || format('benchmark-{0}',
           github.event.issue.number
           || github.run_id)
    }}
  cancel-in-progress: true

on:
  issue_comment:
    types: [created]
  workflow_dispatch:
    inputs:
      platform:
        description: 'Platform: kind or openshift'
        required: true
        default: 'kind'
        type: choice
        options: [kind, openshift]

jobs:
  gate:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
    outputs:
      run_benchmark: ${{ steps.check.outputs.run_benchmark }}
      platform: ${{ steps.check.outputs.platform }}
      pr_number: ${{ steps.check.outputs.pr_number }}
      pr_head_sha: ${{ steps.check.outputs.pr_head_sha }}
      pr_head_repo: ${{ steps.check.outputs.pr_head_repo }}
    steps:
      - name: Check if benchmark requested
        id: check
        uses: actions/github-script@v7
        with:
          script: |
            async function hasWriteAccess(username) {
              try {
                const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  username: username
                });
                const privilegedRoles = ['admin', 'maintain', 'write'];
                return privilegedRoles.includes(permission.permission);
              } catch (e) {
                console.log(`Could not get permissions for ${username}: ${e.message}`);
                return false;
              }
            }

            if (context.eventName !== 'issue_comment' && context.eventName !== 'workflow_dispatch') {
              core.setOutput('run_benchmark', 'false');
              return;
            }

            if (context.eventName === 'workflow_dispatch') {
              const platform = context.payload.inputs.platform;
              const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
              console.log(`Manual benchmark dispatch for ${platform}`);
              
              core.setOutput('run_benchmark', 'true');
              core.setOutput('platform', platform);

              // Try to find a PR for the current branch so we can post results
              const branch = context.ref.replace('refs/heads/', '');
              const { data: prs } = await github.rest.pulls.list({
                owner: context.repo.owner,
                repo: context.repo.repo,
                head: `${context.repo.owner}:${branch}`,
                state: 'open',
              });
              if (prs.length > 0) {
                core.setOutput('pr_number', prs[0].number.toString());
                core.setOutput('pr_head_sha', prs[0].head.sha);
                console.log(`Found open PR #${prs[0].number} for branch ${branch}`);
              } else {
                console.log(`No open PR found for branch ${branch}, skipping PR outputs`);
              }
              return;
            }

            const comment = context.payload.comment.body.trim();
            const issue = context.payload.issue;

            if (!issue.pull_request) {
              console.log('Comment is not on a PR, skipping');
              core.setOutput('run_benchmark', 'false');
              return;
            }

            const validCommands = ['/benchmark kind', '/benchmark openshift'];
            if (!validCommands.includes(comment)) {
              console.log(`Comment "${comment}" is not a valid benchmark command, skipping`);
              core.setOutput('run_benchmark', 'false');
              return;
            }

            const commenter = context.payload.comment.user.login;
            const hasAccess = await hasWriteAccess(commenter);
            if (!hasAccess) {
              console.log(`User ${commenter} does not have write access, ignoring ${comment}`);
              core.setOutput('run_benchmark', 'false');
              return;
            }

            const { data: pr } = await github.rest.pulls.get({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: issue.number
            });

            const baseRepo = `${context.repo.owner}/${context.repo.repo}`;
            const headRepo = pr.head.repo ? pr.head.repo.full_name : baseRepo;

            console.log(`/benchmark kind approved by ${commenter} for PR #${issue.number}`);
            console.log(`PR head SHA: ${pr.head.sha}`);

            await github.rest.reactions.createForIssueComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: context.payload.comment.id,
              content: 'rocket'
            });

            const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
            const platform = comment.includes('openshift') ? 'OpenShift' : 'Kind';
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: issue.number,
              body: `🚀 **Benchmark (${platform})** triggered by \`${comment}\`\n\n[View the benchmark workflow run](${runUrl})`
            });

            core.setOutput('run_benchmark', 'true');
            core.setOutput('platform', platform.toLowerCase());
            core.setOutput('pr_number', issue.number.toString());
            core.setOutput('pr_head_sha', pr.head.sha);
            core.setOutput('pr_head_repo', headRepo);

  build-image:
    needs: gate
    if: needs.gate.outputs.run_benchmark == 'true' && (needs.gate.outputs.platform == 'openshift' || github.event.inputs.platform == 'openshift')
    runs-on: ubuntu-latest
    outputs:
      image_tag: ${{ steps.build.outputs.image_tag }}
    steps:
      - name: Checkout source
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.gate.outputs.pr_head_sha }}

      - name: Log in to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ secrets.CR_USER }}
          password: ${{ secrets.CR_TOKEN }}

      - name: Build and push image
        id: build
        env:
          REGISTRY: ghcr.io
          IMAGE_NAME: ${{ github.repository }}
          GIT_REF: ${{ needs.gate.outputs.pr_head_sha }}
        run: |
          IMAGE_TAG="bench-$(printf '%s' "$GIT_REF" | cut -c1-8)"
          FULL_IMAGE="${REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}"
          echo "Building image: $FULL_IMAGE"
          
          make docker-build IMG="$FULL_IMAGE"
          make docker-push IMG="$FULL_IMAGE"
          
          echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT

  benchmark-kind:
    runs-on: ubuntu-latest
    needs: [gate]
    if: needs.gate.outputs.run_benchmark == 'true' && (needs.gate.outputs.platform == 'kind' || github.event.inputs.platform == 'kind')
    timeout-minutes: 45
    permissions:
      contents: write
      statuses: write
      pull-requests: write
      actions: read
    steps:
      - name: Set pending status on PR head
        if: github.event_name == 'issue_comment'
        uses: actions/github-script@v7
        with:
          script: |
            await github.rest.repos.createCommitStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              sha: '${{ needs.gate.outputs.pr_head_sha }}',
              state: 'pending',
              target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
              description: 'Benchmark running...',
              context: '${{ github.workflow }} / benchmark-kind'
            });

      - name: Validate PR head SHA
        if: github.event_name == 'issue_comment'
        run: |
          if [ -z "${{ needs.gate.outputs.pr_head_sha }}" ]; then
            echo "::error::pr_head_sha is empty — refusing to fall back to main"
            exit 1
          fi
          echo "Checkout will use PR head SHA: ${{ needs.gate.outputs.pr_head_sha }}"

      - name: Checkout source
        uses: actions/checkout@v4
        with:
          repository: ${{ needs.gate.outputs.pr_head_repo || github.repository }}
          ref: ${{ needs.gate.outputs.pr_head_sha || github.sha }}
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract Go version from go.mod
        run: sed -En 's/^go (.*)$/GO_VERSION=\1/p' go.mod >> $GITHUB_ENV

      - name: Set up Go with cache
        uses: actions/setup-go@v6
        with:
          go-version: "${{ env.GO_VERSION }}"
          cache-dependency-path: ./go.sum

      - name: Install dependencies
        run: go mod download

      - name: Install Kind
        run: |
          ARCH=$(uname -m)
          case "$ARCH" in
            x86_64)  KIND_ARCH="amd64" ;;
            aarch64) KIND_ARCH="arm64" ;;
            *)       echo "Unsupported architecture: $ARCH"; exit 1 ;;
          esac
          curl -Lo ./kind "https://kind.sigs.k8s.io/dl/v0.25.0/kind-linux-${KIND_ARCH}"
          chmod +x ./kind
          sudo mv ./kind /usr/local/bin/kind
          kind version

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build WVA image locally
        id: build-image
        env:
          CHECKOUT_SHA: ${{ needs.gate.outputs.pr_head_sha }}
        run: |
          IMAGE_NAME="llm-d-workload-variant-autoscaler"
          IMAGE_TAG="bench-${CHECKOUT_SHA:0:7}"
          FULL_IMAGE="localhost/${IMAGE_NAME}:${IMAGE_TAG}"
          echo "Building local image: $FULL_IMAGE"
          make docker-build IMG="$FULL_IMAGE"
          echo "image=$FULL_IMAGE" >> $GITHUB_OUTPUT

      - name: Deploy e2e infrastructure
        env:
          ENVIRONMENT: kind-emulator
          USE_SIMULATOR: "true"
          CREATE_CLUSTER: "true"
          INSTALL_GATEWAY_CTRLPLANE: "true"
          E2E_TESTS_ENABLED: "true"
          IMG: ${{ steps.build-image.outputs.image }}
          SKIP_BUILD: "true"
          KV_SPARE_TRIGGER: "0.5"
          QUEUE_SPARE_TRIGGER: "4.5"
          INSTALL_GRAFANA: "true"
        run: make deploy-e2e-infra

      - name: Run benchmark
        env:
          ENVIRONMENT: kind-emulator
          USE_SIMULATOR: "true"
          SCALER_BACKEND: prometheus-adapter
          BENCHMARK_RESULTS_FILE: /tmp/benchmark-results.json
          BENCHMARK_GRAFANA_ENABLED: "true"
          BENCHMARK_GRAFANA_SNAPSHOT_FILE: /tmp/benchmark-grafana-snapshot.txt
          BENCHMARK_GRAFANA_SNAPSHOT_JSON: /tmp/benchmark-grafana-snapshot.json
          BENCHMARK_GRAFANA_PANEL_DIR: /tmp/benchmark-panels
          KV_SPARE_TRIGGER: "0.5"
          QUEUE_SPARE_TRIGGER: "4.5"
        run: make test-benchmark

      - name: Upload benchmark results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: |
            /tmp/benchmark-results.json
            /tmp/prefill-benchmark-results.json
            /tmp/benchmark-grafana-snapshot.txt
            /tmp/benchmark-grafana-snapshot.json
            /tmp/benchmark-panels/
          if-no-files-found: warn

      - name: Post benchmark results as PR comment
        if: always() && github.event_name == 'issue_comment'
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const path = require('path');
            const prNumber = parseInt('${{ needs.gate.outputs.pr_number }}');
            const sha = '${{ needs.gate.outputs.pr_head_sha }}';
            const runId = context.runId;
            const repoUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}`;

            // Look up the uploaded artifact to get a direct download link
            let artifactUrl = `${repoUrl}/actions/runs/${runId}`;
            try {
              const { data: { artifacts } } = await github.rest.actions.listWorkflowRunArtifacts({
                owner: context.repo.owner,
                repo: context.repo.repo,
                run_id: runId
              });
              const benchArtifact = artifacts.find(a => a.name === 'benchmark-results');
              if (benchArtifact) {
                artifactUrl = `${repoUrl}/actions/runs/${runId}/artifacts/${benchArtifact.id}`;
              }
            } catch (e) {
              console.log(`Could not look up artifact: ${e.message}`);
            }

            let resultsTable = '⚠️ Benchmark results file not found or could not be parsed.';

            try {
              const data = JSON.parse(fs.readFileSync('/tmp/benchmark-results.json', 'utf8'));

              const fmtTime = (v) => v < 0 ? 'N/A' : `${v.toFixed(1)}s`;

              resultsTable = `| Metric | Value |
            |--------|-------|
            | Scale-up time | ${fmtTime(data.scaleUpTimeSec)} |
            | Scale-down time | ${fmtTime(data.scaleDownTimeSec)} |
            | Max replicas | ${data.maxReplicas} |
            | Avg KV cache usage | ${data.avgKVCacheUsage.toFixed(3)} |
            | Avg queue depth | ${data.avgQueueDepth.toFixed(1)} |
            | Replica oscillation (σ) | ${data.replicaOscillation.toFixed(2)} |
            | Total duration | ${data.totalDurationSec.toFixed(0)}s |`;
            } catch (e) {
              console.log(`Could not read results: ${e.message}`);
            }

            // Upload panel PNGs as release assets and collect URLs for embedding
            let panelImages = '';
            const panelDir = '/tmp/benchmark-panels';
            const hasPanels = fs.existsSync(panelDir) &&
              fs.readdirSync(panelDir).some(f => f.endsWith('.png'));

            if (hasPanels) {
              const pngs = fs.readdirSync(panelDir).filter(f => f.endsWith('.png')).sort();
              const tag = `benchmark-run-${runId}`;

              try {
                // Create a lightweight release to host panel images
                const release = await github.rest.repos.createRelease({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  tag_name: tag,
                  name: `Benchmark panels (PR #${prNumber}, ${sha.substring(0, 7)})`,
                  body: `Auto-generated by benchmark CI run #${runId}`,
                  draft: false,
                  prerelease: true
                });

                const imageUrls = [];
                for (const png of pngs) {
                  const filePath = path.join(panelDir, png);
                  const fileData = fs.readFileSync(filePath);
                  const asset = await github.rest.repos.uploadReleaseAsset({
                    owner: context.repo.owner,
                    repo: context.repo.repo,
                    release_id: release.data.id,
                    name: png,
                    data: fileData,
                    headers: { 'content-type': 'image/png' }
                  });
                  const title = png.replace('panel-', '').replace('.png', '').replace(/-/g, ' ');
                  imageUrls.push(`#### ${title}\n![${title}](${asset.data.browser_download_url})`);
                  console.log(`Uploaded ${png}: ${asset.data.browser_download_url}`);
                }

                if (imageUrls.length > 0) {
                  panelImages = `\n\n<details>\n<summary>Dashboard Panels (${imageUrls.length})</summary>\n\n${imageUrls.join('\n\n')}\n\n</details>`;
                }
              } catch (e) {
                console.log(`Could not upload panel images: ${e.message}`);
              }
            }

            // Check for Grafana snapshot
            const hasSnapshotJson = fs.existsSync('/tmp/benchmark-grafana-snapshot.json');
            let artifactsSection = '';
            if (hasSnapshotJson || hasPanels) {
              const items = [];
              if (hasSnapshotJson) {
                items.push('Grafana snapshot JSON');
              }
              artifactsSection = `\n\n📎 **[Download artifacts](${artifactUrl})**${items.length ? ' — ' + items.join(', ') : ''}`;
            }

            const body = `## Benchmark: scale-up-latency (Kind)

            ${resultsTable}${panelImages}${artifactsSection}

            <details>
            <summary>Environment</summary>

            - Cluster: Kind (emulated GPUs)
            - Model: unsloth/Meta-Llama-3.1-8B (simulator)
            - Commit: ${sha.substring(0, 7)}
            - Scaler: prometheus-adapter
            - [Workflow run](${repoUrl}/actions/runs/${runId})

            </details>`;

            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: prNumber,
              body: body
            });

      - name: Cleanup Kind cluster
        if: always()
        run: kind delete cluster --name kind-wva-gpu-cluster || true

  benchmark-openshift:
    runs-on: [self-hosted, openshift, vllm-d]
    needs: [gate, build-image]
    if: needs.gate.outputs.run_benchmark == 'true' && (needs.gate.outputs.platform == 'openshift' || github.event.inputs.platform == 'openshift')
    timeout-minutes: 60
    permissions:
      contents: write
      statuses: write
      pull-requests: write
      actions: read
    env:
      MODEL_ID: 'unsloth/Meta-Llama-3.1-8B'
      ACCELERATOR_TYPE: 'H100'
      GOTOOLCHAIN: auto
      LLMD_NAMESPACE: llm-d-benchmark-pr-${{ needs.gate.outputs.pr_number || github.run_id }}
      WVA_NAMESPACE: wva-benchmark-pr-${{ needs.gate.outputs.pr_number || github.run_id }}
      WVA_RELEASE_NAME: wva-bench-${{ github.run_id }}
      WVA_IMAGE_TAG: ${{ needs.build-image.outputs.image_tag }}
    steps:
      - name: Set pending status on PR head
        if: github.event_name == 'issue_comment'
        uses: actions/github-script@v7
        with:
          script: |
            await github.rest.repos.createCommitStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              sha: '${{ needs.gate.outputs.pr_head_sha }}',
              state: 'pending',
              target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
              description: 'Benchmark running on OpenShift...',
              context: '${{ github.workflow }} / benchmark-openshift'
            });

      - name: Checkout source
        uses: actions/checkout@v4
        with:
          repository: ${{ needs.gate.outputs.pr_head_repo || github.repository }}
          ref: ${{ needs.gate.outputs.pr_head_sha || github.sha }}
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Go
        uses: actions/setup-go@v6
        with:
          go-version: "1.25.x"
          cache-dependency-path: ./go.sum

      - name: Install tools (kubectl, oc, helm, make)
        run: |
          sudo apt-get update && sudo apt-get install -y make
          KUBECTL_VERSION="v1.31.0"
          curl -fsSL --retry 3 --retry-delay 5 -o kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
          chmod +x kubectl
          sudo mv kubectl /usr/local/bin/
          curl -fsSL --retry 3 --retry-delay 5 -O "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz"
          tar -xzf openshift-client-linux.tar.gz
          sudo mv oc /usr/local/bin/
          rm -f openshift-client-linux.tar.gz kubectl README.md
          curl -fsSL --retry 3 --retry-delay 5 https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash

      - name: Verify cluster access
        run: |
          kubectl cluster-info
          kubectl get nodes

      - name: Get HF token from cluster secret
        id: hf-token
        run: |
          HF_TOKEN=$(kubectl get secret llm-d-hf-token -n default -o jsonpath='{.data.HF_TOKEN}' | base64 -d)
          echo "::add-mask::$HF_TOKEN"
          echo "HF_TOKEN=$HF_TOKEN" >> $GITHUB_ENV

      - name: Clean up resources for this PR
        run: |
          for ns in "$LLMD_NAMESPACE" "$WVA_NAMESPACE"; do
            if kubectl get namespace "$ns" &>/dev/null; then
              kubectl delete hpa -n "$ns" -l app.kubernetes.io/name=workload-variant-autoscaler --ignore-not-found || true
              kubectl delete variantautoscaling -n "$ns" -l app.kubernetes.io/name=workload-variant-autoscaler --ignore-not-found || true
              for release in $(helm list -n "$ns" -q 2>/dev/null); do
                helm uninstall "$release" -n "$ns" --ignore-not-found --wait --timeout 60s || true
              done
              kubectl delete namespace "$ns" --ignore-not-found --timeout=60s || true
            fi
          done

      - name: Apply latest CRDs
        run: kubectl apply -f charts/workload-variant-autoscaler/crds/

      - name: Deploy WVA and llm-d infrastructure
        env:
          ENVIRONMENT: openshift
          INSTALL_GATEWAY_CTRLPLANE: "false"
          E2E_TESTS_ENABLED: "true"
          NAMESPACE_SCOPED: "false"
          LLMD_NS: ${{ env.LLMD_NAMESPACE }}
          WVA_NS: ${{ env.WVA_NAMESPACE }}
          CONTROLLER_INSTANCE: ${{ env.WVA_NAMESPACE }}
          DEPLOY_VA: "false"
          DEPLOY_HPA: "false"
          VLLM_MAX_NUM_SEQS: "5"
          DECODE_REPLICAS: "1"
          MONITORING_NAMESPACE: openshift-user-workload-monitoring
          WVA_METRICS_SECURE: "false"
          KV_SPARE_TRIGGER: "0.5"
          QUEUE_SPARE_TRIGGER: "4.5"
          VLLM_SVC_PORT: "8000"
          INSTALL_GRAFANA: "true"
        run: |
          ./deploy/install.sh --model "$MODEL_ID" --accelerator "$ACCELERATOR_TYPE" --release-name "$WVA_RELEASE_NAME" --environment openshift

      - name: Label namespaces for OpenShift monitoring
        run: |
          kubectl label namespace "$LLMD_NAMESPACE" openshift.io/user-monitoring=true --overwrite
          kubectl label namespace "$WVA_NAMESPACE" openshift.io/user-monitoring=true --overwrite

      - name: Wait for infrastructure to be ready
        run: |
          kubectl rollout status deployment -l app.kubernetes.io/name=workload-variant-autoscaler -n "$WVA_NAMESPACE" --timeout=300s || true
          kubectl rollout status deployment/ms-inference-scheduling-llm-d-modelservice-decode -n "$LLMD_NAMESPACE" --timeout=1500s || true
          echo "--- Services in openshift-user-workload-monitoring ---"
          kubectl get svc -n openshift-user-workload-monitoring
          echo "--- Services in openshift-monitoring ---"
          kubectl get svc -n openshift-monitoring

      - name: Run benchmark
        env:
          ENVIRONMENT: openshift
          USE_SIMULATOR: "false"
          SCALER_BACKEND: prometheus-adapter
          CONTROLLER_NAMESPACE: ${{ env.WVA_NAMESPACE }}
          E2E_MONITORING_NAMESPACE: openshift-user-workload-monitoring
          E2E_EMULATED_LLMD_NAMESPACE: ${{ env.LLMD_NAMESPACE }}
          CONTROLLER_INSTANCE: ${{ env.WVA_NAMESPACE }}
          BENCHMARK_RESULTS_FILE: /tmp/benchmark-results.json
          BENCHMARK_GRAFANA_ENABLED: "true"
          BENCHMARK_GRAFANA_SNAPSHOT_FILE: /tmp/benchmark-grafana-snapshot.txt
          BENCHMARK_GRAFANA_SNAPSHOT_JSON: /tmp/benchmark-grafana-snapshot.json
          BENCHMARK_GRAFANA_PANEL_DIR: /tmp/benchmark-panels
          KV_SPARE_TRIGGER: "0.5"
          QUEUE_SPARE_TRIGGER: "4.5"
        run: |
          # Get token for Thanos querier
          export PROMETHEUS_TOKEN=$(kubectl create token prometheus-k8s -n openshift-monitoring --duration=24h 2>/dev/null || echo "")
          make test-benchmark

      - name: Generate benchmark plots
        if: always()
        run: |
          echo "Installing matplotlib and numpy..."
          if python3 -m venv /tmp/plot-venv 2>&1; then
            /tmp/plot-venv/bin/pip install --quiet matplotlib numpy 2>&1
            PYTHON=/tmp/plot-venv/bin/python3
          else
            echo "venv failed, using PIP_BREAK_SYSTEM_PACKAGES fallback..."
            curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
            PIP_BREAK_SYSTEM_PACKAGES=1 python3 /tmp/get-pip.py --user 2>&1
            PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install --user matplotlib numpy 2>&1
            PYTHON=python3
          fi
          $PYTHON - <<'PLOTEOF'
          import json, os, sys
          try:
              import matplotlib
              matplotlib.use('Agg')
              import matplotlib.pyplot as plt
              import matplotlib.ticker as ticker
              import numpy as np
          except ImportError:
              print("matplotlib not available, skipping plot generation")
              sys.exit(0)

          PANEL_DIR = '/tmp/benchmark-panels'
          PREFILL_FILE = '/tmp/prefill-benchmark-results.json'
          os.makedirs(PANEL_DIR, exist_ok=True)

          if not os.path.exists(PREFILL_FILE):
              print("No prefill results found, skipping plots")
              sys.exit(0)

          with open(PREFILL_FILE) as f:
              results = json.load(f)

          if not isinstance(results, list) or len(results) < 2:
              print("Need at least 2 results (HPA + WVA) for comparison plots")
              sys.exit(0)

          hpa = next((r for r in results if r['autoscaler_type'] == 'HPA'), None)
          wva = next((r for r in results if r['autoscaler_type'] == 'WVA'), None)
          if not hpa or not wva:
              print("Missing HPA or WVA results")
              sys.exit(0)

          plt.rcParams.update({
              'figure.facecolor': 'white', 'axes.facecolor': '#f8f9fa',
              'axes.grid': True, 'grid.alpha': 0.3, 'font.size': 12,
              'axes.titlesize': 14, 'axes.labelsize': 12, 'figure.dpi': 150,
          })
          HPA_C, WVA_C = '#e74c3c', '#2ecc71'

          def bar_pair(ax, hv, wv, title, ylabel, fmt='.1f'):
              bars = ax.bar(['HPA', 'WVA'], [hv, wv], color=[HPA_C, WVA_C], width=0.5, edgecolor='white', linewidth=1.5)
              ax.set_title(title, fontweight='bold')
              ax.set_ylabel(ylabel)
              for bar, val in zip(bars, [hv, wv]):
                  ax.text(bar.get_x() + bar.get_width()/2, bar.get_height() + ax.get_ylim()[1]*0.02,
                          f'{val:{fmt}}', ha='center', va='bottom', fontweight='bold', fontsize=11)

          # 1. Comparison bars (3x3 grid)
          fig, axes = plt.subplots(3, 3, figsize=(20, 13))
          fig.suptitle('HPA vs WVA — Prefill-Heavy Workload (OpenShift)', fontsize=16, fontweight='bold', y=1.02)
          bar_pair(axes[0,0], hpa['throughput']['mean'], wva['throughput']['mean'], 'Mean Throughput', 'tokens/sec', '.0f')
          bar_pair(axes[0,1], hpa['ttft']['count'], wva['ttft']['count'], 'Completed Requests', 'count', 'd')
          bar_pair(axes[0,2], hpa['max_replicas'], wva['max_replicas'], 'Max Replicas', 'replicas', 'd')
          bar_pair(axes[1,0], hpa['avg_kv_cache'], wva['avg_kv_cache'], 'Avg KV Cache', 'utilization', '.3f')
          bar_pair(axes[1,1], hpa['avg_queue_depth'], wva['avg_queue_depth'], 'Avg vLLM Queue', 'requests', '.0f')
          bar_pair(axes[1,2], hpa.get('avg_epp_queue_depth',0), wva.get('avg_epp_queue_depth',0), 'Avg EPP Queue', 'requests', '.0f')
          bar_pair(axes[2,0], hpa['itl']['mean'], wva['itl']['mean'], 'Mean ITL', 'ms', '.2f')
          bar_pair(axes[2,1], hpa['ttft']['mean']/1000, wva['ttft']['mean']/1000, 'Mean TTFT', 'seconds', '.1f')
          bar_pair(axes[2,2], hpa['avg_replicas'], wva['avg_replicas'], 'Avg Replicas', 'replicas', '.1f')
          fig.tight_layout()
          fig.savefig(os.path.join(PANEL_DIR, 'panel-prefill-comparison.png'), bbox_inches='tight', dpi=150)
          plt.close()

          # 2. Replica timeline overlay
          fig, ax = plt.subplots(figsize=(14, 5))
          for data, label, color in [(hpa, 'HPA', HPA_C), (wva, 'WVA', WVA_C)]:
              tl = data['replica_timeline']
              times = [s['elapsed_sec'] for s in tl]
              ready = [s['ready_replicas'] for s in tl]
              ax.step(times, ready, where='post', label=f'{label} (ready)', color=color, linewidth=2.5)
              ax.fill_between(times, ready, step='post', alpha=0.1, color=color)
          ax.set_title('Ready Replicas Over Time — HPA vs WVA', fontsize=14, fontweight='bold')
          ax.set_xlabel('Time (seconds)')
          ax.set_ylabel('Ready Replicas')
          ax.legend(fontsize=12)
          ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True))
          ax.set_ylim(0, max(wva['max_replicas'], hpa['max_replicas']) + 1)
          fig.tight_layout()
          fig.savefig(os.path.join(PANEL_DIR, 'panel-prefill-replica-timeline.png'), bbox_inches='tight', dpi=150)
          plt.close()

          # 3. Queue depth (vLLM + EPP) + KV cache over time
          fig, (ax_qd, ax_epp, ax_kv) = plt.subplots(3, 1, figsize=(14, 11), sharex=True)
          fig.suptitle('Queue Depth & KV Cache Over Time — HPA vs WVA', fontsize=15, fontweight='bold')
          for data, label, color in [(hpa, 'HPA', HPA_C), (wva, 'WVA', WVA_C)]:
              mt = data.get('metrics_timeline', [])
              if mt:
                  times = [s['elapsed_sec'] for s in mt]
                  qd = [s['queue_depth'] for s in mt]
                  epp_qd = [s.get('epp_queue_depth', 0) for s in mt]
                  kv = [s['kv_cache'] for s in mt]
                  ax_qd.plot(times, qd, label=label, color=color, linewidth=2, alpha=0.85)
                  ax_qd.fill_between(times, qd, alpha=0.1, color=color)
                  ax_epp.plot(times, epp_qd, label=label, color=color, linewidth=2, alpha=0.85)
                  ax_epp.fill_between(times, epp_qd, alpha=0.1, color=color)
                  ax_kv.plot(times, kv, label=label, color=color, linewidth=2, alpha=0.85)
                  ax_kv.fill_between(times, kv, alpha=0.1, color=color)
          ax_qd.set_title('vLLM Queue Depth (vllm:num_requests_waiting)', fontweight='bold')
          ax_qd.set_ylabel('Waiting Requests')
          ax_qd.legend(fontsize=11)
          ax_epp.set_title('EPP Queue Depth (inference_extension_flow_control_queue_size)', fontweight='bold')
          ax_epp.set_ylabel('Queued Requests')
          ax_epp.legend(fontsize=11)
          ax_kv.set_title('KV Cache Utilization (vllm:kv_cache_usage_perc)', fontweight='bold')
          ax_kv.set_ylabel('Utilization')
          ax_kv.set_xlabel('Time (seconds)')
          ax_kv.legend(fontsize=11)
          fig.tight_layout()
          fig.savefig(os.path.join(PANEL_DIR, 'panel-prefill-metrics-timeline.png'), bbox_inches='tight', dpi=150)
          plt.close()

          # 4. Throughput percentile distribution
          pct_keys = ['p05', 'p10', 'p25', 'p50', 'p75', 'p90', 'p95', 'p99']
          pct_labels = ['p5', 'p10', 'p25', 'p50', 'p75', 'p90', 'p95', 'p99']
          x = np.arange(len(pct_labels))
          w = 0.35
          fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(18, 5))
          fig.suptitle('Percentile Distributions — HPA vs WVA', fontsize=15, fontweight='bold')
          for ax, metric, unit, div in [(ax1,'ttft','seconds',1000),(ax2,'itl','ms',1),(ax3,'throughput','tok/s',1)]:
              hv = [hpa[metric]['percentiles'][k]/div for k in pct_keys]
              wv = [wva[metric]['percentiles'][k]/div for k in pct_keys]
              ax.bar(x - w/2, hv, w, label='HPA', color=HPA_C, alpha=0.85)
              ax.bar(x + w/2, wv, w, label='WVA', color=WVA_C, alpha=0.85)
              ax.set_xticks(x); ax.set_xticklabels(pct_labels, fontsize=9)
              ax.set_title(f'{metric.upper()} Percentiles', fontweight='bold')
              ax.set_ylabel(unit); ax.legend(fontsize=9)
          fig.tight_layout()
          fig.savefig(os.path.join(PANEL_DIR, 'panel-prefill-percentiles.png'), bbox_inches='tight', dpi=150)
          plt.close()

          print(f"Generated benchmark plots in {PANEL_DIR}")
          for f in sorted(os.listdir(PANEL_DIR)):
              if f.endswith('.png'):
                  print(f"  {f}")
          PLOTEOF

      - name: Upload benchmark results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results-openshift
          path: |
            /tmp/benchmark-results.json
            /tmp/prefill-benchmark-results.json
            /tmp/benchmark-grafana-snapshot.txt
            /tmp/benchmark-grafana-snapshot.json
            /tmp/benchmark-panels/
          if-no-files-found: warn

      - name: Post benchmark results as PR comment
        if: always() && (github.event_name == 'issue_comment' || needs.gate.outputs.pr_number != '')
        uses: actions/github-script@v7
        with:
          script: |
            const fs = require('fs');
            const path = require('path');
            const prNumber = parseInt('${{ needs.gate.outputs.pr_number }}');
            const sha = '${{ needs.gate.outputs.pr_head_sha }}';
            const runId = context.runId;
            const repoUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}`;

            let artifactUrl = `${repoUrl}/actions/runs/${runId}`;
            try {
              const { data: { artifacts } } = await github.rest.actions.listWorkflowRunArtifacts({
                owner: context.repo.owner,
                repo: context.repo.repo,
                run_id: runId
              });
              const benchArtifact = artifacts.find(a => a.name === 'benchmark-results-openshift');
              if (benchArtifact) {
                artifactUrl = `${repoUrl}/actions/runs/${runId}/artifacts/${benchArtifact.id}`;
              }
            } catch (e) {
              console.log(`Could not look up artifact: ${e.message}`);
            }

            let resultsTable = '⚠️ Benchmark results file not found or could not be parsed.';
            try {
              const data = JSON.parse(fs.readFileSync('/tmp/benchmark-results.json', 'utf8'));
              const fmtTime = (v) => v < 0 ? 'N/A' : `${v.toFixed(1)}s`;
              resultsTable = `| Metric | Value |
            |--------|-------|
            | Scale-up time | ${fmtTime(data.scaleUpTimeSec)} |
            | Scale-down time | ${fmtTime(data.scaleDownTimeSec)} |
            | Max replicas | ${data.maxReplicas} |
            | Avg KV cache usage | ${data.avgKVCacheUsage.toFixed(3)} |
            | Avg queue depth | ${data.avgQueueDepth.toFixed(1)} |
            | Replica oscillation (σ) | ${data.replicaOscillation.toFixed(2)} |
            | Total duration | ${data.totalDurationSec.toFixed(0)}s |`;
            } catch (e) {
              console.log(`Could not read results: ${e.message}`);
            }

            let prefillSection = '';
            try {
              const prefillData = JSON.parse(fs.readFileSync('/tmp/prefill-benchmark-results.json', 'utf8'));
              if (Array.isArray(prefillData) && prefillData.length >= 2) {
                const hpa = prefillData.find(r => r.autoscaler_type === 'HPA');
                const wva = prefillData.find(r => r.autoscaler_type === 'WVA');
                if (hpa && wva) {
                  const delta = (h, w, lowerBetter) => {
                    if (h === 0) return '—';
                    const pct = ((w - h) / Math.abs(h)) * 100;
                    const arrow = pct < 0 ? '↓' : '↑';
                    const sign = pct > 0 ? '+' : '';
                    return `${sign}${pct.toFixed(1)}% ${arrow}`;
                  };
                  const fmtP = (obj, key, div=1) => obj && obj.percentiles ? (obj.percentiles[key]/div).toFixed(1) : 'N/A';
                  const fmtM = (obj, div=1, prec=1) => obj ? (obj.mean/div).toFixed(prec) : 'N/A';

                  let table = `| Metric | HPA (Baseline) | WVA | Δ |
            |--------|---------------|-----|---|
            | **Max Replicas** | ${hpa.max_replicas} | **${wva.max_replicas}** | ${delta(hpa.max_replicas, wva.max_replicas)} |
            | **Avg Replicas** | ${hpa.avg_replicas.toFixed(2)} | **${wva.avg_replicas.toFixed(2)}** | ${delta(hpa.avg_replicas, wva.avg_replicas)} |
            | **Avg vLLM Queue Depth** | ${hpa.avg_queue_depth.toFixed(1)} | **${wva.avg_queue_depth.toFixed(1)}** | ${delta(hpa.avg_queue_depth, wva.avg_queue_depth)} |
            | **Avg EPP Queue Depth** | ${(hpa.avg_epp_queue_depth||0).toFixed(1)} | **${(wva.avg_epp_queue_depth||0).toFixed(1)}** | ${delta(hpa.avg_epp_queue_depth||0, wva.avg_epp_queue_depth||0)} |
            | **Avg KV Cache** | ${hpa.avg_kv_cache.toFixed(3)} | ${wva.avg_kv_cache.toFixed(3)} | ${delta(hpa.avg_kv_cache, wva.avg_kv_cache)} |
            | **TTFT mean** | ${fmtM(hpa.ttft, 1000)}ms | **${fmtM(wva.ttft, 1000)}ms** | ${hpa.ttft && wva.ttft ? delta(hpa.ttft.mean, wva.ttft.mean) : '—'} |
            | **TTFT p50** | ${fmtP(hpa.ttft, 'p50', 1000)}s | **${fmtP(wva.ttft, 'p50', 1000)}s** | — |
            | **TTFT p99** | ${fmtP(hpa.ttft, 'p99', 1000)}s | **${fmtP(wva.ttft, 'p99', 1000)}s** | — |
            | **ITL mean** | ${fmtM(hpa.itl, 1, 2)}ms | **${fmtM(wva.itl, 1, 2)}ms** | ${hpa.itl && wva.itl ? delta(hpa.itl.mean, wva.itl.mean) : '—'} |
            | **Throughput mean** | ${fmtM(hpa.throughput)}tok/s | **${fmtM(wva.throughput)}tok/s** | ${hpa.throughput && wva.throughput ? delta(hpa.throughput.mean, wva.throughput.mean) : '—'} |
            | **Throughput p50** | ${fmtP(hpa.throughput, 'p50')}tok/s | **${fmtP(wva.throughput, 'p50')}tok/s** | — |
            | **Completed Requests** | ${hpa.ttft ? hpa.ttft.count : 'N/A'} | **${wva.ttft ? wva.ttft.count : 'N/A'}** | ${hpa.ttft && wva.ttft ? delta(hpa.ttft.count, wva.ttft.count) : '—'} |
            | **Duration** | ${hpa.duration_sec.toFixed(0)}s | ${wva.duration_sec.toFixed(0)}s | — |`;

                  let timelines = '';
                  for (const r of [hpa, wva]) {
                    if (r.replica_timeline && r.replica_timeline.length > 0) {
                      timelines += `\n<details>\n<summary>${r.autoscaler_type} Replica Timeline (${r.replica_timeline.length} snapshots)</summary>\n\n| Time (s) | Spec | Ready |\n|----------|------|-------|\n`;
                      for (const s of r.replica_timeline) {
                        timelines += `| ${s.elapsed_sec.toFixed(0)} | ${s.spec_replicas} | ${s.ready_replicas} |\n`;
                      }
                      timelines += `\n</details>\n`;
                    }
                  }

                  prefillSection = `\n\n---\n\n## Benchmark: prefill-heavy-workload (OpenShift)\n\n${table}\n${timelines}`;
                }
              } else if (Array.isArray(prefillData) && prefillData.length > 0) {
                let rows = '';
                for (const r of prefillData) {
                  rows += `\n### ${r.autoscaler_type}\n\n| Metric | Value |\n|--------|-------|\n| Duration | ${r.duration_sec.toFixed(0)}s |\n| Max Replicas | ${r.max_replicas} |\n| Avg Replicas | ${r.avg_replicas.toFixed(2)} |\n| Avg vLLM Queue Depth | ${r.avg_queue_depth.toFixed(2)} |\n| Avg EPP Queue Depth | ${(r.avg_epp_queue_depth||0).toFixed(2)} |\n| Avg KV Cache | ${r.avg_kv_cache.toFixed(3)} |\n`;
                }
                prefillSection = `\n\n---\n\n## Benchmark: prefill-heavy-workload (OpenShift)\n${rows}`;
              }
            } catch (e) {
              console.log(`Could not read prefill results: ${e.message}`);
            }

            let panelImages = '';
            const panelDir = '/tmp/benchmark-panels';
            const hasPanels = fs.existsSync(panelDir) && fs.readdirSync(panelDir).some(f => f.endsWith('.png'));

            if (hasPanels) {
              const pngs = fs.readdirSync(panelDir).filter(f => f.endsWith('.png')).sort();
              const tag = `benchmark-run-os-${runId}`;
              try {
                const release = await github.rest.repos.createRelease({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  tag_name: tag,
                  name: `Benchmark panels OpenShift (PR #${prNumber}, ${sha.substring(0, 7)})`,
                  body: `Auto-generated by benchmark CI run #${runId}`,
                  draft: false,
                  prerelease: true
                });

                const imageUrls = [];
                for (const png of pngs) {
                  const filePath = path.join(panelDir, png);
                  const fileData = fs.readFileSync(filePath);
                  const asset = await github.rest.repos.uploadReleaseAsset({
                    owner: context.repo.owner,
                    repo: context.repo.repo,
                    release_id: release.data.id,
                    name: png,
                    data: fileData,
                    headers: { 'content-type': 'image/png' }
                  });
                  const title = png.replace('panel-', '').replace('.png', '').replace(/-/g, ' ');
                  imageUrls.push(`#### ${title}\n![${title}](${asset.data.browser_download_url})`);
                }

                if (imageUrls.length > 0) {
                  panelImages = `\n\n<details>\n<summary>Dashboard Panels (${imageUrls.length})</summary>\n\n${imageUrls.join('\n\n')}\n\n</details>`;
                }
              } catch (e) {
                console.log(`Could not upload panel images: ${e.message}`);
              }
            }

            const hasSnapshotJson = fs.existsSync('/tmp/benchmark-grafana-snapshot.json');
            let artifactsSection = '';
            if (hasSnapshotJson || hasPanels) {
              const items = [];
              if (hasSnapshotJson) items.push('Grafana snapshot JSON');
              artifactsSection = `\n\n📎 **[Download artifacts](${artifactUrl})**${items.length ? ' — ' + items.join(', ') : ''}`;
            }

            const body = `## Benchmark: scale-up-latency (OpenShift)

            ${resultsTable}${prefillSection}${panelImages}${artifactsSection}

            <details>
            <summary>Environment</summary>

            - Cluster: OpenShift (Real GPUs)
            - Model: unsloth/Meta-Llama-3.1-8B
            - Accelerator: H100
            - Commit: ${sha.substring(0, 7)}
            - Scaler: prometheus-adapter
            - [Workflow run](${repoUrl}/actions/runs/${runId})

            </details>`;

            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: prNumber,
              body: body
            });

      - name: Cleanup infrastructure
        if: always()
        run: |
          helm uninstall "$WVA_RELEASE_NAME" -n "$WVA_NAMESPACE" --ignore-not-found --wait --timeout 60s || true
          for release in $(helm list -n "$LLMD_NAMESPACE" -q 2>/dev/null); do
            helm uninstall "$release" -n "$LLMD_NAMESPACE" --ignore-not-found --wait --timeout 60s || true
          done
          kubectl delete namespace "$LLMD_NAMESPACE" --ignore-not-found --timeout=120s || true
          kubectl delete namespace "$WVA_NAMESPACE" --ignore-not-found --timeout=120s || true

  report-status:
    runs-on: ubuntu-latest
    needs: [gate, benchmark-kind, benchmark-openshift]
    if: always() && needs.gate.outputs.run_benchmark == 'true'
    permissions:
      statuses: write
    steps:
      - name: Report status to PR
        uses: actions/github-script@v7
        with:
          script: |
            const prHeadSha = '${{ needs.gate.outputs.pr_head_sha }}';
            const platform = '${{ needs.gate.outputs.platform }}';
            
            let benchResult;
            if (platform === 'openshift') {
              benchResult = '${{ needs.benchmark-openshift.result }}';
            } else {
              benchResult = '${{ needs.benchmark-kind.result }}';
            }

            if (!prHeadSha) {
              console.log('No PR head SHA available, skipping status report');
              return;
            }

            let state, description;
            if (benchResult === 'success') {
              state = 'success';
              description = 'Benchmark completed successfully';
            } else if (benchResult === 'skipped') {
              state = 'failure';
              description = 'Benchmark did not run (prerequisite failed or skipped)';
            } else if (benchResult === 'cancelled') {
              state = 'failure';
              description = 'Benchmark cancelled';
            } else {
              state = 'failure';
              description = 'Benchmark failed';
            }

            console.log(`Reporting status to PR commit ${prHeadSha}: ${state} - ${description}`);

            await github.rest.repos.createCommitStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              sha: prHeadSha,
              state: state,
              target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
              description: description,
              context: '${{ github.workflow }} / benchmark-kind'
            });

            console.log('Status reported successfully');