reusable-ci-nightly-benchmark.yaml

name: CI - Generic Nightly Run Benchmark

on:
  workflow_call:
    inputs:
      scenario_dir:
        description: 'Directory housing the scenarios'
        required: false
        type: 'string'
        default: 'cicd'
      standup_method:
        description: 'Standup method to be used ("standalone","fma","modelservice","kustomize")'
        required: false
        type: 'string'
        default: 'standalone'
      standup_scenario:
        description: 'Cluster provider targeted ("ocp","gke","cks")'
        required: false
        type: 'string'
        default: 'kind'
      decode_pods:
        description: 'Number of decode (vllm "standalone") pods (default "auto", means what was configured on the scenario)'
        required: false
        type: 'string'
        default: 'auto'
      prefill_pods:
        description: 'Number of prefill pods (default "auto", means what was configured on the scenario)'
        required: false
        type: 'string'
        default: 'auto'
      accelerator_type:
        description: 'Type of accelerator (gpu, cpu, sim, amd, xpu, hpu, tpu/v6, tpu/v7) (ONLY VALID FOR "kustomize" standyp method)'
        required: false
        type: 'string'
        default: 'gpu'
      backend_type:
        description: 'Type of backend (vllm, sglang) (ONLY VALID FOR "kustomize" standyp method)'
        required: false
        type: 'string'
        default: 'vllm'
      infra_provider:
        description: 'Target cloud provider (ONLY VALID FOR "kustomize" standup method)'
        required: false
        type: 'string'
        default: 'base'
      offloading_target:
        description: 'Which component should be used to hous offloaded blocks ("cpu","memory", default is "")'
        required: false
        type: 'string'
        default: ''
      connector:
        description: 'Accelerator offloading connector (ONLY VALID FOR "kustomize" standyp method)'
        required: false
        type: 'string'
        default: ''
      cluster_namespace:
        description: 'Cluster namespace'
        required: false
        type: 'string'
        default: 'llmdbenchcicd'
      helm_release:
        description: 'Helm release name (for modelservice)'
        required: false
        type: 'string'
        default: 'lmdbenchcicdr'
      workspace_dir:
        description: 'Workspace directory'
        required: false
        type: 'string'
        default: '/tmp/llmdbenchcicd'
      bucket_project:
        description: 'Project associated to the object storage bucket'
        required: false
        type: 'string'
        default: 'llm-d-scale'
      bucket_provider:
        description: 'Type (provider of) the bucket'
        required: false
        type: 'string'
        default: 'none'
      bucket_path:
        description: 'Path to object storage bucket'
        required: false
        type: 'string'
        default: 'llm-d-benchmarks/regressions'
      gateway_class:
        description: 'Class of gateway used ("istio", "agentgateway", "epponly" (aka "standalone"), "none")'
        required: false
        type: 'string'
        default: 'epponly'
      monitoring_enabled:
        description: 'Enabled monitoring ("true", "false")'
        required: false
        type: 'string'
        default: 'true'
      dry_run:
        description: 'Execute workflow in "dry-run" mode ("true", "false")'
        required: false
        type: 'string'
        default: 'false'
      verbose:
        description: 'Execute workflow in "verbose" mode ("true", "false")'
        required: false
        default: 'false'
        type: 'string'
      harness:
        description: 'Harness to be used during "run" operation ("inference-perf", "guidellm", "inferencemax", "vllm-benchmark", "nop")'
        required: false
        default: 'inference-perf'
        type: 'string'
      workload:
        description: 'Workload profile to be used during "run" operation (check list under "workload/profiles")'
        required: false
        default: 'sanity_random.yaml'
        type: 'string'
      cleanup:
        description: 'Cleanup the llm-d stack stood up by this workflow'
        required: false
        default: 'true'
        type: string
      tpu_topology:
        description: 'GKE TPU topology (e.g. 2x4, 2x2x1)'
        required: false
        type: 'string'
        default: '2x4'
      tpu_chips:
        description: 'Total TPU chips to request (0 for simulated)'
        required: false
        type: 'string'
        default: '0'
      tpu_wait_timeout:
        description: 'Minutes to wait for TPU chips to become available (0 = no wait, fail immediately)'
        required: false
        type: 'string'
        default: '0'
      tpu_nodepool:
        description: 'Target TPU node pool name (empty = do not override)'
        required: false
        type: 'string'
        default: ''
      data_access_timeout:
        description: 'Timeout in seconds for data access and harness run execution'
        required: false
        type: 'string'
        default: '1200'
      wait_timeout:
        description: 'Timeout in seconds to wait for benchmark pods to complete'
        required: false
        type: 'string'
        default: '1200'
      pvc_bind_timeout:
        description: 'Timeout in seconds to wait for a pvc to be bound'
        required: false
        type: 'string'
        default: '1200'
      job_timeout:
        description: 'Timeout for the Kubernetes job itself (e.g. 2700s)'
        required: false
        type: 'string'
        default: '2700s'
      standup_timeout:
        description: 'Timeout in seconds for standup of a new llm-d stack'
        required: false
        type: 'string'
        default: '2400'

#  push:
#    branches:
#      - main

  schedule:
    - cron: '0 0 * * *'  # Daily at midnight UTC

jobs:
  ensure-nightly-build-image:
    name: Ensure Nightly Build  Image
    runs-on: ubuntu-latest
    timeout-minutes: 30
    outputs:
      conditional_runs_on: ${{ steps.prepare_set_runs_on.outputs.runners }}
      cluster_provider: ${{ steps.prepare_set_cluster_provider.outputs.cluster_type }}

    steps:
      - name: Install system dependencies
        id: prepare_install_skopeo
        run: |
          sudo apt-get update
          sudo apt-get install -y skopeo yq

      - name: Ensure image is already built
        id: prepare_ensure_benchmark_nightly_image
        run: |
          IMAGE="ghcr.io/llm-d/llm-d-benchmark:nightly"
          POLL=30
          while true; do
            creation_date=$(skopeo inspect docker://${IMAGE} | yq -r .Created) && \
            creation_ts=$(date -d "${creation_date}" +"%s")
            current_ts=$(date +%s) && \
            image_age=$((current_ts - creation_ts))
            if [[ $image_age -gt 86400 ]]; then
              echo "Current image \"${IMAGE}\" is too old (${image_age}s). Waiting $POLL seconds and trying again..."
              sleep $POLL
            else
              echo "Current image \"${IMAGE}\" is recent enough (${image_age}s)."
              exit 0
            fi
          done
        shell: bash

      - name: set cluster_type variable based on workflow and scenario name
        id: prepare_set_cluster_provider
        run: |
          IS_OCP=$(echo "${{ github.workflow }}" | grep -i "\ocp" || true)$(echo "${{ inputs.standup_scenario }}" | grep -i "ocp" || true)
          if [[ ! -z $IS_OCP ]]; then
            export CLUSTER_TYPE=ocp
          fi
          IS_GKE=$(echo "${{ github.workflow }}" | grep -i "\gke" || true)$(echo "${{ inputs.standup_scenario }}" | grep -i "gke" || true)
          if [[ ! -z $IS_GKE ]]; then
            export CLUSTER_TYPE=gke
          fi
          IS_CKS=$(echo "${{ github.workflow }}" | grep -i "\cks" || true)$(echo "${{ inputs.standup_scenario }}" | grep -i "cks" || true)
          if [[ ! -z $IS_CKS ]]; then
            export CLUSTER_TYPE=cks
          fi
          IS_KIND=$(echo "${{ github.workflow }}" | grep -i "\kind" || true)$(echo "${{ inputs.standup_scenario }}" | grep -i "kind" || true)
          if [[ ! -z $IS_KIND ]]; then
            export CLUSTER_TYPE=kind
          fi
          IS_ROCM=$(echo "${{ github.workflow }}" | grep -i "\rocm" || true)$(echo "${{ inputs.standup_scenario }}" | grep -i "rocm" || true)
          if [[ ! -z $IS_ROCM ]]; then
            export CLUSTER_TYPE=rocm
          fi
          IS_XPU=$(echo "${{ github.workflow }}" | grep -i "\xpu" || true)$(echo "${{ inputs.standup_scenario }}" | grep -i "xpu" || true)
          if [[ ! -z $IS_XPU ]]; then
            export CLUSTER_TYPE=xpu
          fi
          IS_HPU=$(echo "${{ github.workflow }}" | grep -i "\hpu" || true)$(echo "${{ inputs.standup_scenario }}" | grep -i "hpu" || true)
          if [[ ! -z $IS_HPU ]]; then
            export CLUSTER_TYPE=hpu
          fi
          echo "Cluster provider is \"$CLUSTER_TYPE\""
          echo "cluster_type=$CLUSTER_TYPE" >> "$GITHUB_OUTPUT"
          echo "CLUSTER_TYPE=$CLUSTER_TYPE" >> "$GITHUB_ENV"

      - name: set runs-on variable based on cluster provider
        id: prepare_set_runs_on
        run: |
          echo "Cluster provider is \"$CLUSTER_TYPE\""
          if [[ "$CLUSTER_TYPE" == "ocp" ]]; then
            echo "Targetting self-hosted runners (OpenShift)"
            RUNNERS_JSON_LIST='["self-hosted", "openshift", "pok-prod"]'
          elif [[ "$CLUSTER_TYPE" == "rocm" ]]; then
            echo "Targetting self-hosted runners (AMD/CI)"
            RUNNERS_JSON_LIST='["rocm"]'
          elif [[ "$CLUSTER_TYPE" == "xpu" ]]; then
            echo "Targetting self-hosted runners (Intel/CI)"
            RUNNERS_JSON_LIST='["xpu"]'
          elif [[ "$CLUSTER_TYPE" == "hpu" ]]; then
            echo "Targetting self-hosted runners (Intel/CI)"
            RUNNERS_JSON_LIST='["hpu-dispatch-runner"]'
          else
            echo "Targetting hosted runners (non-OpenShift)"
            RUNNERS_JSON_LIST='["ubuntu-latest"]'
          fi
          echo "Runners list is \"$RUNNERS_JSON_LIST\""
          echo "runners=$RUNNERS_JSON_LIST" >> "$GITHUB_OUTPUT"

  skip-v7-warning:
    name: Skip TPU v7 Run
    runs-on: ubuntu-latest
    if: inputs.accelerator_type == 'tpu/v7' || inputs.accelerator_type == 'tpu-v7'
    steps:
      - name: Print skip message
        id: prepare_skip_tpu_v7_run
        run: |
          echo "::warning::TPU v7 capacity is currently not available. Skipping benchmark run."
          echo "## TPU v7 Skipped" >> $GITHUB_STEP_SUMMARY
          echo "TPU v7 capacity is currently not available. This run has been skipped." >> $GITHUB_STEP_SUMMARY

  benchmark:
    name: Benchmark
    runs-on: ${{ fromJSON(needs.ensure-nightly-build-image.outputs.conditional_runs_on) }}
#    container:
#      image: ghcr.io/llm-d/llm-d-benchmark:nightly
    needs: [ensure-nightly-build-image]
    if: inputs.accelerator_type != 'tpu/v7' && inputs.accelerator_type != 'tpu-v7'
    timeout-minutes: 240

    env:
      LLMDBENCH_CICD_NS: ${{ inputs.cluster_namespace || 'llmdbenchcicd' }}
      LLMDBENCH_CICD_R: ${{ inputs.helm_release || 'standalone' }}
      LLMDBENCH_CICD_SCENARIO_DIR: ${{ inputs.scenario_dir || 'cicd' }}
      LLMDBENCH_CICD_SCENARIO: ${{ inputs.standup_scenario || 'kind' }}
      LLMDBENCH_CICD_TARGET: ${{ needs.ensure-nightly-build-image.outputs.cluster_provider || 'kind' }}
      LLMDBENCH_CICD_METHOD: ${{ inputs.standup_method || 'standalone' }}
      LLMDBENCH_CICD_ACCELERATOR: ${{ inputs.accelerator_type || 'gpu' }}
      LLMDBENCH_CICD_BACKEND: ${{ inputs.backend_type || 'vllm' }}
      LLMDBENCH_CICD_INFRA_PROVIDER: ${{ inputs.infra_provider || '' }}
      LLMDBENCH_CICD_OFFLOADING_TARGET: ${{ inputs.offloading_target || '' }}
      LLMDBENCH_CICD_CONNECTOR: ${{ inputs.connector || '' }}
      LLMDBENCH_CICD_CLEANUP: ${{ inputs.cleanup || 'true' }}
      LLMDBENCH_CICD_DATA_ACCESS_TIMEOUT: ${{ inputs.data_access_timeout || '1200' }}
      LLMDBENCH_CICD_JOB_TIMEOUT: ${{ inputs.job_timeout || '2700s' }}
      LLMDBENCH_CICD_KUSTOMIZE_STANDUP_TIMEOUT: ${{ inputs.standup_timeout || '2400' }}
      LLMDBENCH_CICD_MODELSERVICE_STANDUP_TIMEOUT: ${{ inputs.standup_timeout || '2400' }}
      LLMDBENCH_CICD_STANDALONE_STANDUP_TIMEOUT: ${{ inputs.standup_timeout || '2400' }}
      LLMDBENCH_CICD_RUN_WAIT_TIMEOUT: ${{ inputs.wait_timeout || '3600' }}
      LLMDBENCH_CICD_RUN_PVC_BIND_TIMEOUT: ${{ inputs.pvc_bind_timeout || '1200' }}
      LLMDBENCH_CICD_DECODE_PODS: ${{ inputs.decode_pods || 'auto' }}
      LLMDBENCH_CICD_PREFILL_PODS: ${{ inputs.prefill_pods || 'auto' }}
      LLMDBENCH_WORKSPACE: ${{ inputs.workspace_dir || '/tmp/llmdbenchcicd' }}
      LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
      HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
      LLMDBENCH_CICD_GATEWAY_CLASS: ${{ inputs.gateway_class || 'istio' }}
      LLMDBENCH_IS_DRY_RUN: ${{ inputs.dry_run || 'false' }}
      LLMDBENCH_IS_VERBOSE: ${{ inputs.verbose || 'false' }}
      LLMDBENCH_MONITORING_ENABLED: ${{ inputs.monitoring_enabled || 'true' }}
      LLMDBENCH_CICD_HARNESS: ${{ inputs.harness || 'inference-perf' }}
      LLMDBENCH_CICD_WORKLOAD: ${{ inputs.workload || 'sanity_random.yaml' }}
      LLMDBENCH_CICD_PRIORITY_CLASS: nightly-llm-d-cicd-critical
      GCS_BASE_PATH: ${{ inputs.bucket_path || 'llm-d-benchmarks/regressions' }}
      GCP_PROJECT_ID: ${{ inputs.bucket_project || 'llm-d-scale' }}
      GKE_CLUSTER_NAME: llm-d-e2e-us-east5
      GKE_CLUSTER_ZONE: us-east5
      TPU_TOPOLOGY: ${{ inputs.tpu_topology || '2x4' }}
      TPU_CHIPS: ${{ inputs.tpu_chips || '0' }}
      TPU_NODEPOOL: ${{ inputs.tpu_nodepool || '' }}

    steps:
      - name: Checkout Code
        id: prepare_checkout_code
        uses: actions/checkout@v7.0.0
        with:
          fetch-depth: 0

      - name: Set up Python
        id: prepare_setup_python
        uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Create Kind cluster
        id: infra_create_kind_cluster
        if: env.LLMDBENCH_CICD_TARGET == 'kind'
        uses: helm/kind-action@v1
        with:
          version: v0.31.0

      - name: Authenticate to Google Cloud
        id: infra_authenticate_google_cloud
        if: inputs.bucket_provider == 'gcs' || env.LLMDBENCH_CICD_TARGET == 'gke'
        uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093
        with:
          credentials_json: ${{ secrets.GKE_SA_KEY }}

      - name: Set up gcloud CLI and kubectl
        id: infra_setup_gcloud_cli
        if: inputs.bucket_provider == 'gcs' || env.LLMDBENCH_CICD_TARGET == 'gke'
        uses: google-github-actions/setup-gcloud@aa5489c8933f4cc7a4f7d45035b3b1440c9c10db
        with:
          project_id: ${{ env.GCP_PROJECT_ID }}
          install_components: 'kubectl,gke-gcloud-auth-plugin'

      - name: Get GKE credentials
        id: infra_get_gke_credentials
        if: env.LLMDBENCH_CICD_TARGET != 'kind'
        run: |
          if [[ $LLMDBENCH_CICD_TARGET == "gke" ]]; then
            echo "Selected \"Google Kubernetes Engine\" kubeconfig"
            gcloud container clusters get-credentials "${{ env.GKE_CLUSTER_NAME }}" --zone "${{ env.GKE_CLUSTER_ZONE }}"
          fi
        shell: bash

      - name: Set up kubeconfig from secret
        id: infra_get_kubeconfig_from_secret
        if: env.LLMDBENCH_CICD_TARGET != 'kind'
        run: |
          mkdir -p ~/.kube
          if [[ $LLMDBENCH_CICD_TARGET == "cks" ]]; then
            echo "Selected \"Coreweave\" kubeconfig"
            echo "${{ secrets.KUBECONFIG_DATA_CKS }}" | base64 -d > ~/.kube/config
            chmod 600 ~/.kube/config
          fi
          if [[ $LLMDBENCH_CICD_TARGET == "ocp" ]]; then
            echo "Selected \"OpenShift\" kubeconfig"
            echo "${{ secrets.KUBECONFIG_DATA_OCP }}" | base64 -d > ~/.kube/config
            chmod 600 ~/.kube/config
          fi
        shell: bash

      - name: Install oc (optional on install.sh)
        id: prereq_install_oc
        if: env.LLMDBENCH_CICD_TARGET != 'kind'
        run: |
          if [[ $LLMDBENCH_CICD_TARGET == "ocp" ]]; then
            # Install oc (OpenShift CLI) — optional on install.sh
            if ! command -v oc &>/dev/null; then
              curl -fsSL --retry 3 --retry-delay 5 -o /tmp/openshift-client-linux.tar.gz \
                "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz"
              mkdir -p /tmp/oc-extract
              tar -xzf /tmp/openshift-client-linux.tar.gz -C /tmp/oc-extract
              sudo mv /tmp/oc-extract/oc /usr/local/bin/
              sudo mv /tmp/oc-extract/kubectl /usr/local/bin/
              sudo chmod +x /usr/local/bin/oc
              sudo chmod +x /usr/local/bin/kubectl
              rm -rf /tmp/oc-extract /tmp/openshift-client-linux.tar.gz
            fi
          fi

      - name: Install specific yq version
        id: prereq_install_yq
        uses: frenck/action-setup-yq@v1
        with:
          version: 'v4.53.3'

      - name: Show cluster names
        id: infra_show_cluster_names
        run: |
          echo "### Attempting to list cluster names..."
          if [[ -f ~/.kube/config ]]; then
            echo "### Cluster names (from \"~/.kube/config\") : $(cat ~/.kube/config | yq ".clusters[].name" | sed ':a;N;$!ba;s/\n/, /g')"
          fi
          if [[ ! -z $KUBECONFIG ]]; then
            echo "### Cluster names (from \"$KUBECONFIG\") : $(cat $KUBECONFIG | yq ".clusters[].name" | sed ':a;N;$!ba;s/\n/, /g')"
          fi
        shell: bash

      - name: Show cluster nodes
        id: infra_show_cluster_nodes
        run: |
          echo "### Attempting to list node names..."
          kubectl get node
        shell: bash

      - name: Show cluster storageclasses
        id: infra_show_cluster_storageclasses
        run: |
          echo "### Attempting to list storage classes..."
          kubectl get storageclass
          if [[ $? -ne 0 ]]; then
            exit 1
          fi
          IS_DEFAULT_SC=$(kubectl get storageclass  | grep "(default)" || true)
          if [[ -z $IS_DEFAULT_SC ]]; then
            echo "ERROR: please define a default StorageClass on your cluster"
            exit 1
          fi
          DEFAULT_SC=$(kubectl get storageclass  | grep "(default)" | cut -d "(" -f 1)
          if [[ $LLMDBENCH_CICD_TARGET == 'kind' ]]; then
            exit 0
          fi
          echo "Ensuring storageclass \"$DEFAULT_SC\" can create a pvc with accessMode \"ReadWriteMany\"..."
          cat <<PVC_EOF > pvc.yaml
          apiVersion: v1
          kind: PersistentVolumeClaim
          metadata:
            name: llm-d-cicd-test-rwx-pvc
          spec:
            accessModes:
              - ReadWriteMany
            storageClassName: $DEFAULT_SC
            resources:
              requests:
                storage: 1G
          PVC_EOF
          kubectl apply -f pvc.yaml
          cat <<POD_EOF > pod.yaml
          apiVersion: v1
          kind: Pod
          metadata:
            name: llm-d-cicd-test-pod
          spec:
            containers:
            - name: llm-d-cicd-test-pod
              image: ubuntu:latest
              command: ["sleep"]
              args: ["infinity"]
              volumeMounts:
              - name: my-storage-volume
                mountPath: /test
            volumes:
            - name: my-storage-volume
              persistentVolumeClaim:
                claimName: llm-d-cicd-test-rwx-pvc
          POD_EOF
          kubectl apply -f pod.yaml
          kubectl wait --for=jsonpath='{.status.phase}'=Bound pvc/llm-d-cicd-test-rwx-pvc --timeout=300s
          rm pod.yaml
          rm pvc.yaml
        shell: bash

      - name: Request TPUs (DWS)
        id: infra_request_tpus
        if: env.LLMDBENCH_CICD_TARGET == 'gke' && contains(inputs.accelerator_type, 'tpu') && inputs.tpu_chips > 0
        run: |
          kubectl create namespace "$LLMDBENCH_CICD_NS" || echo "Namespace already exists"

          # Map accelerator_type (tpu/v6, tpu-v6, tpu/v7, tpu-v7) to GKE accelerator resource name
          if [[ "$LLMDBENCH_CICD_ACCELERATOR" == "tpu/v6" || "$LLMDBENCH_CICD_ACCELERATOR" == "tpu-v6" ]]; then
            ACCELERATOR="tpu-v6e-slice"
          elif [[ "$LLMDBENCH_CICD_ACCELERATOR" == "tpu/v7" || "$LLMDBENCH_CICD_ACCELERATOR" == "tpu-v7" ]]; then
            ACCELERATOR="tpu7x"
          else
            echo "::error::Unsupported TPU accelerator type: $LLMDBENCH_CICD_ACCELERATOR"
            exit 1
          fi

          TOPOLOGY="$TPU_TOPOLOGY"
          CHIPS_TO_REQUEST="$TPU_CHIPS"

          CHIPS_PER_VM=8
          if [[ "$ACCELERATOR" == "tpu7x" ]]; then
            CHIPS_PER_VM=4
          fi

          VMS=$(( CHIPS_TO_REQUEST / CHIPS_PER_VM ))
          if [ "$VMS" -lt 1 ]; then
            VMS=1
          fi

          echo "Dynamic TPU request parameters calculated:"
          echo "  CHIPS_TO_REQUEST: $CHIPS_TO_REQUEST"
          echo "  ACCELERATOR: $ACCELERATOR"
          echo "  TOPOLOGY: $TOPOLOGY"
          echo "  CHIPS_PER_VM: $CHIPS_PER_VM"
          echo "  VMS: $VMS"

          cat <<EOF | kubectl apply -n "$LLMDBENCH_CICD_NS" -f -
          apiVersion: batch/v1
          kind: Job
          metadata:
            name: tpu-request-job
          spec:
            parallelism: $VMS
            completions: $VMS
            template:
              spec:
                nodeSelector:
                  cloud.google.com/gke-flex-start: "true"
                  cloud.google.com/gke-tpu-accelerator: $ACCELERATOR
                  cloud.google.com/gke-tpu-topology: "$TOPOLOGY"
                containers:
                - name: tpu-container
                  image: gcr.io/k8s-staging-perf-tests/sleep:latest
                  args: ["3600s"]
                  resources:
                    requests:
                      google.com/tpu: "$CHIPS_PER_VM"
                    limits:
                      google.com/tpu: "$CHIPS_PER_VM"
                restartPolicy: OnFailure
          EOF

          kubectl wait pod \
            --for=condition=Ready \
            -l job-name=tpu-request-job \
            -n "$LLMDBENCH_CICD_NS" \
            --timeout=$LLMDBENCH_CICD_JOB_TIMEOUT

          # Release the chips so test can use them
          kubectl delete job tpu-request-job -n "$LLMDBENCH_CICD_NS"

          # Export topology and accelerator so subsequent steps (like scenario overrides) can read them
          echo "TPU_ACCELERATOR=$ACCELERATOR" >> $GITHUB_ENV
          echo "TPU_TOPOLOGY=$TOPOLOGY" >> $GITHUB_ENV

          echo "## TPU Status — Nightly ($LLMDBENCH_CICD_SCENARIO on GKE TPU)" >> $GITHUB_STEP_SUMMARY
          echo "| Metric | Count |" >> $GITHUB_STEP_SUMMARY
          echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
          echo "| TPU Accelerator Type | ${{ inputs.accelerator_type }} |" >> $GITHUB_STEP_SUMMARY
          echo "| GKE Resource Name | $ACCELERATOR |" >> $GITHUB_STEP_SUMMARY
          echo "| TPU Topology | $TOPOLOGY |" >> $GITHUB_STEP_SUMMARY
          echo "| Requested Chips | $CHIPS_TO_REQUEST |" >> $GITHUB_STEP_SUMMARY

      - name: Create nightly PriorityClass
        id: infra_create_nightly_prioclass
        run: |
          echo "Creating PriorityClass \"$LLMDBENCH_CICD_PRIORITY_CLASS\"..."
          cat <<PRIORITY_EOF > prioclass.yaml
          apiVersion: scheduling.k8s.io/v1
          kind: PriorityClass
          metadata:
            name: $LLMDBENCH_CICD_PRIORITY_CLASS
          value: 1000000
          preemptionPolicy: PreemptLowerPriority
          globalDefault: false
          description: "High priority for nightly CI/CD llm-d workloads — preempts default-priority pods"
          PRIORITY_EOF
          kubectl apply -f prioclass.yaml
          rm prioclass.yaml

      - name: Label node for simulated accelerators
        id: infra_label_node_for_simulated_accelerators
        if: env.LLMDBENCH_CICD_TARGET == 'kind'
        run: |
          NODE=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')
          echo "### Labeling node: $NODE"
          kubectl label node "$NODE" kubernetes.io/os=linux --overwrite
        shell: bash

      - name: Set GCS full path
        id: prereq_set_gcs_full_path
        if: inputs.bucket_provider == 'gcs'
        run: |
          RUN_DATE=$(date +'%Y-%m-%d')
          GCS_FULL_PATH="gs://${GCS_BASE_PATH}/$LLMDBENCH_CICD_SCENARIO/$LLMDBENCH_CICD_TARGET/$LLMDBENCH_CICD_METHOD/$RUN_DATE/"
          GCS_FULL_PATH=$(echo "$GCS_FULL_PATH" | sed "s^$LLMDBENCH_CICD_SCENARIO/$LLMDBENCH_CICD_SCENARIO^$LLMDBENCH_CICD_SCENARIO^g")
          echo "### GCS_FULL_PATH is \"$GCS_FULL_PATH\""
          echo "GCS_FULL_PATH=$GCS_FULL_PATH" >> $GITHUB_ENV
        shell: bash

      - name: Conditionally clone llm-d from main repo
        id: prereq_conditionally_clone_llmd
        run: |
          #find ~ -type f -name "cks_nvshmem3.3.9.patch" -print
          cd ~
          if [[ ! -d ~/work/llm-d ]];
          then
            mkdir -p ~/work/llm-d/
            cd ~/work/llm-d/
            echo "### llm-d code NOT found, will clone now"
            git clone https://github.com/llm-d/llm-d.git
          else
            echo "### llm-d code is already present. Will then delete llm-d-benchmark"
            pip uninstall -y llmdbenchmark || true
            rm -rf ~/work/llm-d-benchmark/
          fi
          cd ~/work/llm-d/llm-d
          echo "### llm-d code is present (branch $(git branch --show-current))"
        shell: bash

      - name: Install llmdbenchmark
        id: prereq_install_llmdbenchmark
        run: |
          #find ~ -type f -name "br_v0_2_json_schema.json" -print
          cd ~
          if [[ ! -d ~/work/llm-d-benchmark/llm-d-benchmark ]];
          then
            mkdir -p ~/work/llm-d-benchmark/
            cd ~/work/llm-d-benchmark/
            echo "### llm-d-benchmark code NOT found, will clone now"
            git clone https://github.com/llm-d/llm-d-benchmark.git
          fi
          cd ~/work/llm-d-benchmark/llm-d-benchmark/
          if [[ ! -f install.sh ]]; then
            curl -sSL https://raw.githubusercontent.com/llm-d/llm-d-benchmark/main/install.sh -o install.sh
            chmod +x install.sh
          fi
          ./install.sh -y
          llmdbenchmark --version
          if [[ ! -f util/debug_info_on_failure.sh ]]; then
            curl -sSL https://raw.githubusercontent.com/llm-d/llm-d-benchmark/refs/heads/main/util/debug_info_on_failure.sh -o /tmp/debug_info_on_failure.sh
            chmod +x /tmp/debug_info_on_failure.sh
            sudo cp /tmp/debug_info_on_failure.sh /usr/local/bin
          else
            sudo cp util/debug_info_on_failure.sh /usr/local/bin
          fi
          echo "### llm-d-benchmark code is present (branch $(git branch --show-current))"
        shell: bash

      - name: Apply overrides to llm-d-benchmark scenarios when using kustomize (yaml)
        id: prereqs_apply_overrides_to_llmdbenchmark_scenarios
        if: env.LLMDBENCH_CICD_METHOD == 'kustomize'
        run: |
          touch ~/work/lkcmd.txt
          export LLMDBENCH_CICD_SCENARIO_FILE=~/work/llm-d-benchmark/llm-d-benchmark/config/scenarios/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO.yaml
          export LLMDBENCH_CICD_EFFECTIVE_GUIDE_NAME=$(echo $LLMDBENCH_CICD_SCENARIO | sed 's#/$##')
          echo "### Setting \"INFRA_PROVIDER\" to \"$LLMDBENCH_CICD_INFRA_PROVIDER\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
          lkcmd="yq -i \".scenario[0].kustomize.guideVariableOverrides.INFRA_PROVIDER = \\\"$LLMDBENCH_CICD_INFRA_PROVIDER\\\"\" $LLMDBENCH_CICD_SCENARIO_FILE"
          echo -n "$lkcmd" > ~/work/lkcmd.txt
          eval $lkcmd
          cat $LLMDBENCH_CICD_SCENARIO_FILE | yq '.scenario[0].kustomize.guideVariableOverrides.INFRA_PROVIDER'
          echo "### Setting \"REPO_ROOT\" to \"$HOME/work/llm-d/llm-d\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
          lkcmd="yq -i \".scenario[0].kustomize.guideVariableOverrides.REPO_ROOT = \\\"$HOME/work/llm-d/llm-d\\\"\" $LLMDBENCH_CICD_SCENARIO_FILE"
          echo -n "; $lkcmd" >> ~/work/lkcmd.txt
          eval $lkcmd
          cat $LLMDBENCH_CICD_SCENARIO_FILE | yq '.scenario[0].kustomize.guideVariableOverrides.REPO_ROOT'
          echo "### Setting \"GUIDE_NAME\" to \"$LLMDBENCH_CICD_EFFECTIVE_GUIDE_NAME\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
          lkcmd="yq -i \".scenario[0].kustomize.guideVariableOverrides.GUIDE_NAME = \\\"$LLMDBENCH_CICD_EFFECTIVE_GUIDE_NAME\\\"\" $LLMDBENCH_CICD_SCENARIO_FILE"
          echo -n "; $lkcmd" >> ~/work/lkcmd.txt
          eval $lkcmd
          cat ~/work/llm-d-benchmark/llm-d-benchmark/config/scenarios/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO.yaml | yq '.scenario[0].kustomize.guideVariableOverrides.GUIDE_NAME'
          if [[ ! -z $LLMDBENCH_CICD_CONNECTOR ]]; then
            echo "### Setting \"CONNECTOR\" to \"$LLMDBENCH_CICD_CONNECTOR\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
            lkcmd="yq -i \".scenario[0].kustomize.guideVariableOverrides.CONNECTOR = \\\"$LLMDBENCH_CICD_CONNECTOR\\\"\" $LLMDBENCH_CICD_SCENARIO_FILE"
            echo -n "; $lkcmd" >> ~/work/lkcmd.txt
            eval $lkcmd
            cat $LLMDBENCH_CICD_SCENARIO_FILE | yq '.scenario[0].kustomize.guideVariableOverrides.CONNECTOR'
          fi
          if [[ ! -z $LLMDBENCH_CICD_OFFLOADING_TARGET ]]; then
            echo "### Setting \"VARIANT\" to \"$LLMDBENCH_CICD_OFFLOADING_TARGET\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
            lkcmd="yq -i \".scenario[0].kustomize.guideVariableOverrides.VARIANT = \\\"$LLMDBENCH_CICD_OFFLOADING_TARGET\\\"\" $LLMDBENCH_CICD_SCENARIO_FILE"
            echo -n "; $lkcmd" >> ~/work/lkcmd.txt
            eval $lkcmd
            cat $LLMDBENCH_CICD_SCENARIO_FILE | yq '.scenario[0].kustomize.guideVariableOverrides.VARIANT'
          fi
          echo "### Setting \"guideName\" to \"$LLMDBENCH_CICD_EFFECTIVE_GUIDE_NAME\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
          lkcmd="yq -i \".scenario[0].kustomize.guideName = \\\"$LLMDBENCH_CICD_EFFECTIVE_GUIDE_NAME\\\"\" $LLMDBENCH_CICD_SCENARIO_FILE"
          echo -n "; $lkcmd" >> ~/work/lkcmd.txt
          eval $lkcmd
          cat $LLMDBENCH_CICD_SCENARIO_FILE | yq '.scenario[0].kustomize.guideName'
          echo "### Setting \"repoPath\" to \"$HOME/work/llm-d/llm-d\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
          lkcmd="yq -i \".scenario[0].kustomize.repoPath = \\\"$HOME/work/llm-d/llm-d\\\"\" $LLMDBENCH_CICD_SCENARIO_FILE"
          echo -n "; $lkcmd" >> ~/work/lkcmd.txt
          eval $lkcmd
          cat $LLMDBENCH_CICD_SCENARIO_FILE | yq '.scenario[0].kustomize.repoPath'
          echo "### Setting \"NAMESPACE\" to \"$LLMDBENCH_CICD_NS\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
          lkcmd="yq -i \".scenario[0].kustomize.guideVariableOverrides.NAMESPACE = \\\"$LLMDBENCH_CICD_NS\\\"\" $LLMDBENCH_CICD_SCENARIO_FILE"
          echo -n "; $lkcmd" >> ~/work/lkcmd.txt
          eval $lkcmd
          cat ~/work/llm-d-benchmark/llm-d-benchmark/config/scenarios/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO.yaml | yq '.scenario[0].kustomize.guideVariableOverrides.NAMESPACE'
          export LLMDBENCH_CICD_BACKEND_ACCELERATOR=$(echo $LLMDBENCH_CICD_ACCELERATOR/$LLMDBENCH_CICD_BACKEND | sed 's^//^/^g')
          echo "### Setting \"acceleratorBackend\" to \"$LLMDBENCH_CICD_BACKEND_ACCELERATOR\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
          lkcmd="yq -i \".scenario[0].kustomize.acceleratorBackend = \\\"$LLMDBENCH_CICD_BACKEND_ACCELERATOR\\\"\" $LLMDBENCH_CICD_SCENARIO_FILE"
          echo -n "; $lkcmd" >> ~/work/lkcmd.txt
          eval $lkcmd
          cat $LLMDBENCH_CICD_SCENARIO_FILE | yq '.scenario[0].kustomize.acceleratorBackend'
        shell: bash

      - name: Set harness executable
        id: prereqs_set_harness_executable
        if: env.LLMDBENCH_CICD_HARNESS == 'nop' && env.LLMDBENCH_CICD_SCENARIO_DIR == 'cicd'
        run: |
            export LLMDBENCH_CICD_SCENARIO_FILE=~/work/llm-d-benchmark/llm-d-benchmark/config/scenarios/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO.yaml
            echo " ### Setting harness executable to \"llm-d-benchmark.py\" in \"$LLMDBENCH_CICD_SCENARIO_FILE\""
            yq -i '.scenario[0].harness.executable = "llm-d-benchmark.py"' $LLMDBENCH_CICD_SCENARIO_FILE

      - name: Apply overrides to kustomize manifests
        id: prereqs_apply_overrides_to_kustomize_manifests
        if: env.LLMDBENCH_CICD_METHOD == 'kustomize'
        run: |
          LLMDBENCH_CICD_KUSTOMIZE_PATH=~/work/llm-d/llm-d/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO/modelserver/$LLMDBENCH_CICD_ACCELERATOR/$LLMDBENCH_CICD_BACKEND/$LLMDBENCH_CICD_CONNECTOR/$LLMDBENCH_CICD_OFFLOADING_TARGET/$LLMDBENCH_CICD_INFRA_PROVIDER
          echo "Kustomize full path is \"$LLMDBENCH_CICD_KUSTOMIZE_PATH\". Will now attempt to detect model name..."
          detected_model=$(kubectl kustomize $LLMDBENCH_CICD_KUSTOMIZE_PATH | yq .spec.template.spec.containers[0].args | grep -Ev "\- \--|\- '|\---|\- \||    \--|.*exec vllm|'|null|#|\"|find|START_RANK" | sed -e 's^ ^^g' -e 's|^-||g' -e 's^\\^^g' -e '/^$/d' | uniq || true)
          if [[ -z $detected_model ]]; then
            detected_model=$(kubectl kustomize $LLMDBENCH_CICD_KUSTOMIZE_PATH |  yq .spec.leaderWorkerTemplate.workerTemplate.spec.containers[0].args | grep -Ev "\- \--|\- '|\---|\- \||    \--|.*exec vllm|'|null|#|\"|find|START_RANK" | sed -e 's^ ^^g' -e 's|^-||g' -e 's^\\^^g' -e '/^$/d' | uniq || true)
          fi
          if [[ -z $detected_model ]]; then
            echo "### ERROR: unable to detect model"
            exit 1
          fi
          echo "### LLMDBENCH_CICD_DETECTED_MODEL is \"$detected_model\""
          echo "LLMDBENCH_CICD_DETECTED_MODEL=$detected_model"  >> $GITHUB_ENV
          if [[ $LLMDBENCH_CICD_SCENARIO == "predicted-latency-routing" || $LLMDBENCH_CICD_SCENARIO == "flow-control" ]]; then
            find ~/work/llm-d/llm-d/$LLMDBENCH_CICD_SCENARIO_DIR/optimized-baseline/modelserver/ -type f -name "kustomization.yaml" -exec sed -i "s/optimized-baseline/$LLMDBENCH_CICD_SCENARIO/g" {} \;
            find ~/work/llm-d/llm-d/$LLMDBENCH_CICD_SCENARIO_DIR/optimized-baseline/modelserver/ -type f -name "kustomization.yaml" -exec bash -c 'echo -n $1:; yq ".namePrefix" $1' _ {} \;
            find ~/work/llm-d/llm-d/$LLMDBENCH_CICD_SCENARIO_DIR/optimized-baseline/modelserver/ -type f -name "patch-vllm.yaml" -exec sed -i "s/optimized-baseline/$LLMDBENCH_CICD_SCENARIO/g" {} \;
            find ~/work/llm-d/llm-d/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO/$LLMDBENCH_CICD_CONNECTOR/router/ -type f -name "*.yaml" -exec sed -i "s/optimized-baseline/$LLMDBENCH_CICD_SCENARIO/g" {} \;
          fi
          echo "### Setting priority class \"$LLMDBENCH_CICD_PRIORITY_CLASS\" on scenario \"$LLMDBENCH_CICD_SCENARIO\""
          for i in vllm decode prefill; do
            find ~/work/llm-d/llm-d/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO/modelserver/ -type f -name "patch-$i.yaml" -exec yq ".spec.template.spec.priorityClassName=\"$LLMDBENCH_CICD_PRIORITY_CLASS\"" -i {} \;
            find ~/work/llm-d/llm-d/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO/modelserver/ -type f -name "patch-$i.yaml" -exec bash -c 'echo -n $1:; yq ".spec.template.spec.priorityClassName" $1' _ {} \;
          done
          if [[ $LLMDBENCH_CICD_DECODE_PODS != "auto" ]]; then
            echo "### Setting decode pods to \"$LLMDBENCH_CICD_DECODE_PODS\" on scenario \"$LLMDBENCH_CICD_SCENARIO\""
            for i in vllm decode; do
              find ~/work/llm-d/llm-d/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO/modelserver/ -type f -name "patch-$i.yaml" -exec yq ".spec.replicas=$LLMDBENCH_CICD_DECODE_PODS" -i {} \;
              find ~/work/llm-d/llm-d/$LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO/modelserver/ -type f -name "patch-$i.yaml" -exec bash -c 'echo -n $1:; yq ".spec.replicas" $1' _ {} \;
            done
          fi
          if [[ $LLMDBENCH_CICD_PREFILL_PODS != "auto" ]]; then
            echo "### Setting decode pods to \"$LLMDBENCH_CICD_PREFILL_PODS\" on scenario \"$LLMDBENCH_CICD_SCENARIO\""
            for i in prefill; do
              find $LLMDBENCH_CICD_KUSTOMIZE_PATH -type f -name "patch-$i.yaml" -exec yq ".spec.replicas=$LLMDBENCH_CICD_DECODE_PODS" -i {} \;
              find $LLMDBENCH_CICD_KUSTOMIZE_PATH -type f -name "patch-$i.yaml" -exec bash -c 'echo -n $1:; yq ".spec.replicas" $1' _ {} \;
            done
          fi
        shell: bash

      - name: Set up extra command line parameters
        id: prereqs_setup_extrac_cli_parms_llmdbenchmark
        run: |
          export LLMDBENCH_EXTRA_CMD=""
          export LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD=""
          export LLMDBENCH_EXTRA_RUN_OPERATION_CMD=""
          export LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD=""
          if [[ $LLMDBENCH_CICD_METHOD == "kustomize" ]]; then
            export LLMDBENCH_EXTRA_CMD="--non-admin"
          fi
          if [[ "$LLMDBENCH_IS_DRY_RUN" == "true" ]]; then
            export LLMDBENCH_EXTRA_CMD="--dry-run"
          fi
          if [[ "$LLMDBENCH_IS_VERBOSE" == "true" ]]; then
            export LLMDBENCH_EXTRA_CMD="--verbose $LLMDBENCH_EXTRA_CMD "
          fi
          if [[ $LLMDBENCH_CICD_METHOD == "kustomize" ]]; then
            export LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD="--kustomize-deploy-timeout $LLMDBENCH_CICD_KUSTOMIZE_STANDUP_TIMEOUT --modelservice-deploy-timeout $LLMDBENCH_CICD_MODELSERVICE_STANDUP_TIMEOUT --standalone-deploy-timeout $LLMDBENCH_CICD_STANDALONE_STANDUP_TIMEOUT $LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD "
          fi
          export LLMDBENCH_EXTRA_RUN_OPERATION_CMD="--wait-timeout $LLMDBENCH_CICD_RUN_WAIT_TIMEOUT --pvc-bind-timeout $LLMDBENCH_CICD_RUN_PVC_BIND_TIMEOUT --data-access-timeout $LLMDBENCH_CICD_DATA_ACCESS_TIMEOUT $LLMDBENCH_EXTRA_RUN_OPERATION_CMD "
          if [[ ! -z $LLMDBENCH_CICD_DETECTED_MODEL ]]; then
            export LLMDBENCH_EXTRA_RUN_OPERATION_CMD="--model $LLMDBENCH_CICD_DETECTED_MODEL $LLMDBENCH_EXTRA_RUN_OPERATION_CMD "
          fi
          if [[ "$LLMDBENCH_MONITORING_ENABLED" == "false" ]]; then
            export LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD="--no-monitoring $LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD "
            export LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD="$LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD "
          fi
          if [[ ! -z $LLMDBENCH_CICD_DETECTED_MODEL ]]; then
            export LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD="--models $LLMDBENCH_CICD_DETECTED_MODEL $LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD "
            export LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD="--models $LLMDBENCH_CICD_DETECTED_MODEL $LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD "
          fi
          echo "### LLMDBENCH_EXTRA_CMD is \"$LLMDBENCH_EXTRA_CMD\""
          echo "LLMDBENCH_EXTRA_CMD=$LLMDBENCH_EXTRA_CMD" >> $GITHUB_ENV
          echo "### LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD is \"$LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD\""
          echo "LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD=$LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD" >> $GITHUB_ENV
          echo "### LLMDBENCH_EXTRA_RUN_OPERATION_CMD is \"$LLMDBENCH_EXTRA_RUN_OPERATION_CMD\""
          echo "LLMDBENCH_EXTRA_RUN_OPERATION_CMD=$LLMDBENCH_EXTRA_RUN_OPERATION_CMD" >> $GITHUB_ENV
          echo "### LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD is \"$LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD\""
          echo "LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD=$LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD" >> $GITHUB_ENV
          export LLMDBENCH_CICD_GATEWAY_CLASS=$(echo "$LLMDBENCH_CICD_GATEWAY_CLASS" | sed 's^standalone^epponly^g')
          echo "LLMDBENCH_CICD_GATEWAY_CLASS is \"$LLMDBENCH_CICD_GATEWAY_CLASS\""
          echo "LLMDBENCH_CICD_GATEWAY_CLASS=$LLMDBENCH_CICD_GATEWAY_CLASS"  >> $GITHUB_ENV
        shell: bash

      - name: Cleanup target cloud
        id: guide_teardown
        run: |
          cd ~/work/llm-d-benchmark/llm-d-benchmark/
          LLMDBENCH_CMD="llmdbenchmark --spec $LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO $LLMDBENCH_EXTRA_CMD teardown $LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD --namespace $LLMDBENCH_CICD_NS --release $LLMDBENCH_CICD_R --gateway-class $LLMDBENCH_CICD_GATEWAY_CLASS --methods $LLMDBENCH_CICD_METHOD"
          echo "### $(cat ~/work/lkcmd.txt); $LLMDBENCH_CMD ###"
          eval $LLMDBENCH_CMD
          kubectl -n "$LLMDBENCH_CICD_NS" exec -i access-to-harness-data-workload-pvc -- bash -c "rm -rf /requests/*" || true
        shell: bash

      - name: Handle namespace and secret (HF_TOKEN) creation in case of kustomize
        id: infra_handle_namesace_and_secret
        run: |
          cd ~
          touch ~/work/lkcmd.txt
          kubectl create namespace $LLMDBENCH_CICD_NS --dry-run=client -o yaml | kubectl apply -f -
          if [[ $LLMDBENCH_CICD_METHOD == "kustomize" ]]; then
            if [[ -z $LLMDBENCH_HF_TOKEN ]]; then
              echo "#### Assigning fake value to LLMDBENCH_HF_TOKEN"
              export LLMDBENCH_HF_TOKEN=hf_zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
            fi
            echo -n $LLMDBENCH_HF_TOKEN > hf_token.txt
            IS_PLAINTEXT=$(echo $LLMDBENCH_HF_TOKEN | grep "hf_" || true)
            if [[ -z $IS_PLAINTEXT ]]; then
              echo -n $LLMDBENCH_HF_TOKEN | base64 -d > hf_token.txt
            fi
            kubectl create secret generic llm-d-hf-token --from-file=HF_TOKEN=hf_token.txt --namespace "${NAMESPACE}" --dry-run=client -o yaml | kubectl apply -f -
            rm -rf hf_token.txt
          fi
        shell: bash

      - name: Standup
        id: guide_standup
        run: |
          cd ~/work/llm-d-benchmark/llm-d-benchmark/
          LLMDBENCH_CMD="llmdbenchmark --spec $LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO $LLMDBENCH_EXTRA_CMD standup $LLMDBENCH_EXTRA_STANDUP_OPERATION_CMD --namespace $LLMDBENCH_CICD_NS --gateway-class $LLMDBENCH_CICD_GATEWAY_CLASS --release $LLMDBENCH_CICD_R --methods $LLMDBENCH_CICD_METHOD"
          echo "### $(cat ~/work/lkcmd.txt); $LLMDBENCH_CMD ###"
          eval $LLMDBENCH_CMD
        shell: bash

      - name: Debug info (on failure)
        id: infra_debug_info
        if: failure()
        run: |
          if [[ -f /usr/local/bin/debug_info_on_failure.sh ]]; then
            /usr/local/bin/debug_info_on_failure.sh "$LLMDBENCH_CICD_NS"
          else
            echo "### Unable to obtain debug info"
          fi
        shell: bash

      - name: Run
        id: benchmark_run
        run: |
          cd ~/work/llm-d-benchmark/llm-d-benchmark/
          if [[ "$LLMDBENCH_CICD_TARGET" == "kind" && "${{ inputs.standup_method }}" == "fma" ]]; then
            echo "Temporarily unable to test \"fma\" on \"kind\" clusters"
          elif [[ "$LLMDBENCH_CICD_TARGET" == "gke" && "${{ inputs.standup_method }}" == "fma" ]]; then
            echo "Temporarily unable to test \"fma\" on \"gke\" clusters"
          elif [[ "$LLMDBENCH_CICD_TARGET" == "cks" && "${{ inputs.standup_method }}" == "fma" ]]; then
            echo "Temporarily unable to test \"fma\" on \"cks\" clusters"
          else
            LLMDBENCH_CMD="llmdbenchmark --spec $LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO $LLMDBENCH_EXTRA_CMD run $LLMDBENCH_EXTRA_RUN_OPERATION_CMD --namespace $LLMDBENCH_CICD_NS --gateway-class $LLMDBENCH_CICD_GATEWAY_CLASS --methods $LLMDBENCH_CICD_METHOD --harness $LLMDBENCH_CICD_HARNESS --workload $LLMDBENCH_CICD_WORKLOAD"
            echo "### $(cat ~/work/lkcmd.txt); $LLMDBENCH_CMD ###"
            eval $LLMDBENCH_CMD
          fi
        shell: bash

      - name: Teardown
        id: guide_teardown_again
        if: always()
        run: |
          if [[ $LLMDBENCH_CICD_CLEANUP == 'false' ]]; then
            exit 0
          fi
          cd ~/work/llm-d-benchmark/llm-d-benchmark/
          LLMDBENCH_CMD="llmdbenchmark --spec $LLMDBENCH_CICD_SCENARIO_DIR/$LLMDBENCH_CICD_SCENARIO $LLMDBENCH_EXTRA_CMD teardown $LLMDBENCH_EXTRA_TEARDOWN_OPERATION_CMD  --namespace $LLMDBENCH_CICD_NS --gateway-class $LLMDBENCH_CICD_GATEWAY_CLASS --release $LLMDBENCH_CICD_R --methods $LLMDBENCH_CICD_METHOD"
          echo "### $LLMDBENCH_CMD ###"
          $LLMDBENCH_CMD
          kubectl -n "$LLMDBENCH_CICD_NS" exec -i access-to-harness-data-workload-pvc -- bash -c "rm -rf /requests/*" || true
        shell: bash

      - name: Debug info (on failure)
        id: infra_debug_info_again
        if: failure()
        run: |
          if [[ -f /usr/local/bin/debug_info_on_failure.sh ]]; then
            /usr/local/bin/debug_info_on_failure.sh "$LLMDBENCH_CICD_NS"
          else
            echo "### Unable to obtain debug info"
          fi
        shell: bash

      - name: Upload results to GCS
        id: infra_upload_results_to_gcs
        if: env.LLMDBENCH_CICD_TARGET != 'kind'
        run: |
          if [[ "${{ inputs.dry_run }}" == "true" ]]; then
            echo "Skipping data upload to bucket, due to \"dry-run\" mode activated"
          elif [[ "${{ inputs.workload }}" == "sanity_random.yaml" ]]; then
            echo "Skipping data upload to bucket, workload is \"sanity_random.yml\""
          else
            cd "$LLMDBENCH_WORKSPACE"
            REG_RESULTS=$(find . -name results | sed "s^/results$^^g" | cut -d '/' -f 2)
            find $REG_RESULTS/ -name run_metadata.yaml -exec sh -c "echo 'github_run_id: \"${{ github.run_id }}\"' >> {}" \;
            echo "Uploading data from directory \"REG_RESULTS\" to \"$GCS_FULL_PATH\""
            gcloud storage cp -r $REG_RESULTS/ "$GCS_FULL_PATH" || true
          fi
        shell: bash

      - name: Block waiting for Standalone baseline
        id: infra_block_waiting_for_standalone_baseline
        if: env.LLMDBENCH_CICD_TARGET == 'ocp' && inputs.bucket_provider == 'gcs' && inputs.standup_method == 'fma'
        run: |
          if [[ "${{ inputs.dry_run }}" == "true" ]]; then
            echo "Skipping waiting for \"standalone baseline\", due to \"dry-run\" mode activated"
            exit 0
          else
            RUN_DATE=$(date +'%Y-%m-%d')
            POLL=30
            while true; do
              # Match different standalone outputs
              brl=$(gcloud storage ls -l -r "gs://${GCS_BASE_PATH}/$LLMDBENCH_CICD_SCENARIO/standalone/$RUN_DATE/*/results/*/benchmark_report*" 2>/dev/null)
              if [[ $? -ne 0 ]]; then
                echo "Standalone baseline for \"$RUN_DATE\" still not present. Waiting $POLL seconds and trying again..."
                sleep $POLL
              else
                latest_entry=$(echo "$brl" | sort -k 2 | tail -n 2 | head -n 1 | awk '{ print $3 }')
                latest_dir=$(echo "$latest_entry" | sed "s^gs://${GCS_BASE_PATH}/$LLMDBENCH_CICD_SCENARIO/standalone/$RUN_DATE/^^g" | cut -d '/' -f 1)
                echo "Standalone baseline for \"$RUN_DATE\" is \"$latest_dir\"."
                exit 0
              fi
            done
          fi
          exit 1
        shell: bash