benchmark1.yaml

name: Run Benchmark

on:
  workflow_dispatch:
    inputs:
      input_dir:
        description: 'Input directory for benchmark results'
        required: false
        default: '/tmp/cicd/analysis'
      output_dir:
        description: 'Output directory name (S3 prefix and artifact name)'
        required: false
        default: ''

  push:
    branches:
      - main

  schedule:
    - cron: '0 0 * * *'  # Daily at midnight UTC

jobs:
  run-benchmark:
    name: Benchmark Test
    runs-on: [k8s-util]
    timeout-minutes: 240

    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Display OS used
        run: |
          cat /etc/*os-*
        shell: bash

      - name: Set input and output directory environment variables
        run: |
          DEFAULT_INPUT_DIR=/tmp/cicd/analysis
          INPUT_DIR="${{ github.event.inputs.input_dir }}"
          if [ -z "$INPUT_DIR" ]; then
            INPUT_DIR="$DEFAULT_INPUT_DIR"
          fi
          echo "INPUT_DIR=$INPUT_DIR" >> $GITHUB_ENV

          if [ -z "${{ github.event.inputs.output_dir }}" ]; then
            timestamp=$(date -u +%Y%m%dT%H%M%SZ)
            echo "OUTPUT_DIR=benchmark-results-${timestamp}" >> $GITHUB_ENV
            echo "Using generated output dir: benchmark-results-${timestamp}"
          else
            echo "OUTPUT_DIR=${{ github.event.inputs.output_dir }}" >> $GITHUB_ENV
            echo "Using provided output dir: ${{ github.event.inputs.output_dir }}"
          fi

      - name: Set up kubeconfig from secret
        run: |
          mkdir -p ~/.kube
          echo "${{ secrets.KUBECONFIG_DATA }}" | base64 -d > ~/.kube/config
          chmod 600 ~/.kube/config
        shell: bash

      - name: Install yq
        run: |
          export VERSION=v4.45.4
          export BINARY=yq_linux_amd64
          curl -L https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -o ${BINARY}
          chmod +x ${BINARY}
          sudo cp -f $(which yq) || sudo cp -f ${BINARY} /usr/local/bin/yq

      - name: Install make, skopeo, curl, jq
        run: |
          sudo apt-get update
          sudo apt-get install -y make skopeo curl jq rsync
        shell: bash

      - name: Install oc
        run: |
          OC_FILE_NAME=openshift-client-$(uname -s | sed -e "s/Linux/linux/g" -e "s/Darwin/apple-darwin/g")$(echo "-$(uname -m)" | sed -e 's/-x86_64//g' -e 's/-amd64//g' -e 's/aarch64/arm64-rhel9/g').tar.gz
          curl https://mirror.openshift.com/pub/openshift-v4/$(uname -m)/clients/ocp/stable/$OC_FILE_NAME  -o $OC_FILE_NAME
          tar xzf $OC_FILE_NAME
          sudo mv oc /usr/local/bin/
          sudo mv kubectl /usr/local/bin/
          sudo chmod +x /usr/local/bin/oc
          sudo chmod +x /usr/local/bin/kubectl
          rm openshift-client-*.tar.gz

      - name: Install Kustomize
        uses: multani/action-setup-kustomize@v1
        with:
          version: 5.6.0

      - name: Populate python deps
        run: |
          echo -e "pandas\ngrip>=4.6.0\nmatplotlib>=3.7.0\nnumpy>=1.22.0\nseaborn>=0.12.0\nkubernetes>=28.0.0" > requirements.txt

      - name: Install python deps
        uses: actions/setup-python@v5
        with:
          python-version: '3.13'
          cache: 'pip'
      - run: pip install -r requirements.txt

      - name: Install Helm
        run: |
          curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 && chmod 700 get_helm.sh && ./get_helm.sh
        shell: bash

      - name: Cleanup target cloud (standalone)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/teardown.sh -c cicd -t standalone

      - name: Cleanup target cloud (deployer)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/teardown.sh -c cicd -t deployer -d

      - name: Standup (deployer)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/standup.sh -c cicd -t deployer

      - name: Run benchmark
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/run.sh -c cicd -t deployer

      - name: Cleanup target cloud (deployer)
        env:
          LLMDBENCH_HF_TOKEN: ${{ secrets.LLMDBENCH_HF_TOKEN }}
        run: ./setup/teardown.sh -c cicd -t deployer -d

      - name: Install AWS CLI
        run: |
          curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
          unzip awscliv2.zip
          sudo ./aws/install
          aws --version

      - name: Upload results to IBM COS
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          aws configure set default.s3.signature_version s3v4
          aws s3 cp "$INPUT_DIR" "s3://${{ secrets.COS_BUCKET_NAME }}/$OUTPUT_DIR/" \
            --recursive --endpoint-url ${{ secrets.COS_ENDPOINT_URL }}

      - name: Archive benchmark results as GitHub artifact
        if: success() || failure()
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.OUTPUT_DIR }}
          path: ${{ env.INPUT_DIR }}
          retention-days: 14