load-tests.yml

name: load-tests

on:
  push:
    tags:
      - "v[0-9]+.[0-9]+.[0-9]+*"
  workflow_dispatch:
  schedule:
    - cron: "27 0,12 * * *" # Run at 00:27 and 12:27 UTC every day.

# Do not cancel this workflow on main. See https://github.com/open-telemetry/opentelemetry-collector-contrib/pull/16616
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  # Make sure to exit early if cache segment download times out after 2 minutes.
  # We limit cache download as a whole to 5 minutes.
  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 2

permissions: read-all

jobs:
  setup-environment:
    timeout-minutes: 30
    runs-on: oracle-bare-metal-64cpu-1024gb-x86-64-ubuntu-24
    if: ${{ github.actor != 'dependabot[bot]' &&  github.repository_owner == 'open-telemetry' }}
    outputs:
      loadtest_matrix: ${{ steps.splitloadtest.outputs.loadtest_matrix }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
      - uses: ./.github/actions/setup-go-tools
        with:
          go-version: oldstable
      - run: make genoteltestbedcol
      - run: make oteltestbedcol
      - name: Upload Testbed Binaries
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: testbed-binaries
          path: ./bin/*
      - name: Split Loadtest Jobs
        id: splitloadtest
        run: ./.github/workflows/scripts/setup_e2e_tests.sh
      - run: ./.github/workflows/scripts/check-disk-space.sh

  loadtest:
    runs-on: oracle-bare-metal-64cpu-1024gb-x86-64-ubuntu-24
    needs: [setup-environment]
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.setup-environment.outputs.loadtest_matrix) }}
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
      - uses: ./.github/actions/setup-go-tools
        with:
          go-version: oldstable
      - run: mkdir -p results && touch results/TESTRESULTS.md
      - name: Download Testbed Binaries
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: testbed-binaries
          path: bin/
      - run: chmod +x bin/*
      # Remove any pre-installed kwokctl so the action installs the pinned version
      - name: Remove existing KWOK binaries
        run: |
          sudo rm -f /usr/local/bin/kwokctl /usr/local/bin/kwok
      - name: Install KWOK (kwokctl)
        uses: kubernetes-sigs/kwok@c04392afe3cc6929cd3cc79f61d8bdcd83a8ba5d # v0.7.0
        with:
          command: kwokctl
      - name: Loadtest
        run: make -C testbed run-tests
        env:
          TEST_ARGS: "-test.run=${{ matrix.test }}"
      - name: Set results filename
        if: ${{ failure() || success() }}
        id: filename
        run: echo "name=$(echo '${{ matrix.test }}' | sed -e 's/|/_/g')" >> "$GITHUB_OUTPUT"
      - name: Create Test Result Archive
        if: ${{ failure() || success() }}
        continue-on-error: true
        run: tar -cvf test_results_${{steps.filename.outputs.name}}.tar testbed/tests/results
      - name: Upload Test Results
        if: ${{ failure() || success() }}
        continue-on-error: true
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: test-result-archive-${{steps.filename.outputs.name}}
          path: ./*.tar
      - run: cp testbed/tests/results/benchmarks.json testbed/tests/results/${{steps.filename.outputs.name}}.json
      - name: Upload benchmarks.json
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
        with:
          name: benchmark-results-${{steps.filename.outputs.name}}
          path: testbed/tests/results/${{steps.filename.outputs.name}}.json
      - run: ./.github/workflows/scripts/check-disk-space.sh

  update-benchmarks:
    runs-on: ubuntu-24.04
    needs: [loadtest]
    permissions:
      # deployments permission to deploy GitHub pages website
      deployments: write
      # contents permission to update benchmark contents in gh-pages branch
      contents: write
    if: github.event_name != 'pull_request'
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: benchmark-results-*
          merge-multiple: true
          path: results
      - run: jq -s 'map(.[])' results/*.json > output.json
      - uses: benchmark-action/github-action-benchmark@d48d326b4ca9ba73ca0cd0d59f108f9e02a381c7 # v1
        with:
          tool: "customSmallerIsBetter"
          output-file-path: output.json
          gh-pages-branch: benchmarks
          max-items-in-chart: 100
          github-token: ${{ secrets.GITHUB_TOKEN }}
          benchmark-data-dir-path: "docs/benchmarks/loadtests"
          auto-push: true

  notify-failure:
    runs-on: ubuntu-24.04
    needs: [loadtest]
    if: >-
      failure() &&
      github.event_name != 'pull_request' &&
      github.repository_owner == 'open-telemetry'
    permissions:
      issues: write
    steps:
      - name: Download test result archives
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: test-result-archive-*
          path: /tmp/test-results/
        continue-on-error: true
      - name: Check for existing issue
        id: check
        run: |
          issue_number=$(gh issue list -R ${{ github.repository }} \
            --search "load-tests workflow failed" \
            --state open \
            --label "flaky tests" \
            --json number \
            --jq '.[0].number // empty')
          if [ -n "$issue_number" ]; then
            echo "exists=true" >> "$GITHUB_OUTPUT"
            echo "number=$issue_number" >> "$GITHUB_OUTPUT"
          else
            echo "exists=false" >> "$GITHUB_OUTPUT"
          fi
        env:
          GH_TOKEN: ${{ github.token }}
      - name: Collect failure details
        id: details
        run: |
          run_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          short_sha=$(echo "${{ github.sha }}" | cut -c1-7)
          commit_url="${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}"

          # Extract PR number from merge commit message (contains "(#NNNNN)")
          commit_msg=$(gh api "repos/${{ github.repository }}/commits/${{ github.sha }}" --jq '.commit.message' 2>/dev/null || true)
          pr_number=$(echo "$commit_msg" | head -1 | grep -oP '\(#\K\d+' | head -1 || true)

          # Get failed job names (excluding notify-failure itself)
          failed_jobs=$(gh run view "${{ github.run_id }}" -R "${{ github.repository }}" \
            --json jobs \
            -q '[.jobs[] | select(.conclusion == "failure" and (.name | startswith("notify-failure") | not)) | .name] | map("- " + .) | join("\n")')

          # Extract performance table and test failures from archived test results
          performance_table=""
          failure_msgs=""
          for tarfile in /tmp/test-results/*/*.tar; do
            tar -xf "$tarfile" testbed/tests/results/TESTRESULTS.md 2>/dev/null || true
            tar -xf "$tarfile" testbed/tests/results/testoutput.log 2>/dev/null || true
          done
          if [ -f testbed/tests/results/TESTRESULTS.md ]; then
            performance_table=$(cat testbed/tests/results/TESTRESULTS.md)
          fi
          if [ -f testbed/tests/results/testoutput.log ]; then
            failure_msgs=$(grep -E '(--- FAIL|^FAIL)' testbed/tests/results/testoutput.log | head -20)
          fi

          # Build issue body to a file to avoid shell escaping issues
          {
            echo "Auto-generated report for load-tests workflow."
            echo ""
            echo "Link to failed build: ${run_url}"
            echo "Commit: [${short_sha}](${commit_url})"
            if [ -n "$pr_number" ]; then
              echo "PR: [#${pr_number}](${{ github.server_url }}/${{ github.repository }}/pull/${pr_number})"
            fi
            echo ""
            echo "### Component(s)"
            echo ""
            echo "testbed"
            echo ""
            echo "### Failed jobs"
            echo ""
            echo "$failed_jobs"
            if [ -n "$performance_table" ]; then
              echo ""
              echo "### Performance results"
              echo ""
              echo '```'
              echo "$performance_table"
              echo '```'
            fi
            if [ -n "$failure_msgs" ]; then
              echo ""
              echo "#### Test failures"
              echo ""
              echo '```'
              echo "$failure_msgs"
              echo '```'
            fi
            echo ""
            echo "**Note**: Information about any subsequent build failures that happen while"
            echo "this issue is open, will be added as comments with more information to this issue."
          } > /tmp/issue-body.md
        env:
          GH_TOKEN: ${{ github.token }}
      - name: Create issue
        if: steps.check.outputs.exists == 'false'
        run: |
          gh issue create -R "${{ github.repository }}" \
            -t 'load-tests workflow failed' \
            --body-file /tmp/issue-body.md \
            -l 'needs triage' \
            -l 'flaky tests' \
            -l 'ci-cd'
        env:
          GH_TOKEN: ${{ github.token }}
      - name: Comment on existing issue
        if: steps.check.outputs.exists == 'true'
        run: |
          gh issue comment "${{ steps.check.outputs.number }}" \
            -R "${{ github.repository }}" \
            --body-file /tmp/issue-body.md
        env:
          GH_TOKEN: ${{ github.token }}