benchmarks.yml

name: benchmarks

# Secret access requires allowlist in https://github.com/elastic/oblt-infra/tree/main/conf/resources/repos/apm-server

# To call benchmarks workflow programmatically with different params, use the scripts in .github/workflows/benchmarks

on:
  workflow_dispatch:
    inputs:
      runStandalone:
        description: 'Benchmark against standalone APM Server with Moxy'
        required: false
        type: boolean
        default: false
      enableTailSampling:
        description: 'Enable tail-based sampling on APM server'
        required: false
        type: boolean
        default: false
      tailSamplingStorageLimit:
        description: 'Storage size limit of tail-based sampling on APM server'
        required: false
        type: string
        default: "10GB"
      tailSamplingSampleRate:
        description: 'Blanket sample rate used by tail-based sampling'
        required: false
        type: string
        default: "0.1"
      profile:
        description: 'System profile for benchmark, e.g. system-profiles/8GBx1zone.tfvars'
        required: false
        type: string
        # defaults set below (TFVARS_SOURCE)
      runOnStable:
        description: 'Benchmark on latest stable version instead of a build from commit'
        required: false
        type: boolean
        default: false
      benchmarkAgents:
        description: 'Number of benchmark agents sending data to APM Server'
        required: false
        type: string
        # defaults set in Makefile
      benchmarkRun:
        description: 'Benchmark scenarios that only match regex, e.g. BenchmarkAgentAll'
        required: false
        type: string
        # defaults set in Makefile
      warmupTime:
        description: 'Benchmark warmup time for APM Server e.g. 3s, 5m, 2h'
        required: false
        type: string
        # defaults set below (Set up env)
      pgoExport:
        description: 'Benchmark with PGO export'
        required: false
        type: boolean
        default: false
  schedule:
    - cron: '0 17 * * 3' # Scheduled regular benchmarks.
    - cron: '0 0 1 * *' # Scheduled PGO benchmarks.

env:
  PNG_REPORT_FILE: out.png
  BENCHMARK_CPU_OUT: default.pgo
  BENCHMARK_RESULT: benchmark-result.txt
  WORKING_DIRECTORY: testing/benchmark

permissions:
  contents: read

jobs:
  benchmarks:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ${{ env.WORKING_DIRECTORY }}
    permissions:
      contents: write
      id-token: write
    env:
      SSH_KEY: ./id_rsa_terraform
      TF_VAR_private_key: ./id_rsa_terraform
      TF_VAR_public_key: ./id_rsa_terraform.pub
      TF_VAR_run_standalone: ${{ inputs.runStandalone || github.event.schedule=='0 0 1 * *' }}
      TF_VAR_apm_server_tail_sampling: ${{ inputs.enableTailSampling || 'false' }} # set the default again otherwise schedules won't work
      TF_VAR_apm_server_tail_sampling_storage_limit: ${{ inputs.tailSamplingStorageLimit || '10GB' }} # set the default again otherwise schedules won't work
      TF_VAR_apm_server_tail_sampling_sample_rate: ${{ inputs.tailSamplingSampleRate || '0.1' }} # set the default again otherwise schedules won't work
      RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 0 1 * *' }}
      PGO_EXPORT: ${{ inputs.pgoExport || github.event.schedule=='0 0 1 * *' }}
      TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone-aws.tfvars' }} # set the default again otherwise schedules won't work
      TF_VAR_BUILD_ID: ${{ github.run_id }}
      TF_VAR_ENVIRONMENT: ci
      TF_VAR_REPO: ${{ github.repository }}
      TF_VAR_ec_target: qa
      GOBENCH_TAGS: branch=${{ github.head_ref || github.ref }},commit=${{ github.sha }},target_branch=${{ github.base_ref }},enable_tail_sampling=${{ inputs.enableTailSampling }}
      GOBENCH_PASSWORD: ${{ secrets.GOBENCH_PASSWORD }}
      GOBENCH_USERNAME: ${{ secrets.GOBENCH_USERNAME }}
      GOBENCH_HOST: ${{ secrets.GOBENCH_HOST }}
    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-go@v6
        with:
          go-version-file: 'go.mod'
          cache: false

      - uses: rlespinasse/github-slug-action@e6f261660910b273384c5c42b17a0217881b217a # v5.6.0

      - name: Set up env
        run: |
          SLUGGED_BRANCH_NAME=${{ env.GITHUB_HEAD_REF_SLUG || env.GITHUB_REF_SLUG }}
          CREATED_AT=$(date +%s)
          echo "TF_VAR_BRANCH=${SLUGGED_BRANCH_NAME}" >> "$GITHUB_ENV"
          echo "TF_VAR_CREATED_AT=${CREATED_AT}" >> "$GITHUB_ENV"
          echo "USER=benchci-$SLUGGED_BRANCH_NAME-$CREATED_AT" >> "$GITHUB_ENV"

          if [ ! -z "$BENCHMARK_AGENTS" ]; then
            echo "BENCHMARK_AGENTS=$BENCHMARK_AGENTS" >> "$GITHUB_ENV"
          fi
          if [ ! -z "$BENCHMARK_RUN" ]; then
            echo "BENCHMARK_RUN=$BENCHMARK_RUN" >> "$GITHUB_ENV"
          fi
          if [ ! -z "$WARMUP_TIME" ]; then
            echo "BENCHMARK_WARMUP_TIME=$WARMUP_TIME" >> "$GITHUB_ENV"
          elif  [ "${{github.event_name}}" == 'schedule' ] && [ "${{env.RUN_STANDALONE}}" == 'false' ]; then
            echo "BENCHMARK_WARMUP_TIME=10m" >> "$GITHUB_ENV"
          elif  [ "${{github.event_name}}" == 'schedule' ] && [ "${{env.RUN_STANDALONE}}" == 'true' ]; then
            echo "BENCHMARK_WARMUP_TIME=3m" >> "$GITHUB_ENV"
          else
            echo "BENCHMARK_WARMUP_TIME=5m" >> "$GITHUB_ENV"
          fi
        env:
          BENCHMARK_AGENTS: ${{ inputs.benchmarkAgents }}
          BENCHMARK_RUN: ${{ inputs.benchmarkRun }}
          WARMUP_TIME: ${{ inputs.warmupTime }}

      # exclude BenchmarkTraces* from scheduled benchmarks
      - if: github.event_name == 'schedule'
        run: |
          echo "BENCHMARK_RUN=Benchmark[^T]" >> "$GITHUB_ENV"

      - name: Log in to the Elastic Container registry
        uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
        with:
          registry: ${{ secrets.ELASTIC_DOCKER_REGISTRY }}
          username: ${{ secrets.ELASTIC_DOCKER_USERNAME }}
          password: ${{ secrets.ELASTIC_DOCKER_PASSWORD }}

      - uses: elastic/oblt-actions/google/auth@v1
        with:
          project-number: '911195782929'
          project-id: 'elastic-observability-ci'

      - uses: elastic/oblt-actions/aws/auth@v1
        with:
          role-duration-seconds: 21600 # 6 hours

      - uses: google-github-actions/get-secretmanager-secrets@bc9c54b29fdffb8a47776820a7d26e77b379d262 # v3.0.0
        with:
          export_to_environment: true
          secrets: |-
            EC_API_KEY:elastic-observability-ci/elastic-cloud-observability-team-qa-api-key

      - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1
        with:
          terraform_version: ~1.10.0
          terraform_wrapper: false

      - name: Init terraform module
        id: init
        run: make init

      - name: Build apmbench
        run: make apmbench $SSH_KEY terraform.tfvars

      - name: Build APM Server and Moxy
        if: ${{ env.RUN_STANDALONE == 'true' }}
        run: |
          make apm-server
          make moxy

      - name: Override docker committed version
        if: ${{ ! inputs.runOnStable && env.RUN_STANDALONE == 'false' }}
        run: make docker-override-committed-version

      - name: Spin up benchmark environment
        id: deploy
        run: |
          make apply
          admin_console_url=$(terraform output -raw admin_console_url)
          echo "admin_console_url=$admin_console_url" >> "$GITHUB_OUTPUT"
          echo "-> infra setup done"
      - name: Run benchmarks autotuned
        if: ${{ inputs.benchmarkAgents == '' }}
        run: make run-benchmark-autotuned

      - name: Run benchmarks self tuned
        if: ${{ inputs.benchmarkAgents != '' }}
        run: make run-benchmark

      - name: Cat standalone server logs
        if: ${{ env.RUN_STANDALONE == 'true' && failure() }}
        run: make cat-apm-server-logs

      - name: Cat moxy logs
        if: ${{ env.RUN_STANDALONE == 'true' && failure() }}
        run: make cat-moxy-logs

      - name: Index benchmarks result
        run: make index-benchmark-results

      - name: Download PNG
        uses: elastic/oblt-actions/download-kibana-dashboard@v1
        with:
          kibana-host: ${{ secrets.KIBANA_BENCH_ENDPOINT }}
          kibana-user: ${{ secrets.KIBANA_BENCH_USERNAME }}
          kibana-password: ${{ secrets.KIBANA_BENCH_PASSWORD }}
          dashboard-id: "a5bc8390-2f8e-11ed-a369-052d8245fa04"
          from-date: "now-30d"
          to-date: "now"
          png-filename: ${{ env.PNG_REPORT_FILE }}

      - name: Debug PNG files
        working-directory: "${{ github.workspace }}"
        run: |
          if [ -f "${{ env.PNG_REPORT_FILE }}" ]; then
            echo "PNG report file exists in the top-level location."
          elif [ -f " ${{ env.WORKING_DIRECTORY }}/${{ env.PNG_REPORT_FILE }}" ]; then
            echo "PNG report file exists in the working directory."
          else
            echo "PNG report file does not exist in either location."
          fi

      - name: Upload PNG to AWS S3
        id: s3-upload-png
        working-directory: "${{ github.workspace }}"
        env:
          AWS_DEFAULT_REGION: us-east-1
        run: |
          DEST_NAME="github-run-id-${{ github.run_id }}.png"
          aws s3 --debug cp ${{ env.PNG_REPORT_FILE }} s3://elastic-apm-server-benchmark-reports/${DEST_NAME}
          echo "png_report_url=https://elastic-apm-server-benchmark-reports.s3.amazonaws.com/${DEST_NAME}" >> "$GITHUB_OUTPUT"

      - name: Upload benchmark result
        uses: actions/upload-artifact@v7
        with:
          name: benchmark-result
          path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_RESULT }}
          if-no-files-found: error

      # The next section injects CPU profile collected by apmbench into the build.
      # By copying the profile, uploading it to the artifacts and pushing it
      # via a PR to update default.pgo.

      - name: Copy CPU profile
        run: make cp-cpuprof

      - name: Upload CPU profile
        uses: actions/upload-artifact@v7
        with:
          name: cpu-profile
          path: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_CPU_OUT }}
          if-no-files-found: error

      - name: Get token
        id: get_token
        uses: actions/create-github-app-token@v3
        with:
          app-id: ${{ secrets.OBS_AUTOMATION_APP_ID }}
          private-key: ${{ secrets.OBS_AUTOMATION_APP_PEM }}
          permission-contents: write
          permission-pull-requests: write

      # Required to use a service account, otherwise PRs created by
      # GitHub bot won't trigger any CI builds.
      # See https://github.com/peter-evans/create-pull-request/issues/48#issuecomment-537478081
      - name: Configure git user
        uses: elastic/oblt-actions/git/setup@v1
        with:
          github-token: ${{ steps.get_token.outputs.token }}

      - name: Import GPG key
        uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd  # v7.0.0
        with:
          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
          passphrase: ${{ secrets.GPG_PASSPHRASE }}
          git_user_signingkey: true
          git_commit_gpgsign: true

      - name: Open PGO PR
        if: ${{ env.PGO_EXPORT == 'true' }}
        run: ${{ github.workspace }}/.ci/scripts/push-pgo-pr.sh
        env:
          WORKSPACE_PATH: ${{ github.workspace }}
          PROFILE_PATH: ${{ env.WORKING_DIRECTORY }}/${{ env.BENCHMARK_CPU_OUT }}
          GITHUB_TOKEN: ${{ steps.get_token.outputs.token }}
          WORKFLOW: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}

      # Secrets are rotated daily, if the benchmarks run between the rotation window, then
      # there is a high chance things will stop working
      # This is trying to reduce the chances of that happening.
      # See https://github.com/elastic/observability-test-environments/actions/workflows/cluster-rotate-api-keys.yml
      - uses: google-github-actions/get-secretmanager-secrets@bc9c54b29fdffb8a47776820a7d26e77b379d262 # v3.0.0
        if: always()
        with:
          export_to_environment: true
          secrets: |-
            EC_API_KEY:elastic-observability-ci/elastic-cloud-observability-team-qa-api-key

      - name: Tear down benchmark environment
        if: always()
        run: make init destroy

      # Notify failure to Slack only on schedule (nightly run)
      - if: failure() && github.event_name == 'schedule'
        uses: elastic/oblt-actions/slack/notify-result@v1
        with:
          bot-token: ${{ secrets.SLACK_BOT_TOKEN }}
          channel-id: "#apm-server"
          message: Nightly APM Server benchmarks failed! SDH Duty assignee, please have a look and follow this <https://github.com/elastic/observability-dev/blob/main/docs/apm/apm-server/runbooks/benchmarks.md|Runbook>!

      # Notify result to Slack only on schedule (nightly run)
      - if: github.event_name == 'schedule'
        uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
        with:
          method: chat.postMessage
          token: ${{ secrets.SLACK_BOT_TOKEN }}
          payload: |
            {
                "channel":  "#apm-server",
                "text": "${{ github.event_name == 'schedule' && 'Nightly' || '' }} APM Server benchmarks succesfully executed!",
                "blocks": [
                    {
                        "type": "section",
                        "text": {
                            "type": "mrkdwn",
                            "text": "${{ github.event_name == 'schedule' && 'Nightly' || '' }} APM Server benchmarks succesfully executed!"
                        },
                        "accessory": {
                            "type": "button",
                            "style": "primary",
                            "text": {
                                "type": "plain_text",
                                "text": "Workflow Run #${{ github.run_id }}",
                                "emoji": true
                            },
                            "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
                            "action_id": "workflow-run-button"
                        }
                    },
                    {
                        "type": "image",
                        "image_url": "${{ steps.s3-upload-png.outputs.png_report_url }}",
                        "alt_text": "kibana-png-report"
                    },
                    {
                        "type": "actions",
                        "elements": [
                            {
                                "type": "button",
                                "text": {
                                    "type": "plain_text",
                                    "text": "Benchmarks dashboard"
                                },
                                "url": "${{ secrets.KIBANA_BENCH_DASHBOARD }}",
                                "action_id": "kibana-dashboard-button"
                            },
                            {
                                "type": "button",
                                "text": {
                                    "type": "plain_text",
                                    "text": "Elastic Cloud deployment"
                                },
                                "url": "${{ steps.deploy.outputs.admin_console_url }}",
                                "action_id": "admin-console-button"
                            }
                        ]
                    }
                ]
            }