pipeline.xpack.libbeat.yml

# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
name: "beats-xpack-libbeat"

env:
  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64-1782878510"

  IMAGE_UBUNTU_X86_64: "platform-ingest-beats-ubuntu-2204-1782878510"
  IMAGE_WIN_2022: "platform-ingest-beats-windows-2022-1782878510"

  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"

  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
  GCP_WIN_MACHINE_TYPE: "n2-standard-8"

  #Deps
  ASDF_MAGE_VERSION: 1.15.0

  # Unit tests
  RACE_DETECTOR: "true"
  TEST_COVERAGE: "true"

steps:
  - group: "Check/Update"
    key: "x-pack-libbeat-check-update"
    steps:
      - label: "x-pack/libbeat: Run check/update"
        command: |
          set -eo pipefail
          make -C x-pack/libbeat check update
          make check-no-changes
        retry:
          automatic:
            - limit: 1
        agents:
          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
          cpu: "4000m"
          memory: "8Gi"
          # 10GB storage was not enough for check/update.
          ephemeralStorage: "40G"
          useCustomGlobalHooks: true
        notify:
          - github_commit_status:
              context: "x-pack/libbeat: check/update"

      - label: "x-pack/libbeat: Run pre-commit"
        command: "pre-commit run --all-files"
        agents:
          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
          memory: "2Gi"
          useCustomGlobalHooks: true
        notify:
          - github_commit_status:
              context: "x-pack/libbeat: pre-commit"

  - wait: ~
    # with PRs, we want to run mandatory tests only if check/update step succeed
    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
    # this allows building DRA artifacts even if there is flakiness in check/update step
    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
    depends_on: "x-pack-libbeat-check-update"

  - group: "Mandatory Tests"
    key: "x-pack-libbeat-mandatory-tests"

    steps:
      - label: ":ubuntu: x-pack/libbeat: Ubuntu x86_64 Unit Tests"
        key: "mandatory-linux-unit-test"
        command: |
          cd x-pack/libbeat
          mage build unitTest
        retry:
          automatic:
            - limit: 1
        agents:
          provider: "gcp"
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        artifact_paths:
          - "x-pack/libbeat/build/*.xml"
          - "x-pack/libbeat/build/*.json"
        notify:
          - github_commit_status:
              context: "x-pack/libbeat: Ubuntu x86_64 Unit Tests"

      - label: ":ubuntu: x-pack/libbeat: Ubuntu x86_64 Unit Tests with requirefips build tag"
        key: "mandatory-linux-unit-test-fips-tag"
        command: |
          cd x-pack/libbeat
          mage unitTest
        retry:
          automatic:
            - limit: 1
        agents:
          provider: "gcp"
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        env:
          FIPS: "true"
        artifact_paths:
          - "x-pack/libbeat/build/*.xml"
          - "x-pack/libbeat/build/*.json"
        notify:
          - github_commit_status:
              context: "x-pack/libbeat: Ubuntu x86_64 Unit Tests with requirefips build tag"

      - label: ":ubuntu: x-pack/libbeat: Ubuntu x86_64 fips140=only Unit Tests"
        key: "mandatory-linux-unit-test-fips-only"
        command: |
          cd x-pack/libbeat
          mage goFIPSOnlyUnitTest
        retry:
          automatic:
            - limit: 1
        agents:
          provider: "gcp"
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        env:
          FIPS: "true"
        artifact_paths:
          - "x-pack/libbeat/build/*.xml"
          - "x-pack/libbeat/build/*.json"
        notify:
          - github_commit_status:
              context: "x-pack/libbeat: Ubuntu x86_64 fips140=only Unit Tests"

      - label: ":ubuntu: x-pack/libbeat: Go Integration Tests"
        command: |
          cd x-pack/libbeat
          mage goIntegTest
        retry:
          automatic:
            - limit: 1
        agents:
          provider: "gcp"
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
        artifact_paths:
          - "x-pack/libbeat/build/*.xml"
          - "x-pack/libbeat/build/*.json"
          - "x-pack/libbeat/build/integration-tests/*"
          - "x-pack/libbeat/build/integration-tests/Test*/*"
          - "x-pack/libbeat/build/integration-tests/Test*/data/**/*"
        notify:
          - github_commit_status:
              context: "x-pack/libbeat: Go Integration Tests / Ubuntu x86_64"

      - label: ":windows: x-pack/libbeat: Go Integration Tests"
        skip: "Skipped due to https://github.com/elastic/beats/issues/44537"
        command: |
          cd x-pack/libbeat
          mage goIntegTest
        retry:
          automatic:
            - limit: 1
        agents:
          provider: "gcp"
          image: "${IMAGE_WIN_2022}"
          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
        artifact_paths:
          - "x-pack/libbeat/build/*.xml"
          - "x-pack/libbeat/build/*.json"
          - "x-pack/libbeat/build/integration-tests/*"
          - "x-pack/libbeat/build/integration-tests/Test*/*"
          - "x-pack/libbeat/build/integration-tests/Test*/data/**/*"
        notify:
          - github_commit_status:
              context: "x-pack/libbeat: Go Integration Tests / Windows"

      - label: ":ubuntu: x-pack/libbeat: Python Integration Tests"
        key: "mandatory-python-int-test"
        command: |
          cd x-pack/libbeat
          mage pythonIntegTest
        retry:
          automatic:
            - limit: 1
        agents:
          provider: "gcp"
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
        artifact_paths:
          - "x-pack/libbeat/build/*.xml"
          - "x-pack/libbeat/build/*.json"
          - "x-pack/libbeat/build/integration-tests/*"
          - "x-pack/libbeat/build/integration-tests/Test*/*"
          - "x-pack/libbeat/build/integration-tests/Test*/data/**/*"
        notify:
          - github_commit_status:
              context: "x-pack/libbeat: Python Integration Tests"

  - group: "Extended Tests"
    key: "x-pack-libbeat-extended-tests-linux-arm"
    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/

    steps:
      - label: ":ubuntu: x-pack/libbeat: Ubuntu arm64 Unit Tests"
        key: "extended-arm64-unit-tests"
        command: |
          cd x-pack/libbeat
          mage build unitTest
        retry:
          automatic:
            - limit: 1
        agents:
          provider: "aws"
          image: "${AWS_IMAGE_UBUNTU_ARM_64}"
          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
        artifact_paths:
          - "x-pack/libbeat/build/*.xml"
          - "x-pack/libbeat/build/*.json"
        notify:
          - github_commit_status:
              context: "x-pack/libbeat: Ubuntu arm64 Unit Tests"