packaging.pipeline.yml

# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
# TODO: Pre-cache beats-dev/golang-crossbuild container image

env:
  ASDF_MAGE_VERSION: 1.15.0
  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64-1782878510"
  GCP_DEFAULT_MACHINE_TYPE: "c2d-standard-8"
  IMAGE_UBUNTU_X86_64: "platform-ingest-beats-ubuntu-2204-1782878510"

  PLATFORMS: "+all linux/amd64 windows/amd64 darwin/amd64"
  PLATFORMS_ARM: "+all linux/arm64 darwin/arm64 windows/arm64"
  PLATFORMS_AMD64_FIPS: "+all linux/amd64"
  PLATFORMS_ARM64_FIPS: "+all linux/arm64"

# This section is used to define the plugins that will be used in the pipeline.
# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
common:
  - dockerhub_login_plugin: &dockerhub_login_plugin
      elastic/vault-docker-login#v0.8.0:
        secret_path: 'kv/ci-shared/observability-github-secrets/shared/docker/dockerhub/observabilityrobots'
        disable_logout: true
  - docker_elastic_login_plugin: &docker_elastic_login_plugin
      elastic/vault-docker-login#v0.8.0:
        secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
        disable_logout: true
  - google_oidc_plugin: &google_oidc_plugin
      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/elastic-agent/01-gcp-oidc.tf
      # This plugin authenticates to Google Cloud using the OIDC token.
      elastic/oblt-google-auth#v1.3.1:
        lifetime: 10800 # seconds
        project-id: "elastic-observability-ci"
        project-number: "911195782929"

steps:
  # we use concurrency gates (https://buildkite.com/blog/concurrency-gates)
  # to implement two FIFO queues for DRA-snapshot and DRA-staging
  # this prevents parallel builds and possibility of publishing out of order DRA artifacts if the first job takes longer than the second

  - name: Start of concurrency group for DRA Snapshot
    if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
    command: echo "--> Start of concurrency gate dra-snapshot"
    concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH"
    concurrency: 1
    key: start-gate-snapshot

  - name: Start of concurrency group for DRA Staging
    # exceptionally allow building staging from main when VERSION_QUALIFIER is set, to allow prerelease testing
    # TODO remove OR clause below and above comment, and only allow matching /^[0-9]+\.[0-9x]+\$/ for build.branch
    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
    command: echo "--> Start of concurrency gate dra-staging"
    concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
    concurrency: 1
    key: start-gate-staging

  - wait

  - group: Beats dashboards
    key: dashboards
    steps:
      - label: Snapshot dashboards
        if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
        depends_on: start-gate-snapshot
        key: dashboards-snapshot
        # TODO: container with go and make
        agents:
          provider: gcp
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        commands:
          - make build/distributions/dependencies.csv
          - make beats-dashboards
        env:
          SNAPSHOT: true
          DEV: true
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin

      - label: Staging dashboards
        # TODO remove OR clause below (see earlier comment)
        if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true"
        depends_on: start-gate-staging
        key: dashboards-staging
        # TODO: container with go and make
        agents:
          provider: gcp
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        commands: |
          source .buildkite/scripts/version_qualifier.sh
          make build/distributions/dependencies.csv
          make beats-dashboards
        env:
          SNAPSHOT: false
          DEV: false
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin

  - group: Packaging snapshot
    if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
    key: packaging-snapshot
    depends_on: start-gate-snapshot
    steps:
      - label: "SNAPSHOT: {{matrix}}"
        env:
          PLATFORMS: "${PLATFORMS}"
          SNAPSHOT: true
          # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270
          DEV: false
        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
        agents:
          provider: gcp
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin
          - *dockerhub_login_plugin
          - *docker_elastic_login_plugin
        matrix:
          - auditbeat
          - filebeat
          - heartbeat
          - metricbeat
          - packetbeat
          - winlogbeat
          - x-pack/auditbeat
          - x-pack/dockerlogbeat
          - x-pack/filebeat
          - x-pack/heartbeat
          - x-pack/metricbeat
          - x-pack/osquerybeat
          - x-pack/packetbeat
          - x-pack/winlogbeat

      - label: "SNAPSHOT: {{matrix}} Linux/arm64, Darwin/arm64, and Windows/arm64"
        env:
          PLATFORMS: "${PLATFORMS_ARM}"
          SNAPSHOT: true
          # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270
          DEV: false
        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
        agents:
          provider: "aws"
          image: "${AWS_IMAGE_UBUNTU_ARM_64}"
          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin
          - *dockerhub_login_plugin
          - *docker_elastic_login_plugin
        matrix:
          - auditbeat
          - filebeat
          - heartbeat
          - metricbeat
          - packetbeat
          - winlogbeat
          - x-pack/auditbeat
          - x-pack/dockerlogbeat
          - x-pack/filebeat
          - x-pack/heartbeat
          - x-pack/metricbeat
          - x-pack/osquerybeat
          - x-pack/packetbeat
          - x-pack/winlogbeat

      - label: "SNAPSHOT: {{matrix}} Linux/amd64 FIPS"
        env:
          PLATFORMS: "${PLATFORMS_AMD64_FIPS}"
          SNAPSHOT: true
          # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270
          DEV: false
          FIPS: true
        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
        agents:
          provider: gcp
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin
          - *dockerhub_login_plugin
          - *docker_elastic_login_plugin
        matrix:
          - x-pack/auditbeat
          - x-pack/filebeat
          - x-pack/metricbeat

      - label: "SNAPSHOT: {{matrix}} Linux/arm64 FIPS"
        env:
          PLATFORMS: "${PLATFORMS_ARM64_FIPS}"
          SNAPSHOT: true
          # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270
          DEV: false
          FIPS: true
        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
        agents:
          provider: "aws"
          image: "${AWS_IMAGE_UBUNTU_ARM_64}"
          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin
          - *dockerhub_login_plugin
          - *docker_elastic_login_plugin
        matrix:
          - x-pack/auditbeat
          - x-pack/filebeat
          - x-pack/metricbeat

  - group: Packaging Staging
    key: packaging-staging
    depends_on: start-gate-staging
    # TODO remove OR clause below (see earlier comment)
    if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true"
    steps:
      - label: "STAGING: {{matrix}}"
        env:
          PLATFORMS: "${PLATFORMS}"
          SNAPSHOT: false
          DEV: false
        command: |
          source .buildkite/scripts/version_qualifier.sh
          .buildkite/scripts/packaging/package-dra.sh {{matrix}}
        agents:
          provider: gcp
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin
          - *dockerhub_login_plugin
          - *docker_elastic_login_plugin
        matrix:
          - auditbeat
          - filebeat
          - heartbeat
          - metricbeat
          - packetbeat
          - winlogbeat
          - x-pack/auditbeat
          - x-pack/dockerlogbeat
          - x-pack/filebeat
          - x-pack/heartbeat
          - x-pack/metricbeat
          - x-pack/osquerybeat
          - x-pack/packetbeat
          - x-pack/winlogbeat

      - label: "STAGING: {{matrix}} Linux/arm64, Darwin/arm64, and Windows/arm64"
        env:
          PLATFORMS: "${PLATFORMS_ARM}"
          SNAPSHOT: false
          DEV: false
        command: |
          source .buildkite/scripts/version_qualifier.sh
          .buildkite/scripts/packaging/package-dra.sh {{matrix}}
        agents:
          provider: "aws"
          image: "${AWS_IMAGE_UBUNTU_ARM_64}"
          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin
          - *dockerhub_login_plugin
          - *docker_elastic_login_plugin
        matrix:
          - auditbeat
          - filebeat
          - heartbeat
          - metricbeat
          - packetbeat
          - winlogbeat
          - x-pack/auditbeat
          - x-pack/dockerlogbeat
          - x-pack/filebeat
          - x-pack/heartbeat
          - x-pack/metricbeat
          - x-pack/osquerybeat
          - x-pack/packetbeat
          - x-pack/winlogbeat

      - label: "STAGING: {{matrix}} Linux/amd64 FIPS"
        env:
          PLATFORMS: "${PLATFORMS_AMD64_FIPS}"
          SNAPSHOT: false
          # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270
          DEV: false
          FIPS: true
        command: |
          source .buildkite/scripts/version_qualifier.sh
          .buildkite/scripts/packaging/package-dra.sh {{matrix}}
        agents:
          provider: gcp
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin
          - *dockerhub_login_plugin
          - *docker_elastic_login_plugin
        matrix:
          - x-pack/auditbeat
          - x-pack/filebeat
          - x-pack/metricbeat

      - label: "STAGING: {{matrix}} Linux/arm64 FIPS"
        env:
          PLATFORMS: "${PLATFORMS_ARM64_FIPS}"
          SNAPSHOT: false
          # packaging with `DEV=true` may cause linker issues while crosscompiling https://github.com/elastic/beats/issues/41270
          DEV: false
          FIPS: true
        command: |
          source .buildkite/scripts/version_qualifier.sh
          .buildkite/scripts/packaging/package-dra.sh {{matrix}}
        agents:
          provider: "aws"
          image: "${AWS_IMAGE_UBUNTU_ARM_64}"
          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
        timeout_in_minutes: 40
        retry:
          automatic:
            - limit: 1
        artifact_paths:
          - build/distributions/**/*
        plugins:
          - *google_oidc_plugin
          - *dockerhub_login_plugin
          - *docker_elastic_login_plugin
        matrix:
          - x-pack/auditbeat
          - x-pack/filebeat
          - x-pack/metricbeat

  - group: DRA publish
    key: dra
    steps:
      - label: DRA Snapshot
        ## Only for release branches and main
        if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
        key: dra-snapshot
        env:
          DRA_WORKFLOW: snapshot
        depends_on:
          - start-gate-snapshot
          - packaging-snapshot
          - dashboards-snapshot
        command: |
          buildkite-agent artifact download "build/**/*" .
          .buildkite/scripts/packaging/prepare-release-manager.sh snapshot
          .buildkite/scripts/dra.sh
        agents:
          provider: gcp
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        plugins:
          - *google_oidc_plugin
          - *docker_elastic_login_plugin

      - label: DRA Staging
        # TODO remove OR clause below (see earlier comment)
        if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null) || build.env('RUN_STAGING') == "true"
        key: dra-staging
        env:
          DRA_WORKFLOW: staging
        depends_on:
          - start-gate-staging
          - packaging-staging
          - dashboards-staging
        command: |
          source .buildkite/scripts/version_qualifier.sh
          buildkite-agent artifact download "build/**" .
          .buildkite/scripts/packaging/prepare-release-manager.sh staging
          .buildkite/scripts/dra.sh
        agents:
          provider: gcp
          image: "${IMAGE_UBUNTU_X86_64}"
          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
        plugins:
          - *google_oidc_plugin
          - *docker_elastic_login_plugin

  - wait

  - command: echo "End of concurrency gate dra-snapshot <--"
    if: (build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true") && build.env('VERSION_QUALIFIER') == null
    concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH"
    concurrency: 1
    key: end-gate-snapshot

  - command: echo "End of concurrency gate dra-staging <--"
    if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env('VERSION_QUALIFIER') != null
    concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH"
    concurrency: 1
    key: end-gate-staging