From 03fef6cef9a54afbbbb7e0f152567e3d80f6711c Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Fri, 7 Feb 2025 14:03:20 +0200 Subject: [PATCH 1/3] Split packaging into separate steps (#6401) Splitting centralized packaging step into separate steps to speed up builds (cherry picked from commit e956b4d2dbce3aee030900a6955fcfd71340310d) # Conflicts: # .buildkite/scripts/steps/integration-package.sh --- .buildkite/bk.integration.pipeline.yml | 16 ++-- .buildkite/integration.pipeline.yml | 87 +++++++++++++++---- .../scripts/steps/integration-package.sh | 10 +++ 3 files changed, 88 insertions(+), 25 deletions(-) diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml index 6d61b8743fb..178830ce575 100644 --- a/.buildkite/bk.integration.pipeline.yml +++ b/.buildkite/bk.integration.pipeline.yml @@ -28,7 +28,7 @@ steps: steps: - label: "Win2022:sudo:{{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/integration-tests.ps1 {{matrix}} true artifact_paths: - build/** @@ -45,7 +45,7 @@ steps: - label: "Win2022:non-sudo:{{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/integration-tests.ps1 {{matrix}} false artifact_paths: - build/** @@ -63,8 +63,9 @@ steps: - integration-ess steps: - label: "x86_64:non-sudo: {{matrix}}" + # only packaging-ubuntu-x86-64 artifact dependency is required command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - build/** @@ -77,8 +78,9 @@ steps: - default - label: "x86_64:sudo: {{matrix}}" + # due to deb group present in matrix tar.gz and deb packages artifacts are required command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - build/** @@ -102,7 +104,7 @@ steps: - label: "arm:sudo: {{matrix}}" skip: true command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - build/** @@ -126,7 +128,7 @@ steps: - label: "arm:non-sudo: {{matrix}}" skip: true command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - build/** @@ -145,7 +147,7 @@ steps: steps: - label: "x86_64:sudo:rpm" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh rpm true artifact_paths: - build/** diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 3095024feff..45434e62fc8 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -5,33 +5,83 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" steps: - - label: "Integration tests: packaging" - key: "package-it" - command: ".buildkite/scripts/steps/integration-package.sh" - artifact_paths: - - build/distributions/** - agents: - provider: "gcp" - machineType: "n1-standard-8" + - group: "Integration tests: packaging" + key: "int-packaging" + steps: + - label: "Packaging: Ubuntu x86_64" + key: "packaging-ubuntu-x86-64" + env: + PACKAGES: "tar.gz" + PLATFORMS: "linux/amd64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + + - label: "Packaging: Ubuntu arm64" + key: "packaging-ubuntu-arm64" + env: + PACKAGES: "tar.gz" + PLATFORMS: "linux/arm64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + + - label: "Packaging: Windows" + key: "packaging-windows" + env: + PACKAGES: "zip" + PLATFORMS: "windows/amd64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + + - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}" + key: "packaging-containers" + env: + PACKAGES: "{{matrix.ext}}" + PLATFORMS: "{{matrix.arch}}" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + matrix: + setup: + arch: + - linux/amd64 + - linux/arm64 + ext: + - rpm + - deb - label: "Serverless integration test" key: "serverless-integration-tests" depends_on: - - package-it + - int-packaging concurrency_group: elastic-agent-extended-testing/serverless-integration concurrency: 8 env: # we run each step in a different data center to spread the load TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' + buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite artifact_paths: - "build/TEST-**" - "build/diagnostics/*" agents: provider: "gcp" - machineType: "n1-standard-8" + machineType: "n2-standard-8" notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Serverless integration test" @@ -39,46 +89,47 @@ steps: - label: "Extended runtime leak tests" key: "extended-integration-tests" depends_on: - - package-it + - int-packaging concurrency_group: elastic-agent-extended-testing/leak-tests concurrency: 8 env: TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' + buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks artifact_paths: - "build/TEST-**" - "build/diagnostics/*" agents: provider: "gcp" - machineType: "n1-standard-8" + machineType: "n2-standard-8" notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests" - label: "Triggering Integration tests" depends_on: - - package-it + - int-packaging trigger: "elastic-agent-extended-testing-bk" build: commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" - label: "Serverless Beats Tests" + # To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required depends_on: - - package-it + - packaging-ubuntu-x86-64 key: "serverless-beats-integration-tests" concurrency_group: elastic-agent-extended-testing/beats-integration concurrency: 8 env: TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' + buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID .buildkite/scripts/steps/beats_tests.sh agents: provider: "gcp" - machineType: "n1-standard-8" + machineType: "n2-standard-8" notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests" diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index 623a67573bc..34f3d707080 100755 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -3,4 +3,14 @@ set -euo pipefail source .buildkite/scripts/common.sh +<<<<<<< HEAD PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package +======= +# Remove AGENT_PACKAGE_VERSION pinning as soon as 9.0.0 is released +export AGENT_PACKAGE_VERSION=9.0.0 + +export SNAPSHOT="true" +export EXTERNAL="true" + +mage package +>>>>>>> e956b4d2d (Split packaging into separate steps (#6401)) From 16a736db17bdd74f450d10c8aa74c36a26fa170d Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Mon, 10 Feb 2025 09:31:08 +0200 Subject: [PATCH 2/3] resolved conflicts --- .buildkite/scripts/steps/integration-package.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index 34f3d707080..3ba5bf3d7b3 100755 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -3,9 +3,6 @@ set -euo pipefail source .buildkite/scripts/common.sh -<<<<<<< HEAD -PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package -======= # Remove AGENT_PACKAGE_VERSION pinning as soon as 9.0.0 is released export AGENT_PACKAGE_VERSION=9.0.0 @@ -13,4 +10,3 @@ export SNAPSHOT="true" export EXTERNAL="true" mage package ->>>>>>> e956b4d2d (Split packaging into separate steps (#6401)) From 160e3bbf421a659a16e4c5ea589c253686c5be76 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Tue, 11 Feb 2025 19:13:06 +0200 Subject: [PATCH 3/3] removed agent version pin --- .buildkite/scripts/steps/integration-package.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index 3ba5bf3d7b3..cd58c2777a8 100755 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -3,9 +3,6 @@ set -euo pipefail source .buildkite/scripts/common.sh -# Remove AGENT_PACKAGE_VERSION pinning as soon as 9.0.0 is released -export AGENT_PACKAGE_VERSION=9.0.0 - export SNAPSHOT="true" export EXTERNAL="true"