From fb25658422d9fc6e1bd1ea0cfb13c3445f0886b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paolo=20Chil=C3=A0?= Date: Fri, 21 Nov 2025 16:21:19 +0100 Subject: [PATCH] Avoid downloading/uploading unnecessary artifacts in CI runs (#11327) * avoid uploading the whole build/ directory tree * Download only required artifacts for integration tests (cherry picked from commit 3cedabfc0ddef81642dd628eaa440169c97446a4) --- .buildkite/bk.integration.pipeline.yml | 76 +++++++++++++------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml index b397546b1d8..daaaf540974 100644 --- a/.buildkite/bk.integration.pipeline.yml +++ b/.buildkite/bk.integration.pipeline.yml @@ -70,7 +70,7 @@ common: buildkite-agent artifact download build/distributions/*-linux-amd64.docker.tar.gz . --step 'packaging-containers-amd64' .buildkite/scripts/steps/integration_tests_tf.sh kubernetes false artifact_paths: &k8s_test_artifact_paths - - build/** + - build/* - build/diagnostics/** - build/*.pod_logs_dump/* @@ -87,10 +87,10 @@ steps: FORCE_ESS_CREATE: "true" command: | #!/usr/bin/env bash - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-linux-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.sh ech true artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -134,10 +134,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/leak" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-windows-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.ps1 fleet true artifact_paths: - - build/** + - build/* - build/diagnostics/** agents: provider: "gcp" @@ -155,10 +155,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/leak" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-windows-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.ps1 fleet true artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -175,10 +175,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/leak" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-linux-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.sh fleet true artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -204,10 +204,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-windows-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true artifact_paths: - - build/** + - build/* - build/diagnostics/** agents: provider: "gcp" @@ -234,10 +234,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-windows-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false artifact_paths: - - build/** + - build/* - build/diagnostics/** agents: provider: "gcp" @@ -257,10 +257,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-windows-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true artifact_paths: - - build/** + - build/* - build/diagnostics/** agents: provider: "gcp" @@ -287,10 +287,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-windows-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} false artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -317,10 +317,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-linux-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -341,10 +341,11 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step packaging-amd64 + buildkite-agent artifact download build/distributions/elastic-agent-*-linux-x86_64* . --step packaging-amd64 + buildkite-agent artifact download build/distributions/elastic-agent-*-amd64.deb* . --step packaging-amd64 .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -377,10 +378,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-arm64' + buildkite-agent artifact download build/distributions/elastic-agent-*-linux-arm64* . --step 'packaging-arm64' .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - - build/** + - build/* - build/diagnostics/** agents: provider: "aws" @@ -415,10 +416,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-arm64' + buildkite-agent artifact download build/distributions/elastic-agent-*-linux-arm64* . --step 'packaging-arm64' .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -445,10 +446,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-linux-x86_64* . --step 'packaging-amd64' .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.group}} false artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -474,10 +475,11 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step packaging-amd64 + buildkite-agent artifact download build/distributions/elastic-agent-*-linux-x86_64* . --step packaging-amd64 + buildkite-agent artifact download build/distributions/elastic-agent-*-amd64.deb* . --step packaging-amd64 .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.group}} true artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -524,10 +526,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess" command: | - buildkite-agent artifact download build/distributions/** . --step packaging-amd64 + buildkite-agent artifact download build/distributions/elastic-agent-*-x86_64.rpm* . --step packaging-amd64 .buildkite/scripts/steps/integration_tests_tf.sh rpm true artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -624,10 +626,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-windows-x86_64* . --step 'packaging-amd64' .buildkite/scripts/buildkite-integration-tests.ps1 fleet true artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -646,10 +648,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-windows-x86_64* . --step 'packaging-amd64' .buildkite/scripts/buildkite-integration-tests.ps1 fleet true artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: @@ -666,10 +668,10 @@ steps: env: TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/serverless" command: | - buildkite-agent artifact download build/distributions/** . --step 'packaging-amd64' + buildkite-agent artifact download build/distributions/elastic-agent-*-linux-x86_64* . --step 'packaging-amd64' sudo -E .buildkite/scripts/buildkite-integration-tests.sh fleet true artifact_paths: - - build/** + - build/* - build/diagnostics/** retry: automatic: