diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml index 6d61b8743f..178830ce57 100644 --- a/.buildkite/bk.integration.pipeline.yml +++ b/.buildkite/bk.integration.pipeline.yml @@ -28,7 +28,7 @@ steps: steps: - label: "Win2022:sudo:{{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/integration-tests.ps1 {{matrix}} true artifact_paths: - build/** @@ -45,7 +45,7 @@ steps: - label: "Win2022:non-sudo:{{matrix}}" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/integration-tests.ps1 {{matrix}} false artifact_paths: - build/** @@ -63,8 +63,9 @@ steps: - integration-ess steps: - label: "x86_64:non-sudo: {{matrix}}" + # only packaging-ubuntu-x86-64 artifact dependency is required command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - build/** @@ -77,8 +78,9 @@ steps: - default - label: "x86_64:sudo: {{matrix}}" + # due to deb group present in matrix tar.gz and deb packages artifacts are required command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - build/** @@ -102,7 +104,7 @@ steps: - label: "arm:sudo: {{matrix}}" skip: true command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true artifact_paths: - build/** @@ -126,7 +128,7 @@ steps: - label: "arm:non-sudo: {{matrix}}" skip: true command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false artifact_paths: - build/** @@ -145,7 +147,7 @@ steps: steps: - label: "x86_64:sudo:rpm" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} + buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} .buildkite/scripts/steps/integration_tests_tf.sh rpm true artifact_paths: - build/** diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index 3095024fef..45434e62fc 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -5,33 +5,83 @@ env: VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" steps: - - label: "Integration tests: packaging" - key: "package-it" - command: ".buildkite/scripts/steps/integration-package.sh" - artifact_paths: - - build/distributions/** - agents: - provider: "gcp" - machineType: "n1-standard-8" + - group: "Integration tests: packaging" + key: "int-packaging" + steps: + - label: "Packaging: Ubuntu x86_64" + key: "packaging-ubuntu-x86-64" + env: + PACKAGES: "tar.gz" + PLATFORMS: "linux/amd64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + + - label: "Packaging: Ubuntu arm64" + key: "packaging-ubuntu-arm64" + env: + PACKAGES: "tar.gz" + PLATFORMS: "linux/arm64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + + - label: "Packaging: Windows" + key: "packaging-windows" + env: + PACKAGES: "zip" + PLATFORMS: "windows/amd64" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + + - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}" + key: "packaging-containers" + env: + PACKAGES: "{{matrix.ext}}" + PLATFORMS: "{{matrix.arch}}" + command: ".buildkite/scripts/steps/integration-package.sh" + artifact_paths: + - build/distributions/** + agents: + provider: "gcp" + machineType: "n2-standard-8" + matrix: + setup: + arch: + - linux/amd64 + - linux/arm64 + ext: + - rpm + - deb - label: "Serverless integration test" key: "serverless-integration-tests" depends_on: - - package-it + - int-packaging concurrency_group: elastic-agent-extended-testing/serverless-integration concurrency: 8 env: # we run each step in a different data center to spread the load TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' + buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite artifact_paths: - "build/TEST-**" - "build/diagnostics/*" agents: provider: "gcp" - machineType: "n1-standard-8" + machineType: "n2-standard-8" notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Serverless integration test" @@ -39,46 +89,47 @@ steps: - label: "Extended runtime leak tests" key: "extended-integration-tests" depends_on: - - package-it + - int-packaging concurrency_group: elastic-agent-extended-testing/leak-tests concurrency: 8 env: TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' + buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks artifact_paths: - "build/TEST-**" - "build/diagnostics/*" agents: provider: "gcp" - machineType: "n1-standard-8" + machineType: "n2-standard-8" notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests" - label: "Triggering Integration tests" depends_on: - - package-it + - int-packaging trigger: "elastic-agent-extended-testing-bk" build: commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" - label: "Serverless Beats Tests" + # To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required depends_on: - - package-it + - packaging-ubuntu-x86-64 key: "serverless-beats-integration-tests" concurrency_group: elastic-agent-extended-testing/beats-integration concurrency: 8 env: TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a" command: | - buildkite-agent artifact download build/distributions/** . --step 'package-it' + buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID .buildkite/scripts/steps/beats_tests.sh agents: provider: "gcp" - machineType: "n1-standard-8" + machineType: "n2-standard-8" notify: - github_commit_status: context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests" diff --git a/.buildkite/scripts/steps/integration-package.sh b/.buildkite/scripts/steps/integration-package.sh index 623a67573b..cd58c2777a 100755 --- a/.buildkite/scripts/steps/integration-package.sh +++ b/.buildkite/scripts/steps/integration-package.sh @@ -3,4 +3,7 @@ set -euo pipefail source .buildkite/scripts/common.sh -PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package +export SNAPSHOT="true" +export EXTERNAL="true" + +mage package