[7.17](backport #4895) bk: use GCP OIDC

.buildkite/hooks/pre-command

@@ -6,11 +6,12 @@ source .buildkite/scripts/common.sh
 
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 EC_KEY_SECRET_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
-PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
-JOB_GCS_BUCKET="ingest-buildkite-ci"
+JOB_GCS_BUCKET="fleet-server-ci-internal"
 GITHUB_REPO_TOKEN=$VAULT_GITHUB_TOKEN
 
+export JOB_GCS_BUCKET
+
 # Usage:
 #check_if_file_exist_in_repo "infra" "main"
 #Returns FILE_EXISTS_IN_REPO=true if the defined file exists in the difined repo and FILE_EXISTS_IN_REPO=false if not exists
@@ -41,6 +42,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   check_if_file_exist_in_repo "infra" "${BUILDKITE_BRANCH}"                  #TODO should be changed to "main" for rollback...
 fi
 
+<<<<<<< HEAD
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
   export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
   export JOB_GCS_BUCKET
@@ -52,6 +54,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
     export JOB_GCS_BUCKET
   fi
 fi
+=======
+>>>>>>> d4d19b2 (bk: use GCP OIDC (#4895))
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
@@ -64,10 +68,13 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
     export VAULT_SECRET_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
   fi
 fi
+<<<<<<< HEAD
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
     export JOB_GCS_BUCKET
   fi
 fi
+=======
+>>>>>>> d4d19b2 (bk: use GCP OIDC (#4895))

.buildkite/hooks/pre-exit

@@ -5,13 +5,16 @@ set -euo pipefail
 source .buildkite/scripts/common.sh
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
-    unset GOOGLE_APPLICATION_CREDENTIALS
     cleanup
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
+<<<<<<< HEAD
   if [[ "$BUILDKITE_STEP_KEY" == "package-x86-64" || "$BUILDKITE_STEP_KEY" == "package-arm" || "$BUILDKITE_STEP_KEY" == "dra-snapshot" && "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
     unset GOOGLE_APPLICATION_CREDENTIALS
+=======
+  if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-fips-x86-64* || "$BUILDKITE_STEP_KEY" == package-arm* || "$BUILDKITE_STEP_KEY" == package-fips-arm* || "$BUILDKITE_STEP_KEY" == "dra-snapshot" && "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
+>>>>>>> d4d19b2 (bk: use GCP OIDC (#4895))
     unset VAULT_ROLE_ID_SECRET
     unset VAULT_ADDR_SECRET
     unset VAULT_SECRET_ID_SECRET

.buildkite/pipeline.package.mbp.yml

@@ -7,7 +7,19 @@ env:
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
 
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - oidc_plugin: &oidc
+      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
+      # This plugin authenticates to Google Cloud using the OIDC token.
+      elastic/oblt-google-auth#v1.2.0:
+        lifetime: 10800 # seconds
+        project-id: "elastic-observability-ci"
+        project-number: "911195782929"
+
 steps:
+<<<<<<< HEAD
   - group: "Package and Publish"
     key: "package-publish"
     steps:
@@ -36,6 +48,109 @@ steps:
           type:
           - "snapshot"
           - "staging"
+=======
+  - label: "Package x86_64 snapshot"
+    # skip building + packaging snapshot for pre-releases (flagged by a non-empty VERSION_QUALIFIER env var/BK param)
+    # as prereleases are only intended to be used with staging; details in https://github.com/elastic/ingest-dev/issues/4855
+    if: "build.env('VERSION_QUALIFIER') == null"
+    key: "package-x86-64-snapshot"
+    command: ".buildkite/scripts/package.sh snapshot"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
+
+  - label: "Package x86_64 staging"
+    key: "package-x86-64-staging"
+    command: |
+      source .buildkite/scripts/version_qualifier.sh
+      .buildkite/scripts/package.sh staging
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
+
+  - label: "Package FIPS x86_64 snapshot"
+    if: "build.env('VERSION_QUALIFIER') == null"
+    key: "package-fips-x86-64-snapshot"
+    command: ".buildkite/scripts/package.sh snapshot"
+    env:
+      FIPS: "true"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
+
+  - label: "Package FIPS x86_64 staging"
+    key: "package-fips-x86-64-staging"
+    command: |
+      source .buildkite/scripts/version_qualifier.sh
+      .buildkite/scripts/package.sh staging
+    env:
+      FIPS: "true"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"
+      machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
+
+  - label: "Package aarch64 snapshot"
+    if: "build.env('VERSION_QUALIFIER') == null"
+    key: "package-arm-snapshot"
+    command: ".buildkite/scripts/package.sh snapshot"
+    agents:
+      provider: "aws"
+      imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+      instanceType: "t4g.2xlarge"
+    plugins:
+      - *oidc_plugin
+
+  - label: "Package aarch64 staging"
+    key: "package-arm-staging"
+    command: |
+      source .buildkite/scripts/version_qualifier.sh
+      .buildkite/scripts/package.sh staging
+    agents:
+      provider: "aws"
+      imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+      instanceType: "t4g.2xlarge"
+    plugins:
+      - *oidc_plugin
+
+  - label: "Package FIPS aarch64 snapshot"
+    if: "build.env('VERSION_QUALIFIER') == null"
+    key: "package-fips-arm-snapshot"
+    command: ".buildkite/scripts/package.sh snapshot"
+    env:
+      FIPS: "true"
+    agents:
+      provider: "aws"
+      imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+      instanceType: "t4g.2xlarge"
+    plugins:
+      - *oidc_plugin
+
+  - label: "Package FIPS aarch64 staging"
+    key: "package-fips-arm-staging"
+    command: |
+      source .buildkite/scripts/version_qualifier.sh
+      .buildkite/scripts/package.sh staging
+    env:
+      FIPS: "true"
+    agents:
+      provider: "aws"
+      imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+      instanceType: "t4g.2xlarge"
+    plugins:
+      - *oidc_plugin
+>>>>>>> d4d19b2 (bk: use GCP OIDC (#4895))
 
   - label: "DRA snapshot"
     key: "dra-snapshot"
@@ -44,7 +159,12 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+<<<<<<< HEAD
     if: "${FILE_EXISTS_IN_REPO}"
+=======
+    plugins:
+      - *oidc_plugin
+>>>>>>> d4d19b2 (bk: use GCP OIDC (#4895))
     depends_on:
       - step: "package-publish"
         allow_failure: false
@@ -56,7 +176,12 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+<<<<<<< HEAD
     if: "${FILE_EXISTS_IN_REPO} == true && build.env('BUILDKITE_BRANCH') != 'main'"
+=======
+    plugins:
+      - *oidc_plugin
+>>>>>>> d4d19b2 (bk: use GCP OIDC (#4895))
     depends_on:
       - step: "dra-snapshot"
         allow_failure: false

.buildkite/pipeline.yml

@@ -64,6 +64,13 @@ steps:
     depends_on:
       - step: "tests"
         allow_failure: false
+    plugins:
+      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
+      # This plugin authenticates to Google Cloud using the OIDC token.
+      - elastic/oblt-google-auth#v1.2.0:
+          lifetime: 10800 # seconds
+          project-id: "elastic-observability-ci"
+          project-number: "911195782929"
 
   - label: ":jenkins: Release - Package Registry Distribution"
     key: "release-package-registry"

.buildkite/scripts/common.sh

@@ -95,13 +95,6 @@ with_Terraform() {
     terraform version
 }
 
-google_cloud_auth() {
-    local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/google-cloud-credentials.json
-    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
-    gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
-    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
-}
-
 upload_packages_to_gcp_bucket() {
     local pattern=${1}
     local baseUri="gs://${JOB_GCS_BUCKET}/${REPO}"
@@ -112,7 +105,7 @@ upload_packages_to_gcp_bucket() {
         bucketUriDefault="${baseUri}/pull-requests/pr-${GITHUB_PR_NUMBER}"
     fi
     for bucketUri in "${bucketUriCommit}" "${bucketUriDefault}"; do
-        gsutil -m -q cp -r ${pattern} "${bucketUri}"
+        gcloud storage cp --recursive --quiet ${pattern} "${bucketUri}"
     done
 }
 
@@ -131,15 +124,15 @@ upload_mbp_packages_to_gcp_bucket() {
     local pattern=${1}
     local type=${2}
     get_bucket_uri "${type}"
-    gsutil -m -q cp -r ${pattern} ${bucketUri}
+    gcloud storage cp --recursive --quiet ${pattern} ${bucketUri}
 }
 
 download_mbp_packages_from_gcp_bucket() {
     local pattern=${1}
     local type=${2}
     mkdir -p ${WORKSPACE}/${pattern}
     get_bucket_uri "${type}"
-    gsutil -m -q cp -r ${bucketUri}/* ${WORKSPACE}/${pattern}
+    gcloud storage cp --recursive --quiet ${bucketUri}/* ${WORKSPACE}/${pattern}
 }
 
 with_mage() {

.buildkite/scripts/dra_release.sh

@@ -24,8 +24,6 @@ fi
 
 add_bin_path
 
-google_cloud_auth
-
 download_mbp_packages_from_gcp_bucket "${FOLDER_PATH}" "${TYPE}"
 
 with_go

.buildkite/scripts/package.sh

@@ -38,5 +38,4 @@ case "${TYPE}" in
     ;;
 esac
 
-google_cloud_auth
 upload_mbp_packages_to_gcp_bucket "build/distributions/**/*" "${TYPE}"

.buildkite/scripts/release_test.sh

@@ -4,9 +4,9 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
-echo "Checking gsutil command..."
-if ! command -v gsutil &> /dev/null ; then
-    echo "⚠️ gsutil is not installed"
+echo "Checking gcloud command..."
+if ! command -v gcloud &> /dev/null ; then
+    echo "⚠️ gcloud is not installed"
     exit 1
 fi
 
@@ -16,8 +16,6 @@ with_go
 
 make docker-release
 
-google_cloud_auth
-
 upload_packages_to_gcp_bucket "build/distributions/"
 
 make test-release