buildkite: use GCP OIDC (elastic#13856)

v1v · web-flow · commit 47b443259787 · 2025-05-09T19:00:45.000+02:00
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -31,8 +31,6 @@ export TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 REPO_BUILD_TAG="${REPO_NAME}/$(buildkite_pr_branch_build_id)"
 export REPO_BUILD_TAG
 
-PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account
-
 BUILDKITE_API_TOKEN_PATH=kv/ci-shared/platform-ingest/buildkite_token
 
 EC_TOKEN_PATH=kv/ci-shared/platform-ingest/platform-ingest-ec-qa
@@ -42,6 +40,8 @@ EC_DATA_PATH=secret/ci/elastic-integrations/ec_data
 export ENVIRONMENT="ci"
 export REPO="${REPO_NAME}"
 
+export JOB_GCS_BUCKET_INTERNAL="ecosystem-ci-internal"
+
 branch_name_label() {
     local branch="$1"
 
@@ -106,23 +106,13 @@ if [[ "${BUILDKITE_PIPELINE_SLUG}" =~ ^(integrations|integrations-test-stack)$ ]
     fi
 
     if [[ "${BUILDKITE_STEP_KEY}" =~ ^test-integrations- ]]; then
-        PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
-        export PRIVATE_CI_GCS_CREDENTIALS_SECRET
-        export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
-
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
     fi
 fi
 
 if [[ "${BUILDKITE_PIPELINE_SLUG}" == "integrations-serverless" ]]; then
     if [[ "${BUILDKITE_STEP_KEY}" == "test-integrations-serverless-project" ]]; then
-        # Currently, system tests are not run when testing with an Elastic Serverless project, so it is not required to
-        # add the AWS credentials as in the integrations pipeline.
-
-        PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json "${PRIVATE_CI_GCS_CREDENTIALS_PATH}")
-        export PRIVATE_CI_GCS_CREDENTIALS_SECRET
-        export JOB_GCS_BUCKET_INTERNAL="ingest-buildkite-ci"
 
         BUILDKITE_API_TOKEN=$(retry 5 vault kv get -field buildkite_token "${BUILDKITE_API_TOKEN_PATH}")
         export BUILDKITE_API_TOKEN
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
@@ -7,10 +7,6 @@ set -euo pipefail
 if [[ "$BUILDKITE_PIPELINE_SLUG" =~ ^(integrations|integrations-test-stack)$ ]]; then
     # FIXME: update condition depending on the pipeline steps triggered
     if [[ "$BUILDKITE_STEP_KEY" =~ ^test-integrations- ]]; then
-        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
-        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
-        unset AWS_ACCESS_KEY_ID
-        unset AWS_SECRET_ACCESS_KEY
 
         # Ensure that kind cluster is deleted
         delete_kind_cluster
@@ -25,10 +21,6 @@ fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-serverless" ]]; then
     if [[ "$BUILDKITE_STEP_KEY" == "test-integrations-serverless-project" ]]; then
-        unset ELASTIC_PACKAGE_AWS_ACCESS_KEY
-        unset ELASTIC_PACKAGE_AWS_SECRET_KEY
-        unset AWS_ACCESS_KEY_ID
-        unset AWS_SECRET_ACCESS_KEY
 
         # Ensure that kind cluster is deleted
         delete_kind_cluster
@@ -44,8 +36,6 @@ fi
 unset_secrets
 cleanup
 
-google_cloud_logout_active_account
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "integrations-backport" && "$BUILDKITE_STEP_KEY" == "create-backport-branch" ]]; then
   cd "${WORKSPACE}"
   git config remote.origin.url "https://github.com/elastic/integrations.git"
diff --git a/.buildkite/pipeline.serverless.yml b/.buildkite/pipeline.serverless.yml
@@ -72,6 +72,12 @@ steps:
       # This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
       - elastic/oblt-aws-auth#v0.1.0:
           duration: 10800 # seconds
+      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
+      # This plugin authenticates to Google Cloud using the OIDC token.
+      - elastic/oblt-google-auth#v1.3.0:
+          lifetime: 10800 # seconds
+          project-id: "elastic-observability-ci"
+          project-number: "911195782929"
     artifact_paths:
       - "build/test-results/*.xml"
       - "build/elastic-stack-dump/*/logs/*.log"
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
@@ -10,7 +10,6 @@ platform_type_lowercase="${platform_type,,}"
 
 SCRIPTS_BUILDKITE_PATH="${WORKSPACE}/.buildkite/scripts"
 
-GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
 export ELASTIC_PACKAGE_BIN=${WORKSPACE}/build/elastic-package
 
 API_BUILDKITE_PIPELINES_URL="https://api.buildkite.com/v2/organizations/elastic/pipelines/"
@@ -255,34 +254,6 @@ with_github_cli() {
     gh version
 }
 
-## Logging and logout from Google Cloud
-google_cloud_auth_safe_logs() {
-    local gsUtilLocation
-    gsUtilLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE}")
-    local secretFileLocation=${gsUtilLocation}/${GOOGLE_CREDENTIALS_FILENAME}
-
-    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > "${secretFileLocation}"
-
-    gcloud auth activate-service-account --key-file "${secretFileLocation}" 2> /dev/null
-    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
-}
-
-google_cloud_logout_active_account() {
-  local active_account
-  active_account=$(gcloud auth list --filter=status:ACTIVE --format="value(account)" 2>/dev/null || true)
-  if [[ -n "$active_account" && -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]]; then
-    echo "Logging out from GCP for active account"
-    gcloud auth revoke "$active_account" > /dev/null 2>&1
-  else
-    echo "No active GCP accounts found."
-  fi
-
-  if [ -n "${GOOGLE_APPLICATION_CREDENTIALS+x}" ]; then
-    rm -rf "${GOOGLE_APPLICATION_CREDENTIALS}"
-    unset GOOGLE_APPLICATION_CREDENTIALS
-  fi
-}
-
 ## Helpers for integrations pipelines
 check_git_diff() {
     cd "${WORKSPACE}"
@@ -983,16 +954,16 @@ upload_safe_logs() {
     local source="$2"
     local target="$3"
 
+    echo "--- Uploading safe logs to GCP bucket ${bucket}"
+
     if ! ls ${source} 2>&1 > /dev/null ; then
         echo "upload_safe_logs: artifacts files not found, nothing will be archived"
         return
     fi
 
-    google_cloud_auth_safe_logs
-
-    gsutil cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
+    gcloud storage cp ${source} "gs://${bucket}/buildkite/${REPO_BUILD_TAG}/${target}"
 
-    google_cloud_logout_active_account
+    echo "GCP logout is not required, the BK plugin will do it for us"
 }
 
 clean_safe_logs() {
diff --git a/.buildkite/scripts/trigger_integrations_in_parallel.sh b/.buildkite/scripts/trigger_integrations_in_parallel.sh
@@ -93,6 +93,12 @@ for package in ${PACKAGE_LIST}; do
         # This plugin creates the environment variables required by the service deployer (AWS_SECRET_ACCESS_KEY and AWS_SECRET_KEY_ID)
         - elastic/oblt-aws-auth#v0.1.0:
             duration: 10800 # seconds
+        # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/integrations/01-gcp-buildkite-oidc.tf
+        # This plugin authenticates to Google Cloud using the OIDC token.
+        - elastic/oblt-google-auth#v1.3.0:
+            lifetime: 10800 # seconds
+            project-id: "elastic-observability-ci"
+            project-number: "911195782929"
       artifact_paths:
         - build/test-results/*.xml
         - build/test-coverage/*.xml
