Merge pull request #468 from valkum/gcp-token

feat: add support for GCP token propagation

Author: pzeballos

README.md

@@ -93,6 +93,12 @@ Will propagate `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`
 
 When the `AWS_WEB_IDENTITY_TOKEN_FILE` is specified, it will also mount it automatically for you and make it usable within the container.
 
+#### `propagate-gcp-auth-tokens` (run only, boolean)
+
+Whether or not to automatically propagate gcp auth credentials into the docker container. Avoiding the need to be specified with `environment`. This is useful if you are using a workload identity federation to impersonate a service account and you want to pass it to the docker container. This is compatible with the `gcp-workload-identity-federation` plugin.
+
+Will propagate `GOOGLE_APPLICATION_CREDENTIALS`, `CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE` and `BUILDKITE_OIDC_TMPDIR` and also mount the dir specified by `BUILDKITE_OIDC_TMPDIR` into the container.
+
 #### `command` (run only, array)
 
 Sets the command for the Docker image, and defaults the `shell` option to `false`. Useful if the Docker image has an entrypoint, or doesn't contain a shell.

commands/run.sh

@@ -164,6 +164,23 @@ if [[ "$(plugin_read_config PROPAGATE_AWS_AUTH_TOKENS "false")" =~ ^(true|on|1)$
   fi
 fi
 
+# Propagate gcp auth environment variables into the container e.g. from workload identity federation plugins
+if [[ "$(plugin_read_config PROPAGATE_GCP_AUTH_TOKENS "false")" =~ ^(true|on|1)$ ]] ; then
+  if [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]] ; then
+      run_params+=( --env "GOOGLE_APPLICATION_CREDENTIALS" )
+  fi
+  if [[ -n "${CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE:-}" ]] ; then
+      run_params+=( --env "CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE" )
+  fi
+  if [[ -n "${BUILDKITE_OIDC_TMPDIR:-}" ]] ; then
+      run_params+=( --env "BUILDKITE_OIDC_TMPDIR" )
+      # Add the OIDC temp dir as a volume
+      run_params+=( --volume "${BUILDKITE_OIDC_TMPDIR}:${BUILDKITE_OIDC_TMPDIR}" )
+  fi
+fi
+
+
+
 # If requested, propagate a set of env vars as listed in a given env var to the
 # container.
 if [[ -n "$(plugin_read_config ENV_PROPAGATION_LIST)" ]]; then

docs/examples.md

@@ -10,7 +10,7 @@ The following pipeline will run `test.sh` inside a `app` service container using
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: app
 ```
 
@@ -19,7 +19,7 @@ steps:
 ```yml
 steps:
   - plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: app
           command: ["custom", "command", "values"]
 ```
@@ -30,7 +30,7 @@ The plugin will honor the value of the `COMPOSE_FILE` environment variable if on
 steps:
   - command: test.sh
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: app
           config: docker-compose.tests.yml
           env:
@@ -46,15 +46,15 @@ steps:
   - plugins:
       - docker-login#v2.0.1:
           username: xyz
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           push: app:index.docker.io/myorg/myrepo:tag
   - wait
   - command: test.sh
     plugins:
       - docker-login#v2.0.1:
           username: xyz
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: app
 ```
 
@@ -71,7 +71,7 @@ steps:
   - command: generate-dist.sh
     artifact_paths: "dist/*"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: app
           volumes:
             - "./dist:/folder/dist"
@@ -95,7 +95,7 @@ this plugin offers a `environment` block of its own:
 steps:
   - command: generate-dist.sh
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: app
           env:
             - BUILDKITE_BUILD_NUMBER
@@ -113,7 +113,7 @@ Alternatively, you can have the plugin add all environment variables defined for
 steps:
   - command: use-vars.sh
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: app
           propagate-environment: true
 ```
@@ -129,7 +129,7 @@ steps:
     env:
       COMPOSE_PROFILES: "frontend,debug"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: app
 ```
 
@@ -138,6 +138,7 @@ It is important to understand that, as documented in the official documentation,
 ### Container Labels
 
 When running a command, the plugin will automatically add the following Docker labels to the container specified in the `run` option:
+
 - `com.buildkite.pipeline_name=${BUILDKITE_PIPELINE_NAME}`
 - `com.buildkite.pipeline_slug=${BUILDKITE_PIPELINE_SLUG}`
 - `com.buildkite.build_number=${BUILDKITE_BUILD_NUMBER}`
@@ -165,7 +166,7 @@ Alternatively, if you want to set build arguments when pre-building an image, th
 steps:
   - command: generate-dist.sh
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           args:
             - MY_CUSTOM_ARG=panda
@@ -182,7 +183,7 @@ If you have multiple steps that use the same service/image (such as steps that r
 steps:
   - label: ":docker: Build"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           push: app
 
@@ -192,7 +193,7 @@ steps:
     command: test.sh
     parallelism: 25
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: app
           require-prebuild: true
 ```
@@ -211,7 +212,7 @@ steps:
     agents:
       queue: docker-builder
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build:
             - app
             - tests
@@ -225,7 +226,7 @@ steps:
     command: test.sh
     parallelism: 25
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           run: tests
 ```
 
@@ -237,7 +238,7 @@ If you want to push your Docker images ready for deployment, you can use the `pu
 steps:
   - label: ":docker: Push"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           push: app
 ```
 
@@ -247,7 +248,7 @@ To push multiple images, you can use a list:
 steps:
   - label: ":docker: Push"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           push:
             - first-service
             - second-service
@@ -259,7 +260,7 @@ If you want to push to a specific location (that's not defined as the `image` in
 steps:
   - label: ":docker: Push"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           push:
             - app:index.docker.io/myorg/myrepo/myapp
             - app:index.docker.io/myorg/myrepo/myapp:latest
@@ -273,7 +274,7 @@ A newly spawned agent won't contain any of the docker caches for the first run w
 steps:
   - label: ":docker: Build an image"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           push: app:index.docker.io/myorg/myrepo:my-branch
           cache-from:
@@ -284,7 +285,7 @@ steps:
 
   - label: ":docker: Push to final repository"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           push:
             - app:myregistry:port/myrepo/myapp:latest
 ```
@@ -297,7 +298,7 @@ The values you add in the `cache-from` will be mapped to the corresponding servi
 steps:
   - label: ":docker: Build an image"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           push: app:index.docker.io/myorg/myrepo:my-branch
           cache-from:
@@ -308,7 +309,7 @@ steps:
 
   - label: ":docker: Push to final repository"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           push:
             - app:myregistry:port/myrepo/myapp:latest
 ```
@@ -325,7 +326,7 @@ The `docker` driver can handle most situations but for advance features with the
 steps:
   - label: ":docker: Build an image"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           push: app:index.docker.io/myorg/myrepo:my-branch
           cache-from:
@@ -347,7 +348,7 @@ By default, Builder Instances specified by `name` or that are created with `crea
 steps:
   - label: ":docker: Build an image"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           push: app:index.docker.io/myorg/myrepo:my-branch
           cache-from:
@@ -366,7 +367,7 @@ By default, Builder Instances specified by `name` or that are created with `crea
 steps:
   - label: ":docker: Build an image"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           push: app:index.docker.io/myorg/myrepo:my-branch
           cache-from:
@@ -392,7 +393,7 @@ A newly spawned agent won't contain any of the docker caches for the first run w
 steps:
   - label: ":docker: Build an image and push cache"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           push: app:${DOCKER_REGISTRY}/${IMAGE_REPO}:cache
           cache-from:
@@ -409,7 +410,7 @@ steps:
 
   - label: ":docker: Build an image using remote cache"
     plugins:
-      - docker-compose#v5.5.0:
+      - docker-compose#v5.7.0:
           build: app
           cache-from:
             - "app:type=registry,ref=${DOCKER_REGISTRY}/${IMAGE_REPO}:cache"

plugin.yml

@@ -115,6 +115,10 @@ configuration:
     progress:
       type: string
       enum: [ "auto", "tty", "plain", "json", "quiet" ]
+    propagate-aws-auth-tokens:
+      type: boolean
+    propagate-gcp-auth-tokens:
+      type: boolean
     propagate-environment:
       type: boolean
     propagate-uid-gid:

tests/run.bats

@@ -1340,3 +1340,29 @@ cmd3"
 
   unstub docker
 }
+
+@test "Run with propagate gcp auth tokens" {
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_RUN=myservice
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CHECK_LINKED_CONTAINERS=false
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CLEANUP=false
+  export BUILDKITE_COMMAND="echo hello world"
+  export BUILDKITE_PLUGIN_DOCKER_COMPOSE_PROPAGATE_GCP_AUTH_TOKENS=true
+
+  export BUILDKITE_OIDC_TMPDIR="/tmp/.tmp.Xdasd23"
+  export GOOGLE_APPLICATION_CREDENTIALS="${BUILDKITE_OIDC_TMPDIR}/credentials.json"
+  export CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE="${GOOGLE_APPLICATION_CREDENTIALS}"
+
+  stub docker \
+    "compose -f docker-compose.yml -p buildkite1111 up -d --scale myservice=0 myservice : echo ran myservice dependencies" \
+    "compose -f docker-compose.yml -p buildkite1111 run --name buildkite1111_myservice_build_1 --env GOOGLE_APPLICATION_CREDENTIALS --env CLOUDSDK_AUTH_CREDENTIAL_FILE_OVERRIDE --env BUILDKITE_OIDC_TMPDIR --volume \"/tmp/.tmp.Xdasd23:/tmp/.tmp.Xdasd23\" -T --rm myservice /bin/sh -e -c 'echo hello world' : echo ran myservice"
+
+  stub buildkite-agent \
+    "meta-data exists docker-compose-plugin-built-image-tag-myservice : exit 1"
+
+  run "$PWD"/hooks/command
+
+  assert_success
+  assert_output --partial "ran myservice"
+  unstub docker
+  unstub buildkite-agent
+}