Merge pull request #534 from buildkite/SUP-3228-default-extra-volume-mounts

petetomasik · web-flow · commit 630ad3a6ffc5 · 2025-03-10T14:10:28.000-04:00
SUP-3228: Allow `extraVolumeMounts` for default params
diff --git a/README.md b/README.md
@@ -676,10 +676,139 @@ config:
 
 ### Extra volume mounts
 
+#### All containers
+
 In some situations, for example if you want to use [git mirrors](https://buildkite.com/docs/agent/v3#promoted-experiments-git-mirrors) you may want to attach extra volume mounts (in addition to the `/workspace` one) in all the pod containers.
 
 See [this example](internal/integration/fixtures/extra-volume-mounts.yaml), that will declare a new volume in the `podSpec` and mount it in all the containers. The benefit, is to have the same mounted path in all containers, including the `checkout` container.
 
+#### `checkout` containers
+
+In order to attach extra volumes to your `checkout` containers, define `config.default-checkout-params.extraVolumeMounts` in your configuration. Example:
+```yaml
+# values.yaml
+config:
+  default-checkout-params:
+    gitCredentialsSecret:
+      secretName: my-git-credentials
+    extraVolumeMounts:
+      - name: checkout-extra-dir
+        mountPath: /extra-checkout
+  pod-spec-patch:
+    containers:
+      - name: checkout
+        image: "buildkite/agent:latest"
+    volumes:
+      - name: checkout-extra-dir
+        hostPath:
+          path: /my/extra/dir/checkout
+          type: DirectoryOrCreate
+```
+
+Or `checkout.extraVolumeMounts` in the `kubernetes` plugin. Example:
+```yaml
+# pipeline.yml
+...
+  kubernetes:
+    checkout:
+      extraVolumeMounts:
+        - name: checkout-extra-dir
+          mountPath: /extra-checkout
+    podSpecPatch:
+      containers:
+        - name: checkout
+          image: "buildkite/agent:latest"
+      volumes:
+        - name: checkout-extra-dir
+          hostPath:
+            path: /my/extra/dir/checkout
+            type: DirectoryOrCreate
+```
+
+#### `command` containers
+
+In order to attach extra volumes to your `container-#` (`command`) containers, define `config.default-command-params.extraVolumeMounts` in your configuration. Example:
+```yaml
+# values.yaml
+config:
+  default-command-params:
+    extraVolumeMounts:
+      - name: command-extra-dir
+        mountPath: /extra-command
+  pod-spec-patch:
+    containers:
+      - name: container-0
+        image: "buildkite/agent:latest"
+    volumes:
+      - name: command-extra-dir
+        hostPath:
+          path: /my/extra/dir/command
+          type: DirectoryOrCreate
+```
+
+Or `commandParams.extraVolumeMounts` in the `kubernetes` plugin. Example:
+```yaml
+# pipeline.yml
+...
+  kubernetes:
+    commandParams:
+      extraVolumeMounts:
+        - name: command-extra-dir
+          mountPath: /extra-command
+    podSpecPatch:
+      containers:
+        - name: container-0
+          image: "buildkite/agent:latest"
+      volumes:
+        - name: command-extra-dir
+          hostPath:
+            path: /my/extra/dir/command
+            type: DirectoryOrCreate
+```
+
+#### `sidecar` containers
+
+In order to attach extra volumes to your `sidecar` containers, define `config.default-sidecar-params.extraVolumeMounts` in your configuration. Example:
+```yaml
+# values.yaml
+config:
+  default-sidecar-params:
+    extraVolumeMounts:
+      - name: sidecar-extra-dir
+        mountPath: /extra-sidecar
+  pod-spec-patch:
+    containers:
+      - name: checkout
+        image: "buildkite/agent:latest"
+    volumes:
+      - name: sidecar-extra-dir
+        hostPath:
+          path: /my/extra/dir/sidecar
+          type: DirectoryOrCreate
+```
+
+Or `sidecarParams.extraVolumeMounts` in the `kubernetes` plugin. Example:
+```yaml
+# pipeline.yml
+...
+  kubernetes:
+    sidecars:
+      - image: nginx:latest
+    sidecarParams:
+      extraVolumeMounts:
+        - name: sidecar-extra-dir
+          mountPath: /extra-sidecar
+    podSpecPatch:
+      containers:
+        - name: checkout
+          image: "buildkite/agent:latest"
+      volumes:
+        - name: sidecar-extra-dir
+          hostPath:
+            path: /my/extra/dir/sidecar
+            type: DirectoryOrCreate
+```
+
 ### Skipping checkout (v0.13.0 and later)
 
 For some steps, you may wish to avoid checkout (cloning a source repository).
diff --git a/charts/agent-stack-k8s/values.schema.json b/charts/agent-stack-k8s/values.schema.json
@@ -481,6 +481,14 @@
               "items": {
                 "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.EnvFromSource"
               }
+            },
+            "extraVolumeMounts": {
+              "type": "array",
+              "default": [],
+              "title": "extra volumes to mount to all checkout containers",
+              "items": {
+                "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount"
+              }
             }
           }
         },
@@ -502,6 +510,14 @@
               "items": {
                 "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.EnvFromSource"
               }
+            },
+            "extraVolumeMounts": {
+              "type": "array",
+              "default": [],
+              "title": "extra volumes to mount to all command containers",
+              "items": {
+                "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount"
+              }
             }
           }
         },
@@ -517,6 +533,14 @@
               "items": {
                 "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.EnvFromSource"
               }
+            },
+            "extraVolumeMounts": {
+              "type": "array",
+              "default": [],
+              "title": "extra volumes to mount to all sidecar containers",
+              "items": {
+                "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount"
+              }
             }
           }
         },
diff --git a/internal/controller/config/checkout_params.go b/internal/controller/config/checkout_params.go
@@ -18,6 +18,7 @@ type CheckoutParams struct {
 	GitMirrors           *GitMirrorsParams          `json:"gitMirrors,omitempty"`
 	GitCredentialsSecret *corev1.SecretVolumeSource `json:"gitCredentialsSecret,omitempty"`
 	EnvFrom              []corev1.EnvFromSource     `json:"envFrom,omitempty"`
+	ExtraVolumeMounts    []corev1.VolumeMount       `json:"extraVolumeMounts,omitempty"`
 }
 
 func (co *CheckoutParams) ApplyTo(podSpec *corev1.PodSpec, ctr *corev1.Container) {
@@ -31,6 +32,7 @@ func (co *CheckoutParams) ApplyTo(podSpec *corev1.PodSpec, ctr *corev1.Container
 	appendCommaSepToEnv(ctr, "BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG", co.SubmoduleCloneConfig)
 	co.GitMirrors.ApplyTo(podSpec, ctr)
 	ctr.EnvFrom = append(ctr.EnvFrom, co.EnvFrom...)
+	ctr.VolumeMounts = append(ctr.VolumeMounts, co.ExtraVolumeMounts...)
 }
 
 func (co *CheckoutParams) GitCredsSecret() *corev1.SecretVolumeSource {
diff --git a/internal/controller/config/command_params.go b/internal/controller/config/command_params.go
@@ -13,15 +13,17 @@ import (
 // CommandParams contains parameters that provide additional control over all
 // command container(s).
 type CommandParams struct {
-	Interposer Interposer             `json:"interposer,omitempty"`
-	EnvFrom    []corev1.EnvFromSource `json:"envFrom,omitempty"`
+	Interposer        Interposer             `json:"interposer,omitempty"`
+	EnvFrom           []corev1.EnvFromSource `json:"envFrom,omitempty"`
+	ExtraVolumeMounts []corev1.VolumeMount   `json:"extraVolumeMounts,omitempty"`
 }
 
 func (cmd *CommandParams) ApplyTo(ctr *corev1.Container) {
 	if cmd == nil || ctr == nil {
 		return
 	}
 	ctr.EnvFrom = append(ctr.EnvFrom, cmd.EnvFrom...)
+	ctr.VolumeMounts = append(ctr.VolumeMounts, cmd.ExtraVolumeMounts...)
 }
 
 // Command interprets the command and args fields of the container into a
diff --git a/internal/controller/config/sidecar_params.go b/internal/controller/config/sidecar_params.go
@@ -5,12 +5,14 @@ import corev1 "k8s.io/api/core/v1"
 // SidecarParams contains parameters that provide additional control over all sidecar
 // container(s).
 type SidecarParams struct {
-	EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
+	EnvFrom           []corev1.EnvFromSource `json:"envFrom,omitempty"`
+	ExtraVolumeMounts []corev1.VolumeMount   `json:"extraVolumeMounts,omitempty"`
 }
 
 func (sc *SidecarParams) ApplyTo(ctr *corev1.Container) {
 	if sc == nil || ctr == nil {
 		return
 	}
 	ctr.EnvFrom = append(ctr.EnvFrom, sc.EnvFrom...)
+	ctr.VolumeMounts = append(ctr.VolumeMounts, sc.ExtraVolumeMounts...)
 }
diff --git a/internal/controller/scheduler/scheduler.go b/internal/controller/scheduler/scheduler.go
@@ -498,8 +498,8 @@ func (w *worker) Build(podSpec *corev1.PodSpec, skipCheckout bool, inputs buildI
 			if c.Name == "" {
 				c.Name = fmt.Sprintf("%s-%d", "sidecar", i)
 			}
-			c.VolumeMounts = append(c.VolumeMounts, volumeMounts...)
 			w.cfg.DefaultSidecarParams.ApplyTo(&c)
+			c.VolumeMounts = append(c.VolumeMounts, volumeMounts...)
 			inputs.k8sPlugin.SidecarParams.ApplyTo(&c)
 			c.EnvFrom = append(c.EnvFrom, inputs.k8sPlugin.GitEnvFrom...)
 			podSpec.Containers = append(podSpec.Containers, c)
diff --git a/internal/integration/fixtures/extra-volume-mounts-command.yaml b/internal/integration/fixtures/extra-volume-mounts-command.yaml
@@ -0,0 +1,41 @@
+agents:
+  queue: "{{.queue}}"
+steps:
+  - label: ":k8s: Write file to extraVolumeMount on command containers"
+    key: write-file-to-extra-volume-mount-command
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: alpine:latest
+                command: ["sh"]
+                args:
+                  - "-c"
+                  - "touch /tmp/extra-volume-mount-command/foo-$${BUILDKITE_JOB_ID}.txt"
+              - image: alpine:latest
+                command:
+                  - |-
+                    COUNT=0
+                    until [[ $$((COUNT++)) == 15 ]]; do
+                      [[ -f "/tmp/extra-volume-mount-command/foo-$${BUILDKITE_JOB_ID}.txt" ]] && break
+                      echo "⚠️ Waiting for /tmp/extra-volume-mount-command/foo-$${BUILDKITE_JOB_ID}.txt to be written..."
+                      sleep 1
+                    done
+
+                    if ! [[ -f "/tmp/extra-volume-mount-command/foo-$${BUILDKITE_JOB_ID}.txt" ]]; then
+                      echo "⛔ /tmp/extra-volume-mount-command/foo-$${BUILDKITE_JOB_ID}.txt has not been written"
+                      exit 1
+                    fi
+
+                    echo "✅ /tmp/extra-volume-mount-command/foo-$${BUILDKITE_JOB_ID}.txt has been written"
+                    rm -f "/tmp/extra-volume-mount-command/foo-$${BUILDKITE_JOB_ID}.txt"
+            volumes:
+              - name: host-volume
+                hostPath:
+                  path: "/tmp/volumes/{{.queue}}"
+                  type: DirectoryOrCreate
+          commandParams:
+            extraVolumeMounts:
+              - name: host-volume
+                mountPath: /tmp/extra-volume-mount-command
+                subPath: extra-volume-mount-command
diff --git a/internal/integration/fixtures/extra-volume-mounts-sidecar.yaml b/internal/integration/fixtures/extra-volume-mounts-sidecar.yaml
@@ -0,0 +1,51 @@
+agents:
+  queue: "{{.queue}}"
+steps:
+  - label: ":k8s: Write file to extraVolumeMount on sidecar containers"
+    key: write-file-to-extra-volume-mount-sidecar
+    plugins:
+      - kubernetes:
+          sidecars:
+            - image: alpine:latest
+              command: ["sh"]
+              args:
+                - "-c"
+                - |-
+                  touch /tmp/extra-volume-mount-sidecar/pass-the-parcel
+                  rm -f "/tmp/extra-volume-mount/pass-the-parcel"
+                  mv /tmp/extra-volume-mount-sidecar/pass-the-parcel /tmp/extra-volume-mount/pass-the-parcel
+                  ls -lah /tmp/extra-volume-mount-sidecar
+                  ls -lah /tmp/extra-volume-mount
+          podSpec:
+            containers:
+              - image: alpine:latest
+                command:
+                  - |-
+                    COUNT=0
+                    until [[ $$((COUNT++)) == 15 ]]; do
+                      [[ -f "/tmp/extra-volume-mount/pass-the-parcel" ]] && break
+                      echo "⚠️   Waiting for my package to be delivered..."
+                      sleep 1
+                    done
+
+                    if ! [[ -f "/tmp/extra-volume-mount/pass-the-parcel" ]]; then
+                      echo "⛔ My package has not been delivered!"
+                      exit 1
+                    fi
+
+                    echo "✅ My package has been delivered!"
+                    rm -f "/tmp/extra-volume-mount/pass-the-parcel"
+            volumes:
+              - name: host-volume
+                hostPath:
+                  path: "/tmp/volumes/{{.queue}}"
+                  type: DirectoryOrCreate
+          sidecarParams:
+            extraVolumeMounts:
+              - name: host-volume
+                mountPath: /tmp/extra-volume-mount-sidecar
+                subPath: extra-volume-mount-sidecar
+          extraVolumeMounts:
+            - name: host-volume
+              mountPath: /tmp/extra-volume-mount
+              subPath: extra-volume-mount
diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go
@@ -373,6 +373,34 @@ func TestExtraVolumeMounts(t *testing.T) {
 	tc.AssertSuccess(ctx, build)
 }
 
+func TestExtraVolumeMountsCommandContainers(t *testing.T) {
+	tc := testcase{
+		T:       t,
+		Fixture: "extra-volume-mounts-command.yaml",
+		Repo:    repoHTTP,
+		GraphQL: api.NewClient(cfg.BuildkiteToken, cfg.GraphQLEndpoint),
+	}.Init()
+	ctx := context.Background()
+	pipelineID := tc.PrepareQueueAndPipelineWithCleanup(ctx)
+	tc.StartController(ctx, cfg)
+	build := tc.TriggerBuild(ctx, pipelineID)
+	tc.AssertSuccess(ctx, build)
+}
+
+func TestExtraVolumeMountsSidecars(t *testing.T) {
+	tc := testcase{
+		T:       t,
+		Fixture: "extra-volume-mounts-sidecar.yaml",
+		Repo:    repoHTTP,
+		GraphQL: api.NewClient(cfg.BuildkiteToken, cfg.GraphQLEndpoint),
+	}.Init()
+	ctx := context.Background()
+	pipelineID := tc.PrepareQueueAndPipelineWithCleanup(ctx)
+	tc.StartController(ctx, cfg)
+	build := tc.TriggerBuild(ctx, pipelineID)
+	tc.AssertSuccess(ctx, build)
+}
+
 func TestInvalidPodSpec(t *testing.T) {
 	tc := testcase{
 		T:       t,
diff --git a/internal/integration/testcase_test.go b/internal/integration/testcase_test.go
@@ -42,7 +42,7 @@ type testcase struct {
 // k8s labels are limited to length 63, we use the pipeline name as a label.
 // So we sometimes need to limit the length of the pipeline name too.
 func (t *testcase) ShortPipelineName() string {
-	return t.PipelineName[:min(len(t.PipelineName), 63)]
+	return strings.Trim(t.PipelineName[:min(len(t.PipelineName), 63)], "-")
 }
 
 func (t testcase) Init() testcase {

Original file line number	Diff line number	Diff line change
`@@ -481,6 +481,14 @@`
`481`	`481`	`"items": {`
`482`	`482`	`"$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.EnvFromSource"`
`483`	`483`	`}`
	`484`	`+ },`
	`485`	`+ "extraVolumeMounts": {`
	`486`	`+ "type": "array",`
	`487`	`+ "default": [],`
	`488`	`+ "title": "extra volumes to mount to all checkout containers",`
	`489`	`+ "items": {`
	`490`	`+ "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount"`
	`491`	`+ }`
`484`	`492`	`}`
`485`	`493`	`}`
`486`	`494`	`},`
`@@ -502,6 +510,14 @@`
`502`	`510`	`"items": {`
`503`	`511`	`"$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.EnvFromSource"`
`504`	`512`	`}`
	`513`	`+ },`
	`514`	`+ "extraVolumeMounts": {`
	`515`	`+ "type": "array",`
	`516`	`+ "default": [],`
	`517`	`+ "title": "extra volumes to mount to all command containers",`
	`518`	`+ "items": {`
	`519`	`+ "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount"`
	`520`	`+ }`
`505`	`521`	`}`
`506`	`522`	`}`
`507`	`523`	`},`
`@@ -517,6 +533,14 @@`
`517`	`533`	`"items": {`
`518`	`534`	`"$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.EnvFromSource"`
`519`	`535`	`}`
	`536`	`+ },`
	`537`	`+ "extraVolumeMounts": {`
	`538`	`+ "type": "array",`
	`539`	`+ "default": [],`
	`540`	`+ "title": "extra volumes to mount to all sidecar containers",`
	`541`	`+ "items": {`
	`542`	`+ "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount"`
	`543`	`+ }`
`520`	`544`	`}`
`521`	`545`	`}`
`522`	`546`	`},`
Original file line number	Diff line number	Diff line change
`@@ -18,6 +18,7 @@ type CheckoutParams struct {`
`18`	`18`	GitMirrors *GitMirrorsParams `json:"gitMirrors,omitempty"`
`19`	`19`	GitCredentialsSecret *corev1.SecretVolumeSource `json:"gitCredentialsSecret,omitempty"`
`20`	`20`	EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
	`21`	+ ExtraVolumeMounts []corev1.VolumeMount `json:"extraVolumeMounts,omitempty"`
`21`	`22`	`}`
`22`	`23`
`23`	`24`	`func (co CheckoutParams) ApplyTo(podSpec corev1.PodSpec, ctr *corev1.Container) {`
`@@ -31,6 +32,7 @@ func (co CheckoutParams) ApplyTo(podSpec corev1.PodSpec, ctr *corev1.Container`
`31`	`32`	`appendCommaSepToEnv(ctr, "BUILDKITE_GIT_SUBMODULE_CLONE_CONFIG", co.SubmoduleCloneConfig)`
`32`	`33`	`co.GitMirrors.ApplyTo(podSpec, ctr)`
`33`	`34`	`ctr.EnvFrom = append(ctr.EnvFrom, co.EnvFrom...)`
	`35`	`+ ctr.VolumeMounts = append(ctr.VolumeMounts, co.ExtraVolumeMounts...)`
`34`	`36`	`}`
`35`	`37`
`36`	`38`	`func (co CheckoutParams) GitCredsSecret() corev1.SecretVolumeSource {`
Original file line number	Diff line number	Diff line change
`@@ -13,15 +13,17 @@ import (`
`13`	`13`	`// CommandParams contains parameters that provide additional control over all`
`14`	`14`	`// command container(s).`
`15`	`15`	`type CommandParams struct {`
`16`		- Interposer Interposer `json:"interposer,omitempty"`
`17`		- EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
	`16`	+ Interposer Interposer `json:"interposer,omitempty"`
	`17`	+ EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
	`18`	+ ExtraVolumeMounts []corev1.VolumeMount `json:"extraVolumeMounts,omitempty"`
`18`	`19`	`}`
`19`	`20`
`20`	`21`	`func (cmd CommandParams) ApplyTo(ctr corev1.Container) {`
`21`	`22`	`if cmd == nil \|\| ctr == nil {`
`22`	`23`	`return`
`23`	`24`	`}`
`24`	`25`	`ctr.EnvFrom = append(ctr.EnvFrom, cmd.EnvFrom...)`
	`26`	`+ ctr.VolumeMounts = append(ctr.VolumeMounts, cmd.ExtraVolumeMounts...)`
`25`	`27`	`}`
`26`	`28`
`27`	`29`	`// Command interprets the command and args fields of the container into a`
Original file line number	Diff line number	Diff line change
`@@ -5,12 +5,14 @@ import corev1 "k8s.io/api/core/v1"`
`5`	`5`	`// SidecarParams contains parameters that provide additional control over all sidecar`
`6`	`6`	`// container(s).`
`7`	`7`	`type SidecarParams struct {`
`8`		- EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
	`8`	+ EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
	`9`	+ ExtraVolumeMounts []corev1.VolumeMount `json:"extraVolumeMounts,omitempty"`
`9`	`10`	`}`
`10`	`11`
`11`	`12`	`func (sc SidecarParams) ApplyTo(ctr corev1.Container) {`
`12`	`13`	`if sc == nil \|\| ctr == nil {`
`13`	`14`	`return`
`14`	`15`	`}`
`15`	`16`	`ctr.EnvFrom = append(ctr.EnvFrom, sc.EnvFrom...)`
	`17`	`+ ctr.VolumeMounts = append(ctr.VolumeMounts, sc.ExtraVolumeMounts...)`
`16`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,7 @@ type testcase struct {`
`42`	`42`	`// k8s labels are limited to length 63, we use the pipeline name as a label.`
`43`	`43`	`// So we sometimes need to limit the length of the pipeline name too.`
`44`	`44`	`func (t *testcase) ShortPipelineName() string {`
`45`		`- return t.PipelineName[:min(len(t.PipelineName), 63)]`
	`45`	`+ return strings.Trim(t.PipelineName[:min(len(t.PipelineName), 63)], "-")`
`46`	`46`	`}`
`47`	`47`
`48`	`48`	`func (t testcase) Init() testcase {`