Merge pull request #428 from buildkite/configurable-workspace-volume

Configurable workspace volume

Author: DrJosh9000

README.md

@@ -559,6 +559,26 @@ Sidecar containers can be added to your job by specifying them under the top-lev
 
 There is no guarantee that your sidecars will have started before your job, so using retries or a tool like [wait-for-it](https://github.com/vishnubob/wait-for-it) is a good idea to avoid flaky tests.
 
+### The workspace volume
+
+By default the workspace directory (`/workspace`) is mounted as an `emptyDir` ephemeral volume. Other volumes may be more desirable (e.g. a volume claim backed by an NVMe device).
+The default workspace volume can be set as stack configuration, e.g.
+
+```yaml
+# values.yaml
+config:
+  workspace-volume:
+    name: workspace-2-the-reckoning
+    ephemeral:
+      volumeClaimTemplate:
+        spec:
+          accessModes: ["ReadWriteOnce"]
+          storageClassName: my-special-storage-class
+          resources:
+            requests:
+              storage: 1Gi
+```
+
 ### Extra volume mounts
 
 In some situations, for example if you want to use [git mirrors](https://buildkite.com/docs/agent/v3#promoted-experiments-git-mirrors) you may want to attach extra volume mounts (in addition to the `/workspace` one) in all the pod containers.

charts/agent-stack-k8s/values.schema.json

@@ -262,6 +262,9 @@
           "title": "Causes the controller to prohibit the kubernetes plugin specified within jobs (pipeline YAML) - enabling this causes jobs with a kubernetes plugin to fail, preventing the pipeline YAML from having any influence over the podSpec",
           "examples": [true]
         },
+        "workspaceVolume": {
+          "$ref": "https://kubernetesjsonschema.dev/master/_definitions.json#/definitions/io.k8s.api.core.v1.Volume"
+        },
         "agent-config": {
           "type": "object",
           "default": null,

cmd/controller/controller_test.go

@@ -36,10 +36,28 @@ func TestReadAndParseConfig(t *testing.T) {
 		ClusterUUID:                  "beefcafe-abbe-baba-abba-deedcedecade",
 		ProhibitKubernetesPlugin:     true,
 		GraphQLEndpoint:              "http://graphql.buildkite.localhost/v1",
+
+		WorkspaceVolume: &corev1.Volume{
+			Name: "workspace-2-the-reckoning",
+			VolumeSource: corev1.VolumeSource{
+				Ephemeral: &corev1.EphemeralVolumeSource{
+					VolumeClaimTemplate: &corev1.PersistentVolumeClaimTemplate{
+						Spec: corev1.PersistentVolumeClaimSpec{
+							AccessModes:      []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
+							StorageClassName: ptr("my-special-storage-class"),
+							Resources: corev1.VolumeResourceRequirements{
+								Requests: corev1.ResourceList{
+									"storage": resource.MustParse("1Gi"),
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 		AgentConfig: &config.AgentConfig{
 			Endpoint: ptr("http://agent.buildkite.localhost/v3"),
 		},
-
 		DefaultCommandParams: &config.CommandParams{
 			Interposer: config.InterposerVector,
 			EnvFrom: []corev1.EnvFromSource{{

examples/config.yaml

@@ -28,6 +28,19 @@ tags:
 # preventing the pipeline YAML from having any influence over the podSpec
 prohibit-kubernetes-plugin: true
 
+# The workspace volume can be overriden from its default (an emptyDir named
+# 'workspace').
+workspace-volume:
+  name: workspace-2-the-reckoning
+  ephemeral:
+    volumeClaimTemplate:
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        storageClassName: my-special-storage-class
+        resources:
+          requests:
+            storage: 1Gi
+
 # Applies to all agents
 agent-config:
   # Setting a custom Agent REST API endpoint is usually only useful if you have

internal/controller/config/config.go

@@ -49,6 +49,10 @@ type Config struct {
 	ImagePullBackOffGracePeriod  time.Duration   `json:"image-pull-backoff-grace-period"  validate:"omitempty"`
 	JobCancelCheckerPollInterval time.Duration   `json:"job-cancel-checker-poll-interval" validate:"omitempty"`
 
+	// WorkspaceVolume allows supplying a volume for /workspace. By default
+	// an EmptyDir volume is created for it.
+	WorkspaceVolume *corev1.Volume `json:"workspace-volume" validate:"omitempty"`
+
 	AgentConfig           *AgentConfig    `json:"agent-config"            validate:"omitempty"`
 	DefaultCheckoutParams *CheckoutParams `json:"default-checkout-params" validate:"omitempty"`
 	DefaultCommandParams  *CommandParams  `json:"default-command-params"  validate:"omitempty"`

internal/controller/controller.go

@@ -66,6 +66,7 @@ func Run(
 		AgentTokenSecretName:   cfg.AgentTokenSecret,
 		JobTTL:                 cfg.JobTTL,
 		AdditionalRedactedVars: cfg.AdditionalRedactedVars,
+		WorkspaceVolume:        cfg.WorkspaceVolume,
 		AgentConfig:            cfg.AgentConfig,
 		DefaultCheckoutParams:  cfg.DefaultCheckoutParams,
 		DefaultCommandParams:   cfg.DefaultCommandParams,

internal/controller/scheduler/scheduler.go

@@ -46,6 +46,7 @@ type Config struct {
 	AgentTokenSecretName   string
 	JobTTL                 time.Duration
 	AdditionalRedactedVars []string
+	WorkspaceVolume        *corev1.Volume
 	AgentConfig            *config.AgentConfig
 	DefaultCheckoutParams  *config.CheckoutParams
 	DefaultCommandParams   *config.CommandParams
@@ -286,7 +287,25 @@ func (w *worker) Build(podSpec *corev1.PodSpec, skipCheckout bool, inputs buildI
 		Value: strings.Join(redactedVars, ","),
 	})
 
-	volumeMounts := []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}}
+	// workspaceVolume is shared among most containers, so set it up first.
+	workspaceVolume := w.cfg.WorkspaceVolume
+	if workspaceVolume == nil {
+		// The default workspace volume is an empty dir volume.
+		workspaceVolume = &corev1.Volume{
+			Name: "workspace",
+			VolumeSource: corev1.VolumeSource{
+				EmptyDir: &corev1.EmptyDirVolumeSource{},
+			},
+		}
+	}
+	podSpec.Volumes = append(podSpec.Volumes, *workspaceVolume)
+
+	// Set up other volumes (hooks, plugins, keys).
+	w.cfg.AgentConfig.ApplyVolumesTo(podSpec)
+
+	// Volume mounts shared among most containers: the workspace volume, and
+	// any others supplied with ExtraVolumeMounts.
+	volumeMounts := []corev1.VolumeMount{{Name: workspaceVolume.Name, MountPath: "/workspace"}}
 	if inputs.k8sPlugin != nil {
 		volumeMounts = append(volumeMounts, inputs.k8sPlugin.ExtraVolumeMounts...)
 	}
@@ -614,14 +633,7 @@ func (w *worker) Build(podSpec *corev1.PodSpec, skipCheckout bool, inputs buildI
 
 	podSpec.InitContainers = append(initContainers, podSpec.InitContainers...)
 
-	podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{
-		Name: "workspace",
-		VolumeSource: corev1.VolumeSource{
-			EmptyDir: &corev1.EmptyDirVolumeSource{},
-		},
-	})
-	w.cfg.AgentConfig.ApplyVolumesTo(podSpec)
-
+	// Only attempt the job once.
 	podSpec.RestartPolicy = corev1.RestartPolicyNever
 
 	// Allow podSpec to be overridden by the agent configuration and the k8s plugin