Merge pull request #406 from buildkite/clean-up-pending-cancelled

Clean up pending pods for cancelled jobs

Author: DrJosh9000

charts/agent-stack-k8s/values.schema.json

@@ -174,7 +174,7 @@
         "image": {
           "type": "string",
           "default": "",
-          "title": "The image Schema",
+          "title": "The container image used to obtain buildkite-agent, and for running commands without any k8s-specific configuration. The default for each release of agent-stack-k8s is set to a version tag of gchr.io/buildkite/agent matching agent-stack-k8s's go.mod file",
           "examples": [""]
         },
         "debug": {
@@ -232,6 +232,18 @@
           },
           "examples": [["SECRET_RECIPE"]]
         },
+        "image-pull-backoff-grace-period": {
+          "type": "string",
+          "default": "30s",
+          "title": "Duration after starting a pod that the controller will wait before considering cancelling a job due to ImagePullBackOff (e.g. when the podSpec specifies container images that cannot be pulled). Must be a Go duration string",
+          "examples": ["60s"]
+        },
+        "job-cancel-checker-poll-interval": {
+          "type": "string",
+          "default": "5s",
+          "title": "Controls the interval between job state queries while a pod is still Pending. Must be a Go duration string",
+          "examples": ["10s"]
+        },
         "prohibit-kubernetes-plugin": {
           "type": "boolean",
           "default": false,

cmd/controller/controller.go

@@ -90,6 +90,11 @@ func AddConfigFlags(cmd *cobra.Command) {
 		config.DefaultImagePullBackOffGracePeriod,
 		"Duration after starting a pod that the controller will wait before considering cancelling a job due to ImagePullBackOff (e.g. when the podSpec specifies container images that cannot be pulled)",
 	)
+	cmd.Flags().Duration(
+		"job-cancel-checker-poll-interval",
+		config.DefaultJobCancelCheckerPollInterval,
+		"Controls the interval between job state queries while a pod is still Pending",
+	)
 	cmd.Flags().Bool(
 		"prohibit-kubernetes-plugin",
 		false,

cmd/controller/controller_test.go

@@ -19,20 +19,21 @@ func ptr[T any](v T) *T {
 
 func TestReadAndParseConfig(t *testing.T) {
 	expected := config.Config{
-		Debug:                       true,
-		AgentTokenSecret:            "my-kubernetes-secret",
-		BuildkiteToken:              "my-graphql-enabled-token",
-		Image:                       "my.registry.dev/buildkite-agent:latest",
-		JobTTL:                      300 * time.Second,
-		ImagePullBackOffGradePeriod: 60 * time.Second,
-		PollInterval:                5 * time.Second,
-		MaxInFlight:                 100,
-		Namespace:                   "my-buildkite-ns",
-		Org:                         "my-buildkite-org",
-		Tags:                        []string{"queue=my-queue", "priority=high"},
-		ClusterUUID:                 "beefcafe-abbe-baba-abba-deedcedecade",
-		ProhibitKubernetesPlugin:    true,
-		GraphQLEndpoint:             "http://graphql.buildkite.localhost/v1",
+		Debug:                        true,
+		AgentTokenSecret:             "my-kubernetes-secret",
+		BuildkiteToken:               "my-graphql-enabled-token",
+		Image:                        "my.registry.dev/buildkite-agent:latest",
+		JobTTL:                       300 * time.Second,
+		ImagePullBackOffGracePeriod:  60 * time.Second,
+		JobCancelCheckerPollInterval: 10 * time.Second,
+		PollInterval:                 5 * time.Second,
+		MaxInFlight:                  100,
+		Namespace:                    "my-buildkite-ns",
+		Org:                          "my-buildkite-org",
+		Tags:                         []string{"queue=my-queue", "priority=high"},
+		ClusterUUID:                  "beefcafe-abbe-baba-abba-deedcedecade",
+		ProhibitKubernetesPlugin:     true,
+		GraphQLEndpoint:              "http://graphql.buildkite.localhost/v1",
 		AgentConfig: &config.AgentConfig{
 			Endpoint: ptr("http://agent.buildkite.localhost/v3"),
 		},

examples/config.yaml

@@ -3,6 +3,7 @@ debug: true
 image: my.registry.dev/buildkite-agent:latest
 job-ttl: 5m
 image-pull-backoff-grace-period: 60s
+job-cancel-checker-poll-interval: 10s
 poll-interval: 5s
 max-in-flight: 100
 namespace: my-buildkite-ns

internal/controller/config/config.go

@@ -11,11 +11,12 @@ import (
 )
 
 const (
-	UUIDLabel                          = "buildkite.com/job-uuid"
-	BuildURLAnnotation                 = "buildkite.com/build-url"
-	JobURLAnnotation                   = "buildkite.com/job-url"
-	DefaultNamespace                   = "default"
-	DefaultImagePullBackOffGracePeriod = 30 * time.Second
+	UUIDLabel                           = "buildkite.com/job-uuid"
+	BuildURLAnnotation                  = "buildkite.com/build-url"
+	JobURLAnnotation                    = "buildkite.com/job-url"
+	DefaultNamespace                    = "default"
+	DefaultImagePullBackOffGracePeriod  = 30 * time.Second
+	DefaultJobCancelCheckerPollInterval = 5 * time.Second
 )
 
 var DefaultAgentImage = "ghcr.io/buildkite/agent:" + version.Version()
@@ -40,10 +41,11 @@ type Config struct {
 
 	// ClusterUUID field is mandatory for most new orgs.
 	// Some old orgs allows unclustered setup.
-	ClusterUUID                 string          `json:"cluster-uuid"                    validate:"omitempty"`
-	AdditionalRedactedVars      stringSlice     `json:"additional-redacted-vars"        validate:"omitempty"`
-	PodSpecPatch                *corev1.PodSpec `json:"pod-spec-patch"                  validate:"omitempty"`
-	ImagePullBackOffGradePeriod time.Duration   `json:"image-pull-backoff-grace-period" validate:"omitempty"`
+	ClusterUUID                  string          `json:"cluster-uuid"                     validate:"omitempty"`
+	AdditionalRedactedVars       stringSlice     `json:"additional-redacted-vars"         validate:"omitempty"`
+	PodSpecPatch                 *corev1.PodSpec `json:"pod-spec-patch"                   validate:"omitempty"`
+	ImagePullBackOffGracePeriod  time.Duration   `json:"image-pull-backoff-grace-period"  validate:"omitempty"`
+	JobCancelCheckerPollInterval time.Duration   `json:"job-cancel-checker-poll-interval" validate:"omitempty"`
 
 	AgentConfig           *AgentConfig    `json:"agent-config" validate:"omitempty"`
 	DefaultCheckoutParams *CheckoutParams `json:"default-checkout-params" validate:"omitempty"`
@@ -83,7 +85,8 @@ func (c Config) MarshalLogObject(enc zapcore.ObjectEncoder) error {
 	if err := enc.AddReflected("pod-spec-patch", c.PodSpecPatch); err != nil {
 		return err
 	}
-	enc.AddDuration("image-pull-backoff-grace-period", c.ImagePullBackOffGradePeriod)
+	enc.AddDuration("image-pull-backoff-grace-period", c.ImagePullBackOffGracePeriod)
+	enc.AddDuration("job-cancel-checker-poll-interval", c.JobCancelCheckerPollInterval)
 	if err := enc.AddReflected("agent-config", c.AgentConfig); err != nil {
 		return err
 	}

internal/controller/controller.go

@@ -80,13 +80,13 @@ func Run(
 		logger.Fatal("failed to register completions informer", zap.Error(err))
 	}
 
-	imagePullBackOffWatcher := scheduler.NewImagePullBackOffWatcher(
-		logger.Named("imagePullBackoffWatcher"),
+	podWatcher := scheduler.NewPodWatcher(
+		logger.Named("podWatcher"),
 		k8sClient,
 		cfg,
 	)
-	if err := imagePullBackOffWatcher.RegisterInformer(ctx, informerFactory); err != nil {
-		logger.Fatal("failed to register imagePullBackoffWatcher informer", zap.Error(err))
+	if err := podWatcher.RegisterInformer(ctx, informerFactory); err != nil {
+		logger.Fatal("failed to register podWatcher informer", zap.Error(err))
 	}
 
 	select {