feat: add program to cleanup dangling droplets

Makefile

@@ -1,5 +1,6 @@
 WORKER_BIN=./build/worker
 SERVER_BIN=./build/server
+CLEANUP_BIN=./build/cleanup
 GO_FILES=$(shell find . -name '*.go' -type f -not -path "./vendor/*")
 GO_DEPS=go.mod go.sum
 
@@ -57,9 +58,13 @@ ${SERVER_BIN}: ${GO_FILES} ${GO_DEPS} ## Build the server binary
 	@mkdir -p ./build
 	cd server && go build -o ../build/server ./cmd
 
-build: ${WORKER_BIN} ${SERVER_BIN} ## Build the worker and server binaries
+${CLEANUP_BIN}: ${GO_FILES} ${GO_DEPS}
+	@echo "Building cleanup binary..."
+	@mkdir -p ./build
+	go build -o ./build/cleanup ./cmd/cleanup
 
-.PHONY: tidy deps build
+.PHONY: build
+build: ${WORKER_BIN} ${SERVER_BIN} ${CLEANUP_BIN}
 
 ###############################################################################
 ###                                  Proto                                  ###
@@ -157,6 +162,10 @@ start-frontend: ## Start the frontend
 start-backend: ## Start the backend
 	go run ./server/cmd/main.go
 
+.PHONY: start-cleanup
+start-cleanup:
+	go run ./cmd/cleanup/main.go --dry-run
+
 local-docker: ## Start IronBird for local Docker workflows (no cloud dependencies)
 	@echo "🚀 Starting IronBird in Local Docker Mode"
 	@echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

cleanup.Dockerfile

@@ -0,0 +1,20 @@
+FROM golang:1.25-bookworm AS builder
+WORKDIR /app
+
+RUN mkdir -p /root/.cache/go-build
+RUN go env -w GOMODCACHE=/root/.cache/go-build
+
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/root/.cache/go-build go mod download
+
+COPY . .
+
+RUN go build -o ./build/cleanup ./cmd/cleanup
+
+FROM alpine:latest
+
+RUN apk add --no-cache ca-certificates libc6-compat gcompat
+
+COPY --from=builder /app/build/cleanup /app/cleanup
+
+ENTRYPOINT ["/app/cleanup"]

cmd/cleanup/main.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/digitalocean/godo"
+	"github.com/skip-mev/ironbird/petri/core/provider/digitalocean"
+	"go.uber.org/zap"
+)
+
+var (
+	token       = flag.String("token", "", "DigitalOcean API token")
+	dryRun      = flag.Bool("dry-run", false, "Perform a dry run without actually deleting droplets")
+	namePrefix  = flag.String("prefix", "petri", "Name prefix to filter droplets")
+	longRunning = flag.String("long-running-tag", "LONG_RUNNING", "Tag name that indicates a droplet should not be deleted")
+)
+
+func main() {
+	logger, _ := zap.NewProduction()
+	defer logger.Sync()
+
+	flag.Parse()
+
+	if *token == "" {
+		*token = os.Getenv("DIGITALOCEAN_TOKEN")
+		if *token == "" {
+			logger.Fatal("DigitalOcean token is required. Set via --token flag or DIGITALOCEAN_TOKEN environment variable")
+		}
+	}
+
+	ctx := context.Background()
+
+	doClient := digitalocean.NewGodoClient(*token)
+
+	logger.Info("Starting droplet cleanup",
+		zap.String("prefix", *namePrefix),
+		zap.String("long_running_tag", *longRunning),
+		zap.Bool("dry_run", *dryRun))
+
+	if err := cleanupDroplets(ctx, doClient, logger); err != nil {
+		logger.Fatal("Failed to cleanup droplets", zap.Error(err))
+	}
+
+	logger.Info("Droplet cleanup completed successfully")
+}
+
+func cleanupDroplets(ctx context.Context, client digitalocean.DoClient, logger *zap.Logger) error {
+	opts := &godo.ListOptions{
+		Page:    1,
+		PerPage: 200,
+	}
+
+	var allDroplets []godo.Droplet
+	for {
+		droplets, err := client.ListDroplets(ctx, opts)
+		if err != nil {
+			return fmt.Errorf("failed to list droplets: %w", err)
+		}
+
+		allDroplets = append(allDroplets, droplets...)
+
+		if len(droplets) < opts.PerPage {
+			break
+		}
+
+		opts.Page++
+	}
+
+	logger.Info("Retrieved droplets", zap.Int("total_count", len(allDroplets)))
+
+	var dropletsToDelete []godo.Droplet
+	now := time.Now()
+	for _, droplet := range allDroplets {
+		if !strings.HasPrefix(droplet.Name, *namePrefix) {
+			continue
+		}
+
+		hasLongRunningTag := false
+		for _, tag := range droplet.Tags {
+			if tag == *longRunning {
+				hasLongRunningTag = true
+				break
+			}
+		}
+
+		if hasLongRunningTag {
+			logger.Info("Skipping droplet with long-running tag",
+				zap.String("name", droplet.Name),
+				zap.Int("id", droplet.ID))
+			continue
+		}
+
+		// Parse the creation time and skip if created in the last 30 minutes
+		createdAt, err := time.Parse(time.RFC3339, droplet.Created)
+		if err != nil {
+			logger.Warn("Failed to parse droplet creation time, skipping",
+				zap.String("name", droplet.Name),
+				zap.Int("id", droplet.ID),
+				zap.String("created_at", droplet.Created),
+				zap.Error(err))
+			continue
+		}
+
+		if now.Sub(createdAt) < 30*time.Minute {
+			logger.Info("Skipping recently created droplet",
+				zap.String("name", droplet.Name),
+				zap.Int("id", droplet.ID),
+				zap.String("created_at", droplet.Created),
+				zap.Duration("age", now.Sub(createdAt)))
+			continue
+		}
+
+		dropletsToDelete = append(dropletsToDelete, droplet)
+	}
+
+	logger.Info("Found droplets to delete", zap.Int("count", len(dropletsToDelete)))
+
+	if *dryRun {
+		logger.Info("Dry run mode - would delete the following droplets:")
+		for _, droplet := range dropletsToDelete {
+			logger.Info("Would delete droplet",
+				zap.String("name", droplet.Name),
+				zap.Int("id", droplet.ID),
+				zap.Strings("tags", droplet.Tags),
+				zap.String("created_at", droplet.Created))
+		}
+		return nil
+	}
+
+	var deletedCount int
+	for _, droplet := range dropletsToDelete {
+		logger.Info("Deleting droplet",
+			zap.String("name", droplet.Name),
+			zap.Int("id", droplet.ID))
+
+		if err := client.DeleteDropletByID(ctx, droplet.ID); err != nil {
+			logger.Error("Failed to delete droplet",
+				zap.String("name", droplet.Name),
+				zap.Int("id", droplet.ID),
+				zap.Error(err))
+			continue
+		}
+
+		deletedCount++
+		logger.Info("Successfully deleted droplet",
+			zap.String("name", droplet.Name),
+			zap.Int("id", droplet.ID))
+
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	logger.Info("Cleanup completed",
+		zap.Int("deleted_count", deletedCount),
+		zap.Int("total_candidates", len(dropletsToDelete)))
+
+	return nil
+}

petri/core/provider/digitalocean/client.go

@@ -13,6 +13,7 @@ type DoClient interface {
 	// Droplet operations
 	CreateDroplet(ctx context.Context, req *godo.DropletCreateRequest) (*godo.Droplet, error)
 	GetDroplet(ctx context.Context, dropletID int) (*godo.Droplet, error)
+	ListDroplets(ctx context.Context, opts *godo.ListOptions) ([]godo.Droplet, error)
 	DeleteDropletByTag(ctx context.Context, tag string) error
 	DeleteDropletByID(ctx context.Context, id int) error
 
@@ -87,6 +88,14 @@ func (c *godoClient) GetDroplet(ctx context.Context, dropletID int) (*godo.Dropl
 	return droplet, nil
 }
 
+func (c *godoClient) ListDroplets(ctx context.Context, opts *godo.ListOptions) ([]godo.Droplet, error) {
+	droplets, res, err := c.Droplets.List(ctx, opts)
+	if err := checkResponse(res, err); err != nil {
+		return nil, err
+	}
+	return droplets, nil
+}
+
 func (c *godoClient) DeleteDropletByTag(ctx context.Context, tag string) error {
 	res, err := c.Droplets.DeleteByTag(ctx, tag)
 	return checkResponse(res, err)

petri/core/provider/digitalocean/droplet.go

@@ -52,6 +52,9 @@ func (p *Provider) CreateDroplet(ctx context.Context, definition provider.TaskDe
 	}
 
 	state := p.GetState()
+
+	tags := []string{state.PetriTag}
+
 	req := &godo.DropletCreateRequest{
 		Name:    fmt.Sprintf("%s-%s", state.PetriTag, definition.Name),
 		Region:  doConfig["region"],
@@ -60,7 +63,7 @@ func (p *Provider) CreateDroplet(ctx context.Context, definition provider.TaskDe
 		Image: godo.DropletCreateImage{
 			ID: int(imageId),
 		},
-		Tags:     []string{state.PetriTag},
+		Tags:     tags,
 		UserData: formatUserData(userDataCommands),
 	}