Skip to content

Commit e35e422

Browse files
committed
feat: add program to cleanup dangling droplets (#115)
1 parent dd6bc15 commit e35e422

File tree

7 files changed

+269
-6
lines changed

7 files changed

+269
-6
lines changed

Makefile

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
WORKER_BIN=./build/worker
22
SERVER_BIN=./build/server
3+
CLEANUP_BIN=./build/cleanup
34
GO_FILES=$(shell find . -name '*.go' -type f -not -path "./vendor/*")
45
GO_DEPS=go.mod go.sum
56

@@ -57,9 +58,13 @@ ${SERVER_BIN}: ${GO_FILES} ${GO_DEPS} ## Build the server binary
5758
@mkdir -p ./build
5859
cd server && go build -o ../build/server ./cmd
5960

60-
build: ${WORKER_BIN} ${SERVER_BIN} ## Build the worker and server binaries
61+
${CLEANUP_BIN}: ${GO_FILES} ${GO_DEPS}
62+
@echo "Building cleanup binary..."
63+
@mkdir -p ./build
64+
go build -o ./build/cleanup ./cmd/cleanup
6165

62-
.PHONY: tidy deps build
66+
.PHONY: build
67+
build: ${WORKER_BIN} ${SERVER_BIN} ${CLEANUP_BIN}
6368

6469
###############################################################################
6570
### Proto ###
@@ -157,6 +162,10 @@ start-frontend: ## Start the frontend
157162
start-backend: ## Start the backend
158163
go run ./server/cmd/main.go
159164

165+
.PHONY: start-cleanup
166+
start-cleanup:
167+
go run ./cmd/cleanup/main.go --dry-run
168+
160169
local-docker: ## Start IronBird for local Docker workflows (no cloud dependencies)
161170
@echo "🚀 Starting IronBird in Local Docker Mode"
162171
@echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

cleanup.Dockerfile

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
FROM golang:1.25-bookworm AS builder
2+
WORKDIR /app
3+
4+
RUN mkdir -p /root/.cache/go-build
5+
RUN go env -w GOMODCACHE=/root/.cache/go-build
6+
7+
COPY go.mod go.sum ./
8+
RUN --mount=type=cache,target=/root/.cache/go-build go mod download
9+
10+
COPY . .
11+
12+
RUN go build -o ./build/cleanup ./cmd/cleanup
13+
14+
FROM alpine:latest
15+
16+
RUN apk add --no-cache ca-certificates libc6-compat gcompat
17+
18+
COPY --from=builder /app/build/cleanup /app/cleanup
19+
20+
ENTRYPOINT ["/app/cleanup"]

cmd/cleanup/main.go

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"flag"
6+
"fmt"
7+
"os"
8+
"strings"
9+
"time"
10+
11+
"github.com/digitalocean/godo"
12+
"github.com/skip-mev/ironbird/petri/core/provider/digitalocean"
13+
"go.uber.org/zap"
14+
)
15+
16+
var (
17+
token = flag.String("token", "", "DigitalOcean API token")
18+
dryRun = flag.Bool("dry-run", false, "Perform a dry run without actually deleting droplets")
19+
namePrefix = flag.String("prefix", "petri", "Name prefix to filter droplets")
20+
longRunning = flag.String("long-running-tag", "LONG_RUNNING", "Tag name that indicates a droplet should not be deleted")
21+
)
22+
23+
func main() {
24+
logger, _ := zap.NewProduction()
25+
defer func() { _ = logger.Sync() }()
26+
27+
flag.Parse()
28+
29+
if *token == "" {
30+
*token = os.Getenv("DIGITALOCEAN_TOKEN")
31+
if *token == "" {
32+
logger.Fatal("DigitalOcean token is required. Set via --token flag or DIGITALOCEAN_TOKEN environment variable")
33+
}
34+
}
35+
36+
ctx := context.Background()
37+
38+
doClient := digitalocean.NewGodoClient(*token)
39+
40+
logger.Info("Starting droplet cleanup",
41+
zap.String("prefix", *namePrefix),
42+
zap.String("long_running_tag", *longRunning),
43+
zap.Bool("dry_run", *dryRun))
44+
45+
if err := cleanupDroplets(ctx, doClient, logger); err != nil {
46+
logger.Fatal("Failed to cleanup droplets", zap.Error(err))
47+
}
48+
49+
logger.Info("Droplet cleanup completed successfully")
50+
}
51+
52+
func cleanupDroplets(ctx context.Context, client digitalocean.DoClient, logger *zap.Logger) error {
53+
opts := &godo.ListOptions{
54+
Page: 1,
55+
PerPage: 200,
56+
}
57+
58+
var allDroplets []godo.Droplet
59+
for {
60+
droplets, err := client.ListDroplets(ctx, opts)
61+
if err != nil {
62+
return fmt.Errorf("failed to list droplets: %w", err)
63+
}
64+
65+
allDroplets = append(allDroplets, droplets...)
66+
67+
if len(droplets) < opts.PerPage {
68+
break
69+
}
70+
71+
opts.Page++
72+
}
73+
74+
logger.Info("Retrieved droplets", zap.Int("total_count", len(allDroplets)))
75+
76+
var dropletsToDelete []godo.Droplet
77+
now := time.Now()
78+
for _, droplet := range allDroplets {
79+
if !strings.HasPrefix(droplet.Name, *namePrefix) {
80+
continue
81+
}
82+
83+
hasLongRunningTag := false
84+
for _, tag := range droplet.Tags {
85+
if tag == *longRunning {
86+
hasLongRunningTag = true
87+
break
88+
}
89+
}
90+
91+
if hasLongRunningTag {
92+
logger.Info("Skipping droplet with long-running tag",
93+
zap.String("name", droplet.Name),
94+
zap.Int("id", droplet.ID))
95+
continue
96+
}
97+
98+
// Parse the creation time and skip if created in the last 30 minutes
99+
createdAt, err := time.Parse(time.RFC3339, droplet.Created)
100+
if err != nil {
101+
logger.Warn("Failed to parse droplet creation time, skipping",
102+
zap.String("name", droplet.Name),
103+
zap.Int("id", droplet.ID),
104+
zap.String("created_at", droplet.Created),
105+
zap.Error(err))
106+
continue
107+
}
108+
109+
if now.Sub(createdAt) < 30*time.Minute {
110+
logger.Info("Skipping recently created droplet",
111+
zap.String("name", droplet.Name),
112+
zap.Int("id", droplet.ID),
113+
zap.String("created_at", droplet.Created),
114+
zap.Duration("age", now.Sub(createdAt)))
115+
continue
116+
}
117+
118+
dropletsToDelete = append(dropletsToDelete, droplet)
119+
}
120+
121+
logger.Info("Found droplets to delete", zap.Int("count", len(dropletsToDelete)))
122+
123+
if *dryRun {
124+
logger.Info("Dry run mode - would delete the following droplets:")
125+
for _, droplet := range dropletsToDelete {
126+
logger.Info("Would delete droplet",
127+
zap.String("name", droplet.Name),
128+
zap.Int("id", droplet.ID),
129+
zap.Strings("tags", droplet.Tags),
130+
zap.String("created_at", droplet.Created))
131+
}
132+
return nil
133+
}
134+
135+
var deletedCount int
136+
for _, droplet := range dropletsToDelete {
137+
logger.Info("Deleting droplet",
138+
zap.String("name", droplet.Name),
139+
zap.Int("id", droplet.ID))
140+
141+
if err := client.DeleteDropletByID(ctx, droplet.ID); err != nil {
142+
logger.Error("Failed to delete droplet",
143+
zap.String("name", droplet.Name),
144+
zap.Int("id", droplet.ID),
145+
zap.Error(err))
146+
continue
147+
}
148+
149+
deletedCount++
150+
logger.Info("Successfully deleted droplet",
151+
zap.String("name", droplet.Name),
152+
zap.Int("id", droplet.ID))
153+
154+
time.Sleep(100 * time.Millisecond)
155+
}
156+
157+
logger.Info("Cleanup completed",
158+
zap.Int("deleted_count", deletedCount),
159+
zap.Int("total_candidates", len(dropletsToDelete)))
160+
161+
return nil
162+
}

petri/core/provider/digitalocean/client.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ type DoClient interface {
1313
// Droplet operations
1414
CreateDroplet(ctx context.Context, req *godo.DropletCreateRequest) (*godo.Droplet, error)
1515
GetDroplet(ctx context.Context, dropletID int) (*godo.Droplet, error)
16+
ListDroplets(ctx context.Context, opts *godo.ListOptions) ([]godo.Droplet, error)
1617
DeleteDropletByTag(ctx context.Context, tag string) error
1718
DeleteDropletByID(ctx context.Context, id int) error
1819

@@ -87,6 +88,14 @@ func (c *godoClient) GetDroplet(ctx context.Context, dropletID int) (*godo.Dropl
8788
return droplet, nil
8889
}
8990

91+
func (c *godoClient) ListDroplets(ctx context.Context, opts *godo.ListOptions) ([]godo.Droplet, error) {
92+
droplets, res, err := c.Droplets.List(ctx, opts)
93+
if err := checkResponse(res, err); err != nil {
94+
return nil, err
95+
}
96+
return droplets, nil
97+
}
98+
9099
func (c *godoClient) DeleteDropletByTag(ctx context.Context, tag string) error {
91100
res, err := c.Droplets.DeleteByTag(ctx, tag)
92101
return checkResponse(res, err)

petri/core/provider/digitalocean/droplet.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ func (p *Provider) CreateDroplet(ctx context.Context, definition provider.TaskDe
5252
}
5353

5454
state := p.GetState()
55+
56+
tags := []string{state.PetriTag}
57+
5558
req := &godo.DropletCreateRequest{
5659
Name: fmt.Sprintf("%s-%s", state.PetriTag, definition.Name),
5760
Region: doConfig["region"],
@@ -60,7 +63,7 @@ func (p *Provider) CreateDroplet(ctx context.Context, definition provider.TaskDe
6063
Image: godo.DropletCreateImage{
6164
ID: int(imageId),
6265
},
63-
Tags: []string{state.PetriTag},
66+
Tags: tags,
6467
UserData: formatUserData(userDataCommands),
6568
}
6669

petri/core/provider/digitalocean/mocks/mock_do_client.go

Lines changed: 57 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

types/config_test.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,9 @@ telemetry:
3939
url: http://loki:3100
4040
builder:
4141
build_kit_address: tcp://buildkit:1234
42-
registry:
42+
local:
43+
image_name: test-local-image
44+
ecr:
4345
url: test.registry.com
4446
image_name: test/image
4547
auth_env_configs:
@@ -91,8 +93,9 @@ server_address: localhost:9006
9193
assert.Equal(t, "http://loki:3100", config.Telemetry.Loki.URL)
9294

9395
assert.Equal(t, "tcp://buildkit:1234", config.Builder.BuildKitAddress)
94-
assert.Equal(t, "test.registry.com", config.Builder.Registry.URL)
95-
assert.Equal(t, "test/image", config.Builder.Registry.ImageName)
96+
assert.Equal(t, "test-local-image", config.Builder.Local.ImageName)
97+
assert.Equal(t, "test.registry.com", config.Builder.ECR.URL)
98+
assert.Equal(t, "test/image", config.Builder.ECR.ImageName)
9699
assert.Equal(t, "test_value", config.Builder.AuthEnvConfigs["TEST_ENV"])
97100

98101
assert.Contains(t, config.Chains, "test-chain")

0 commit comments

Comments
 (0)