Skip to content

Commit 0d2e270

Browse files
wolfeidauwolfeidau
authored
Merge pull request #3551 from buildkite/mdc-723-add-the-restore-and-save-commands-to-the-buildkite-agent
Add cache save and restore using github.com/buildkite/zstash
2 parents fbbbc08 + 04c0aa1 commit 0d2e270

9 files changed

Lines changed: 993 additions & 26 deletions

File tree

‎clicommand/cache_restore.go‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
package clicommand
2+
3+
import (
4+
"context"
5+
"slices"
6+
7+
"github.com/buildkite/agent/v3/internal/cache"
8+
"github.com/urfave/cli"
9+
)
10+
11+
const cacheRestoreHelpDescription = `Usage:
12+
13+
buildkite-agent cache restore [options]
14+
15+
Description:
16+
17+
Restores files from the cache for the current job based on the cache configuration
18+
defined in your cache config file (defaults to .buildkite/cache.yml).
19+
20+
The cache configuration file defines which files or directories should be restored
21+
and their associated cache keys. Caches are scoped by organization, pipeline, and
22+
branch. If an exact cache match is not found, the command will attempt to use
23+
fallback keys if defined in your cache configuration.
24+
25+
Note: This feature is currently in development and subject to change. It is not
26+
yet available to all customers.
27+
28+
Example:
29+
30+
$ buildkite-agent cache restore
31+
32+
This will restore all caches defined in .buildkite/cache.yml. You can also restore
33+
specific caches by providing their IDs:
34+
35+
$ buildkite-agent cache restore --ids "node"
36+
37+
The cache will be retrieved from the bucket specified by --bucket-url or your
38+
cache configuration.
39+
40+
Configuration File Format:
41+
42+
The cache configuration file should be in YAML format:
43+
44+
dependencies:
45+
- id: node
46+
key: '{{ id }}-{{ agent.os }}-{{ agent.arch }}-{{ checksum "package-lock.json" }}'
47+
fallback_keys:
48+
- '{{ id }}-{{ agent.os }}-{{ agent.arch }}-'
49+
paths:
50+
- node_modules
51+
52+
Cache Restoration Results:
53+
54+
The command will report one of three outcomes for each cache:
55+
- Cache hit: Exact key match found and restored
56+
- Fallback used: No exact match, but a fallback key was found and restored
57+
- Cache miss: No matching cache found
58+
59+
The command automatically uses the following environment variables when available:
60+
- BUILDKITE_BRANCH (for branch scoping)
61+
- BUILDKITE_PIPELINE_SLUG (for pipeline scoping)
62+
- BUILDKITE_ORGANIZATION_SLUG (for organization scoping)`
63+
64+
type CacheRestoreConfig struct {
65+
GlobalConfig
66+
APIConfig
67+
CacheConfig
68+
}
69+
70+
var CacheRestoreCommand = cli.Command{
71+
Name: "restore",
72+
Usage: "Restores files from the cache",
73+
Description: cacheRestoreHelpDescription,
74+
Flags: slices.Concat(globalFlags(), apiFlags(), cacheFlags()),
75+
Action: func(c *cli.Context) error {
76+
ctx := context.Background()
77+
ctx, cfg, l, _, done := setupLoggerAndConfig[CacheRestoreConfig](ctx, c)
78+
defer done()
79+
80+
l.Info("Cache restore command executed")
81+
82+
apiCfg := loadAPIClientConfig(cfg, "AgentAccessToken")
83+
84+
// Build cache configuration
85+
cacheCfg := cache.Config{
86+
BucketURL: cfg.BucketURL,
87+
Branch: cfg.Branch,
88+
Pipeline: cfg.Pipeline,
89+
Organization: cfg.Organization,
90+
CacheConfigFile: cfg.CacheConfigFile,
91+
Ids: cfg.Ids,
92+
APIEndpoint: apiCfg.Endpoint,
93+
APIToken: apiCfg.Token,
94+
}
95+
96+
// Perform cache restore (logging happens inside)
97+
return cache.Restore(ctx, l, cacheCfg)
98+
},
99+
}

‎clicommand/cache_save.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
package clicommand
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"slices"
7+
8+
"github.com/buildkite/agent/v3/internal/cache"
9+
"github.com/urfave/cli"
10+
)
11+
12+
const cacheSaveHelpDescription = `Usage:
13+
14+
buildkite-agent cache save [options]
15+
16+
Description:
17+
18+
Saves files to the cache for the current build based on the cache configuration
19+
defined in your cache config file (defaults to .buildkite/cache.yml).
20+
21+
The cache configuration file defines which files or directories should be cached
22+
and their associated cache keys. Caches are scoped by organization, pipeline, and
23+
branch.
24+
25+
Note: This feature is currently in development and subject to change. It is not
26+
yet available to all customers.
27+
28+
Example:
29+
30+
$ buildkite-agent cache save
31+
32+
This will save all caches defined in .buildkite/cache.yml. You can also save
33+
specific caches by providing their IDs:
34+
35+
$ buildkite-agent cache save --ids "node"
36+
37+
The cache will be stored in the bucket specified by --bucket-url or your
38+
cache configuration. If a cache with the same key already exists, it will
39+
not be overwritten.
40+
41+
Configuration File Format:
42+
43+
The cache configuration file should be in YAML format:
44+
45+
dependencies:
46+
- id: node
47+
key: '{{ id }}-{{ agent.os }}-{{ agent.arch }}-{{ checksum "package-lock.json" }}'
48+
fallback_keys:
49+
- '{{ id }}-{{ agent.os }}-{{ agent.arch }}-'
50+
paths:
51+
- node_modules
52+
53+
The command automatically uses the following environment variables when available:
54+
- BUILDKITE_BRANCH (for branch scoping)
55+
- BUILDKITE_PIPELINE_SLUG (for pipeline scoping)
56+
- BUILDKITE_ORGANIZATION_SLUG (for organization scoping)`
57+
58+
type CacheSaveConfig struct {
59+
GlobalConfig
60+
APIConfig
61+
CacheConfig
62+
}
63+
64+
var CacheSaveCommand = cli.Command{
65+
Name: "save",
66+
Usage: "Saves files to the cache",
67+
Description: cacheSaveHelpDescription,
68+
Flags: slices.Concat(globalFlags(), apiFlags(), cacheFlags()),
69+
Action: func(c *cli.Context) error {
70+
ctx := context.Background()
71+
ctx, cfg, l, _, done := setupLoggerAndConfig[CacheSaveConfig](ctx, c)
72+
defer done()
73+
74+
l.Info("Cache save command executed")
75+
76+
apiCfg := loadAPIClientConfig(cfg, "AgentAccessToken")
77+
78+
if apiCfg.Token == "" {
79+
return fmt.Errorf("an API token must be provided to save caches")
80+
}
81+
82+
// Build cache configuration
83+
cacheCfg := cache.Config{
84+
BucketURL: cfg.BucketURL,
85+
Branch: cfg.Branch,
86+
Pipeline: cfg.Pipeline,
87+
Organization: cfg.Organization,
88+
CacheConfigFile: cfg.CacheConfigFile,
89+
Ids: cfg.Ids,
90+
APIEndpoint: apiCfg.Endpoint,
91+
APIToken: apiCfg.Token,
92+
}
93+
94+
// Perform cache save (logging happens inside)
95+
return cache.Save(ctx, l, cacheCfg)
96+
},
97+
}

‎clicommand/cache_shared.go‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package clicommand
2+
3+
import "github.com/urfave/cli"
4+
5+
// CacheConfig includes cache-related shared options for easy inclusion across
6+
// cache command config structs (via embedding).
7+
type CacheConfig struct {
8+
Ids []string `cli:"ids"`
9+
Registry string `cli:"registry"`
10+
BucketURL string `cli:"bucket-url" validate:"required"`
11+
Branch string `cli:"branch" validate:"required"`
12+
Pipeline string `cli:"pipeline" validate:"required"`
13+
Organization string `cli:"organization" validate:"required"`
14+
CacheConfigFile string `cli:"cache-config-file"`
15+
}
16+
17+
func cacheFlags() []cli.Flag {
18+
return []cli.Flag{
19+
cli.StringSliceFlag{
20+
Name: "ids",
21+
Value: &cli.StringSlice{},
22+
Usage: "Comma-separated list of cache IDs (if empty, processes all caches)",
23+
EnvVar: "BUILDKITE_CACHE_IDS",
24+
},
25+
cli.StringFlag{
26+
Name: "registry",
27+
Value: "~",
28+
Usage: "The slug of the cache registry to use, defaults to the default registry (~)",
29+
EnvVar: "BUILDKITE_CACHE_REGISTRY",
30+
},
31+
cli.StringFlag{
32+
Name: "bucket-url",
33+
Value: "",
34+
Usage: "The URL of the bucket (e.g., s3://bucket-name)",
35+
EnvVar: "BUILDKITE_CACHE_BUCKET_URL",
36+
},
37+
cli.StringFlag{
38+
Name: "branch",
39+
Value: "",
40+
Usage: "Which branch should the cache be associated with",
41+
EnvVar: "BUILDKITE_BRANCH",
42+
},
43+
cli.StringFlag{
44+
Name: "pipeline",
45+
Value: "",
46+
Usage: "The pipeline slug for this cache",
47+
EnvVar: "BUILDKITE_PIPELINE_SLUG",
48+
},
49+
cli.StringFlag{
50+
Name: "organization",
51+
Value: "",
52+
Usage: "The organization slug for this cache",
53+
EnvVar: "BUILDKITE_ORGANIZATION_SLUG",
54+
},
55+
cli.StringFlag{
56+
Name: "cache-config-file",
57+
Value: ".buildkite/cache.yml",
58+
Usage: "Path to the cache configuration YAML file",
59+
EnvVar: "BUILDKITE_CACHE_CONFIG_FILE",
60+
},
61+
}
62+
}

‎clicommand/commands.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,16 @@ var BuildkiteAgentCommands = []cli.Command{
4343
BuildCancelCommand,
4444
},
4545
},
46+
{
47+
Name: "cache",
48+
Category: categoryJobCommands,
49+
Usage: "Manage build caches",
50+
Hidden: true, // currently in experimental phase
51+
Subcommands: []cli.Command{
52+
CacheSaveCommand,
53+
CacheRestoreCommand,
54+
},
55+
},
4656
{
4757
Name: "env",
4858
Category: categoryJobCommands,

‎clicommand/config_completeness_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@ var commandConfigPairs = []configCommandPair{
2828
{Config: ArtifactUploadConfig{}, Command: ArtifactUploadCommand},
2929
{Config: BuildCancelConfig{}, Command: BuildCancelCommand},
3030
{Config: BootstrapConfig{}, Command: BootstrapCommand},
31+
{Config: CacheRestoreConfig{}, Command: CacheRestoreCommand},
32+
{Config: CacheSaveConfig{}, Command: CacheSaveCommand},
3133
{Config: EnvDumpConfig{}, Command: EnvDumpCommand},
3234
{Config: EnvGetConfig{}, Command: EnvGetCommand},
3335
{Config: EnvSetConfig{}, Command: EnvSetCommand},

‎go.mod‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,14 +15,15 @@ require (
1515
github.com/aws/aws-sdk-go-v2 v1.39.4
1616
github.com/aws/aws-sdk-go-v2/config v1.31.15
1717
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.11
18-
github.com/aws/aws-sdk-go-v2/service/ec2 v1.258.1
19-
github.com/aws/aws-sdk-go-v2/service/kms v1.46.2
18+
github.com/aws/aws-sdk-go-v2/service/ec2 v1.257.2
19+
github.com/aws/aws-sdk-go-v2/service/kms v1.46.0
2020
github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf
2121
github.com/buildkite/bintest/v3 v3.3.0
2222
github.com/buildkite/go-pipeline v0.16.0
2323
github.com/buildkite/interpolate v0.1.5
2424
github.com/buildkite/roko v1.4.0
2525
github.com/buildkite/shellwords v1.0.1
26+
github.com/buildkite/zstash v0.5.0
2627
github.com/creack/pty v1.1.19
2728
github.com/denisbrodbeck/machineid v1.0.1
2829
github.com/dustin/go-humanize v1.0.1
@@ -90,12 +91,17 @@ require (
9091
github.com/alexflint/go-arg v1.5.1 // indirect
9192
github.com/alexflint/go-scalar v1.2.0 // indirect
9293
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
94+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect
9395
github.com/aws/aws-sdk-go-v2/credentials v1.18.19 // indirect
9496
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.11 // indirect
9597
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.11 // indirect
9698
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
99+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.11 // indirect
97100
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect
101+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.2 // indirect
98102
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.11 // indirect
103+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 // indirect
104+
github.com/aws/aws-sdk-go-v2/service/s3 v1.88.7 // indirect
99105
github.com/aws/aws-sdk-go-v2/service/sso v1.29.8 // indirect
100106
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.3 // indirect
101107
github.com/aws/aws-sdk-go-v2/service/sts v1.38.9 // indirect
@@ -126,11 +132,12 @@ require (
126132
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
127133
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
128134
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
129-
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
135+
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
130136
github.com/hashicorp/go-version v1.7.0 // indirect
131137
github.com/jmespath/go-jmespath v0.4.0 // indirect
132138
github.com/json-iterator/go v1.1.12 // indirect
133139
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
140+
github.com/klauspost/compress v1.18.1 // indirect
134141
github.com/kylelemons/godebug v1.1.0 // indirect
135142
github.com/lestrrat-go/blackmagic v1.0.3 // indirect
136143
github.com/lestrrat-go/httpcc v1.0.1 // indirect
@@ -155,6 +162,7 @@ require (
155162
github.com/qri-io/jsonpointer v0.1.1 // indirect
156163
github.com/rivo/uniseg v0.4.7 // indirect
157164
github.com/russross/blackfriday/v2 v2.1.0 // indirect
165+
github.com/saracen/zipextra v0.0.0-20250129175152-f1aa42d25216 // indirect
158166
github.com/secure-systems-lab/go-securesystemslib v0.9.0 // indirect
159167
github.com/segmentio/asm v1.2.0 // indirect
160168
github.com/shirou/gopsutil/v4 v4.25.8 // indirect
@@ -163,19 +171,20 @@ require (
163171
github.com/tklauser/go-sysconf v0.3.15 // indirect
164172
github.com/tklauser/numcpus v0.10.0 // indirect
165173
github.com/vektah/gqlparser/v2 v2.5.25 // indirect
174+
github.com/wolfeidau/quickzip v1.0.2 // indirect
166175
github.com/yusufpapurcu/wmi v1.2.4 // indirect
167-
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
176+
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
168177
go.opentelemetry.io/collector/component v1.31.0 // indirect
169178
go.opentelemetry.io/collector/featuregate v1.31.0 // indirect
170179
go.opentelemetry.io/collector/internal/telemetry v0.125.0 // indirect
171180
go.opentelemetry.io/collector/pdata v1.31.0 // indirect
172181
go.opentelemetry.io/collector/semconv v0.125.0 // indirect
173182
go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect
174-
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
183+
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect
175184
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.38.0 // indirect
176185
go.opentelemetry.io/otel/log v0.11.0 // indirect
177186
go.opentelemetry.io/otel/metric v1.38.0 // indirect
178-
go.opentelemetry.io/proto/otlp v1.7.1 // indirect
187+
go.opentelemetry.io/proto/otlp v1.8.0 // indirect
179188
go.uber.org/atomic v1.11.0 // indirect
180189
go.uber.org/multierr v1.11.0 // indirect
181190
go.uber.org/zap v1.27.0 // indirect
@@ -184,8 +193,8 @@ require (
184193
golang.org/x/text v0.30.0 // indirect
185194
golang.org/x/time v0.14.0 // indirect
186195
golang.org/x/tools v0.37.0 // indirect
187-
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
188-
google.golang.org/genproto/googleapis/api v0.0.0-20250825161204-c5933d9347a5 // indirect
196+
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
197+
google.golang.org/genproto/googleapis/api v0.0.0-20250929231259-57b25ae835d4 // indirect
189198
google.golang.org/genproto/googleapis/rpc v0.0.0-20251014184007-4626949a642f // indirect
190199
google.golang.org/grpc v1.76.0 // indirect
191200
google.golang.org/protobuf v1.36.10 // indirect

0 commit comments

Comments
 (0)