Skip to content

Commit fa7e989

Browse files
kemingykemingy
authored
refactor: use template for buildkit config file (#1658)
* refactor: use template for buildkit config file Signed-off-by: Keming <kemingyang@tensorchord.ai> * format Signed-off-by: Keming <kemingyang@tensorchord.ai> * fix test Signed-off-by: Keming <kemingyang@tensorchord.ai> --------- Signed-off-by: Keming <kemingyang@tensorchord.ai>
1 parent 45dffe7 commit fa7e989

9 files changed

Lines changed: 178 additions & 91 deletions

File tree

‎pkg/app/bootstrap.go‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import (
3030
"github.com/tensorchord/envd/pkg/home"
3131
sshconfig "github.com/tensorchord/envd/pkg/ssh/config"
3232
"github.com/tensorchord/envd/pkg/types"
33+
"github.com/tensorchord/envd/pkg/util/buildkitutil"
3334
"github.com/tensorchord/envd/pkg/util/fileutil"
3435
)
3536

@@ -289,24 +290,22 @@ func buildkit(clicontext *cli.Context) error {
289290

290291
logrus.Debug("bootstrap the buildkitd container")
291292
var bkClient buildkitd.Client
292-
mirror := clicontext.String("dockerhub-mirror")
293-
setRegistryCA := clicontext.IsSet("registry-ca-keypair")
294-
useHTTP := clicontext.Bool("use-http")
295-
registry := clicontext.String("registry")
296-
297-
if setRegistryCA && useHTTP {
298-
return errors.New("cannot use both registry CA and HTTP")
293+
config := buildkitutil.BuildkitConfig{
294+
Registry: clicontext.String("registry"),
295+
Mirror: clicontext.String("dockerhub-mirror"),
296+
UseHTTP: clicontext.Bool("use-http"),
297+
SetCA: clicontext.IsSet("registry-ca-keypair"),
299298
}
300299

301300
if c.Builder == types.BuilderTypeMoby {
302301
bkClient, err = buildkitd.NewMobyClient(clicontext.Context,
303-
c.Builder, c.BuilderAddress, mirror, registry, setRegistryCA, useHTTP)
302+
c.Builder, c.BuilderAddress, &config)
304303
if err != nil {
305304
return errors.Wrap(err, "failed to create moby buildkit client")
306305
}
307306
} else {
308307
bkClient, err = buildkitd.NewClient(clicontext.Context,
309-
c.Builder, c.BuilderAddress, mirror, registry, setRegistryCA, useHTTP)
308+
c.Builder, c.BuilderAddress, &config)
310309
if err != nil {
311310
return errors.Wrap(err, "failed to create buildkit client")
312311
}

‎pkg/app/prune.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,13 @@ func prune(clicontext *cli.Context) error {
8181
var bkClient buildkitd.Client
8282
if c.Builder == types.BuilderTypeMoby {
8383
bkClient, err = buildkitd.NewMobyClient(clicontext.Context,
84-
c.Builder, c.BuilderAddress, "", "", false, false)
84+
c.Builder, c.BuilderAddress, nil)
8585
if err != nil {
8686
return errors.Wrap(err, "failed to create moby buildkit client")
8787
}
8888
} else {
8989
bkClient, err = buildkitd.NewClient(clicontext.Context,
90-
c.Builder, c.BuilderAddress, "", "", false, false)
90+
c.Builder, c.BuilderAddress, nil)
9191
if err != nil {
9292
return errors.Wrap(err, "failed to create buildkit client")
9393
}

‎pkg/builder/build.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939
"github.com/tensorchord/envd/pkg/lang/version"
4040
"github.com/tensorchord/envd/pkg/progress/progresswriter"
4141
"github.com/tensorchord/envd/pkg/types"
42+
"github.com/tensorchord/envd/pkg/util/buildkitutil"
4243
)
4344

4445
func New(ctx context.Context, opt Options) (Builder, error) {
@@ -88,15 +89,16 @@ func New(ctx context.Context, opt Options) (Builder, error) {
8889
}
8990

9091
var cli buildkitd.Client
92+
bc := buildkitutil.BuildkitConfig{}
9193
if c.Builder == types.BuilderTypeMoby {
9294
cli, err = buildkitd.NewMobyClient(ctx,
93-
c.Builder, c.BuilderAddress, "", "", false, false)
95+
c.Builder, c.BuilderAddress, &bc)
9496
if err != nil {
9597
return nil, errors.Wrap(err, "failed to create moby buildkit client")
9698
}
9799
} else {
98100
cli, err = buildkitd.NewClient(ctx,
99-
c.Builder, c.BuilderAddress, "", "", false, false)
101+
c.Builder, c.BuilderAddress, &bc)
100102
if err != nil {
101103
return nil, errors.Wrap(err, "failed to create buildkit client")
102104
}

‎pkg/buildkitd/buildkitd.go‎

Lines changed: 17 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737
"github.com/tensorchord/envd/pkg/driver/nerdctl"
3838
"github.com/tensorchord/envd/pkg/flag"
3939
"github.com/tensorchord/envd/pkg/types"
40+
"github.com/tensorchord/envd/pkg/util/buildkitutil"
4041
"github.com/tensorchord/envd/pkg/util/envutil"
4142
)
4243

@@ -62,12 +63,9 @@ type Client interface {
6263
}
6364

6465
type generalClient struct {
65-
containerName string
66-
image string
67-
mirror string
68-
registry string
69-
enableRegistryCA bool
70-
useHTTP bool
66+
containerName string
67+
image string
68+
buildkitConfig *buildkitutil.BuildkitConfig
7169

7270
driver types.BuilderType
7371
socket string
@@ -77,17 +75,14 @@ type generalClient struct {
7775
}
7876

7977
func NewMobyClient(ctx context.Context, driver types.BuilderType,
80-
socket, mirror, registry string, enableRegistryCA bool, useHTTP bool) (Client, error) {
78+
socket string, config *buildkitutil.BuildkitConfig) (Client, error) {
8179
logrus.Debug("getting moby buildkit client")
8280
c := &generalClient{
83-
containerName: socket,
84-
image: viper.GetString(flag.FlagBuildkitdImage),
85-
registry: registry,
86-
mirror: mirror,
87-
enableRegistryCA: enableRegistryCA,
88-
useHTTP: useHTTP,
89-
socket: socket,
90-
driver: driver,
81+
containerName: socket,
82+
image: viper.GetString(flag.FlagBuildkitdImage),
83+
buildkitConfig: config,
84+
socket: socket,
85+
driver: driver,
9186
}
9287
c.logger = logrus.WithFields(logrus.Fields{
9388
"container": c.containerName,
@@ -115,16 +110,13 @@ func NewMobyClient(ctx context.Context, driver types.BuilderType,
115110
}
116111

117112
func NewClient(ctx context.Context, driver types.BuilderType,
118-
socket, mirror, registry string, enableRegistryCA bool, useHTTP bool) (Client, error) {
113+
socket string, config *buildkitutil.BuildkitConfig) (Client, error) {
119114
c := &generalClient{
120-
containerName: socket,
121-
image: viper.GetString(flag.FlagBuildkitdImage),
122-
mirror: mirror,
123-
registry: registry,
124-
enableRegistryCA: enableRegistryCA,
125-
useHTTP: useHTTP,
126-
socket: socket,
127-
driver: driver,
115+
containerName: socket,
116+
image: viper.GetString(flag.FlagBuildkitdImage),
117+
buildkitConfig: config,
118+
socket: socket,
119+
driver: driver,
128120
}
129121
c.logger = logrus.WithFields(logrus.Fields{
130122
"container": c.containerName,
@@ -179,8 +171,7 @@ func (c *generalClient) maybeStart(ctx context.Context,
179171
}
180172

181173
if client != nil {
182-
if _, err := client.StartBuildkitd(ctx, c.image, c.containerName, c.mirror, c.registry,
183-
c.enableRegistryCA, c.useHTTP, runningTimeout); err != nil {
174+
if _, err := client.StartBuildkitd(ctx, c.image, c.containerName, c.buildkitConfig, runningTimeout); err != nil {
184175
return "", err
185176
}
186177
}

‎pkg/driver/client.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,14 @@ import (
2020
"time"
2121

2222
"github.com/docker/docker/api/types"
23+
24+
"github.com/tensorchord/envd/pkg/util/buildkitutil"
2325
)
2426

2527
type Client interface {
2628
// Load loads the image from the reader to the docker host.
2729
Load(ctx context.Context, r io.ReadCloser, quiet bool) error
28-
StartBuildkitd(ctx context.Context, tag, name, mirror, registry string, enableRegistryCA, useHTTP bool, timeout time.Duration) (string, error)
30+
StartBuildkitd(ctx context.Context, tag, name string, config *buildkitutil.BuildkitConfig, timeout time.Duration) (string, error)
2931

3032
Exec(ctx context.Context, cname string, cmd []string) error
3133

‎pkg/driver/docker/docker.go‎

Lines changed: 16 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -36,30 +36,11 @@ import (
3636
"github.com/sirupsen/logrus"
3737

3838
"github.com/tensorchord/envd/pkg/driver"
39+
"github.com/tensorchord/envd/pkg/util/buildkitutil"
3940
"github.com/tensorchord/envd/pkg/util/fileutil"
4041
)
4142

42-
const buildkitdMirror = `
43-
[registry."docker.io"]
44-
mirrors = ["%s"]
45-
`
46-
const buildkitdHTTP = `
47-
[registry."docker.io"]
48-
http = true
49-
`
50-
const buildkitdRegistry = `
51-
[registry."%s"]
52-
http = %v
53-
`
5443
const buildkitdCertPath = "/etc/registry"
55-
const buildkitdWithCA = `
56-
[registry."docker.io"]
57-
mirrors = ["%s"]
58-
ca=["/etc/registry/ca.pem"]
59-
[[registry."docker.io".keypair]]
60-
key="/etc/registry/key.pem"
61-
cert="/etc/registry/cert.pem"
62-
`
6344

6445
var (
6546
anchoredIdentifierRegexp = regexp.MustCompile(`^([a-f0-9]{64})$`)
@@ -193,12 +174,11 @@ func (c dockerClient) ResumeContainer(ctx context.Context, name string) (string,
193174
return name, nil
194175
}
195176

196-
func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name, mirror, registry string,
197-
enableRegistryCA, useHTTP bool, timeout time.Duration) (string, error) {
177+
func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name string, bc *buildkitutil.BuildkitConfig, timeout time.Duration) (string, error) {
198178
logger := logrus.WithFields(logrus.Fields{
199-
"tag": tag,
200-
"container": name,
201-
"mirror": mirror,
179+
"tag": tag,
180+
"container": name,
181+
"buildkit-config": bc,
202182
})
203183
logger.Debug("starting buildkitd")
204184
if _, _, err := c.ImageInspectWithRaw(ctx, tag); err != nil {
@@ -226,22 +206,17 @@ func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name, mirror, reg
226206
Privileged: true,
227207
AutoRemove: true,
228208
}
229-
var cfg string
230-
if mirror != "" {
231-
if enableRegistryCA {
232-
cfg = fmt.Sprintf(buildkitdWithCA, mirror)
233-
hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{
234-
Type: mount.TypeBind,
235-
Source: fileutil.DefaultConfigDir,
236-
Target: buildkitdCertPath,
237-
})
238-
} else {
239-
cfg = fmt.Sprintf(buildkitdMirror, mirror)
240-
}
241-
} else if registry != "" {
242-
cfg = fmt.Sprintf(buildkitdRegistry, registry, useHTTP)
243-
} else if useHTTP {
244-
cfg = buildkitdHTTP
209+
210+
if bc.SetCA {
211+
hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{
212+
Type: mount.TypeBind,
213+
Source: fileutil.DefaultConfigDir,
214+
Target: buildkitdCertPath,
215+
})
216+
}
217+
cfg, err := bc.String()
218+
if err != nil {
219+
return "", errors.Wrap(err, "failed to generate buildkit config")
245220
}
246221
config.Entrypoint = []string{
247222
"/bin/sh",

‎pkg/driver/nerdctl/nerdctl.go‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"github.com/sirupsen/logrus"
2929

3030
"github.com/tensorchord/envd/pkg/driver"
31+
"github.com/tensorchord/envd/pkg/util/buildkitutil"
3132
)
3233

3334
type nerdctlClient struct {
@@ -63,13 +64,12 @@ func (nc *nerdctlClient) Load(ctx context.Context, r io.ReadCloser, quiet bool)
6364
return nil
6465
}
6566

66-
func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name, mirror, registry string,
67-
enableRegistryCA, useHTTP bool, timeout time.Duration) (string, error) {
67+
func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name string, bc *buildkitutil.BuildkitConfig, timeout time.Duration) (string, error) {
6868
logger := logrus.WithFields(logrus.Fields{
69-
"tag": tag,
70-
"container": name,
71-
"mirror": mirror,
72-
"driver": "nerdctl",
69+
"tag": tag,
70+
"container": name,
71+
"buildkit-config": bc,
72+
"driver": "nerdctl",
7373
})
7474
logger.Debug("starting buildkitd")
7575

@@ -83,10 +83,11 @@ func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name, mirror,
8383
if !existed {
8484
buildkitdCmd := "buildkitd"
8585
// TODO: support mirror CA keypair
86-
if mirror != "" {
87-
cfg := fmt.Sprintf(`
88-
[registry."docker.io"]
89-
mirrors = ["%s"]`, mirror)
86+
if bc.Registry != "" || bc.Mirror != "" || bc.UseHTTP {
87+
cfg, err := bc.String()
88+
if err != nil {
89+
return "", errors.Wrap(err, "failed to generate buildkit config")
90+
}
9091
buildkitdCmd = fmt.Sprintf("mkdir /etc/buildkit && echo '%s' > /etc/buildkit/buildkitd.toml && buildkitd", cfg)
9192
logger.Debugf("setting buildkit config: %s", cfg)
9293
}

‎pkg/util/buildkitutil/buildkit.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
// Copyright 2023 The envd Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package buildkitutil
16+
17+
import (
18+
"strings"
19+
"text/template"
20+
)
21+
22+
const buildkitConfigTemplate = `
23+
[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}"]{{ if .Mirror }}
24+
mirrors = ["{{ .Mirror }}"]{{ end }}
25+
http = {{ .UseHTTP }}
26+
{{ if .SetCA}}ca=["/etc/registry/ca.pem"]
27+
[[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}".keypair]]
28+
key="/etc/registry/key.pem"
29+
cert="/etc/registry/cert.pem"
30+
{{ end }}
31+
`
32+
33+
type BuildkitConfig struct {
34+
Registry string
35+
Mirror string
36+
UseHTTP bool
37+
SetCA bool
38+
}
39+
40+
func (c *BuildkitConfig) String() (string, error) {
41+
tmpl, err := template.New("buildkitConfig").Parse(buildkitConfigTemplate)
42+
if err != nil {
43+
return "", err
44+
}
45+
var config strings.Builder
46+
err = tmpl.Execute(&config, c)
47+
return config.String(), err
48+
}

0 commit comments

Comments
 (0)