Skip to content

Commit 6569021

Browse files
authored
feat: build image by platform (#1582)
* feat: build image by platform Signed-off-by: hang lv <xlv20@fudan.edu.cn> * feat: support sequential multi-platform image building Signed-off-by: hang lv <xlv20@fudan.edu.cn> * feat: set goos/goarch as default target platform Signed-off-by: hang lv <xlv20@fudan.edu.cn> * fix: lint Signed-off-by: hang lv <xlv20@fudan.edu.cn> * fix: pull correct base image Signed-off-by: hang lv <xlv20@fudan.edu.cn> --------- Signed-off-by: hang lv <xlv20@fudan.edu.cn>
1 parent 2839aa3 commit 6569021

18 files changed

Lines changed: 135 additions & 33 deletions

File tree

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ require (
180180
go.opentelemetry.io/otel/sdk v1.14.0 // indirect
181181
go.opentelemetry.io/otel/trace v1.14.0 // indirect
182182
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
183-
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
183+
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
184184
golang.org/x/mod v0.10.0 // indirect
185185
golang.org/x/net v0.9.0 // indirect
186186
golang.org/x/oauth2 v0.6.0 // indirect

pkg/app/build.go

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package app
1616

1717
import (
18+
"strings"
1819
"time"
1920

2021
"github.com/sirupsen/logrus"
@@ -23,6 +24,7 @@ import (
2324
buildutil "github.com/tensorchord/envd/pkg/app/build"
2425
"github.com/tensorchord/envd/pkg/app/telemetry"
2526
sshconfig "github.com/tensorchord/envd/pkg/ssh/config"
27+
"github.com/tensorchord/envd/pkg/util/runtimeutil"
2628
)
2729

2830
var CommandBuild = &cli.Command{
@@ -89,6 +91,12 @@ To build and push the image to a registry:
8991
Usage: "Import the cache (e.g. type=registry,ref=<image>)",
9092
Aliases: []string{"ic"},
9193
},
94+
&cli.StringFlag{
95+
Name: "platform",
96+
Usage: `Specify the target platforms for the build output (for example, windows/amd64 or linux/amd64,darwin/arm64).
97+
Build images with same tags could cause image overwriting, platform suffixes will be added to differentiate the images.`,
98+
DefaultText: runtimeutil.GetRuntimePlatform(),
99+
},
92100
},
93101
Action: build,
94102
}
@@ -106,12 +114,24 @@ func build(clicontext *cli.Context) error {
106114
logger := logrus.WithField("builder-options", opt)
107115
logger.Debug("starting build command")
108116

109-
builder, err := buildutil.GetBuilder(clicontext, opt)
110-
if err != nil {
111-
return err
112-
}
113-
if err = buildutil.InterpretEnvdDef(builder); err != nil {
114-
return err
117+
platforms := strings.Split(opt.Platform, ",")
118+
for _, platform := range platforms {
119+
o := opt
120+
o.Platform = platform
121+
if len(platforms) > 1 {
122+
// Transform the platform suffix to comply with the tag naming rule.
123+
o.Tag += "-" + strings.Replace(platform, "/", "-", 1)
124+
}
125+
builder, err := buildutil.GetBuilder(clicontext, o)
126+
if err != nil {
127+
return err
128+
}
129+
if err = buildutil.InterpretEnvdDef(builder); err != nil {
130+
return err
131+
}
132+
if err := buildutil.BuildImage(clicontext, builder); err != nil {
133+
return err
134+
}
115135
}
116-
return buildutil.BuildImage(clicontext, builder)
136+
return nil
117137
}

pkg/app/build/build.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ func ParseBuildOpt(clicontext *cli.Context) (builder.Options, error) {
135135
exportCache := clicontext.String("export-cache")
136136
importCache := clicontext.String("import-cache")
137137
useProxy := clicontext.Bool("use-proxy")
138+
platform := clicontext.String("platform")
138139

139140
opt := builder.Options{
140141
ManifestFilePath: manifest,
@@ -148,6 +149,7 @@ func ParseBuildOpt(clicontext *cli.Context) (builder.Options, error) {
148149
ExportCache: exportCache,
149150
ImportCache: importCache,
150151
UseHTTPProxy: useProxy,
152+
Platform: platform,
151153
}
152154

153155
debug := clicontext.Bool("debug")

pkg/app/up.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/tensorchord/envd/pkg/home"
3030
sshconfig "github.com/tensorchord/envd/pkg/ssh/config"
3131
"github.com/tensorchord/envd/pkg/types"
32+
"github.com/tensorchord/envd/pkg/util/runtimeutil"
3233
)
3334

3435
var CommandUp = &cli.Command{
@@ -141,6 +142,11 @@ var CommandUp = &cli.Command{
141142
Usage: "Import the cache (e.g. type=registry,ref=<image>)",
142143
Aliases: []string{"ic"},
143144
},
145+
&cli.StringFlag{
146+
Name: "platform",
147+
Usage: "Specify the target platform for the build output, (for example, windows/amd64, linux/amd64, or darwin/arm64)",
148+
DefaultText: runtimeutil.GetRuntimePlatform(),
149+
},
144150
},
145151

146152
Action: up,

pkg/builder/build.go

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,15 @@ import (
1919
"io"
2020
"os"
2121
"path/filepath"
22+
"strings"
2223

2324
"github.com/cockroachdb/errors"
2425
"github.com/docker/cli/cli/config"
2526
"github.com/moby/buildkit/client"
2627
"github.com/moby/buildkit/client/llb"
2728
"github.com/moby/buildkit/session"
2829
"github.com/moby/buildkit/session/auth/authprovider"
30+
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
2931
"github.com/sirupsen/logrus"
3032
"golang.org/x/sync/errgroup"
3133

@@ -157,7 +159,11 @@ func (b generalBuilder) Interpret() error {
157159

158160
func (b generalBuilder) Compile(ctx context.Context) (*llb.Definition, error) {
159161
envName := filepath.Base(b.BuildContextDir)
160-
def, err := b.graph.Compile(ctx, envName, b.PubKeyPath, b.Options.ProgressMode)
162+
platform, err := parsePlatform(b.Platform)
163+
if err != nil {
164+
return nil, err
165+
}
166+
def, err := b.graph.Compile(ctx, envName, b.PubKeyPath, platform, b.Options.ProgressMode)
161167
if err != nil {
162168
return nil, errors.Wrap(err, "failed to compile build.envd")
163169
}
@@ -191,8 +197,9 @@ func (b generalBuilder) imageConfig(ctx context.Context) (string, error) {
191197

192198
env := b.graph.GetEnviron()
193199
user := b.graph.GetUser()
200+
platform := b.graph.GetPlatform()
194201

195-
data, err := ImageConfigStr(labels, ports, ep, env, user)
202+
data, err := ImageConfigStr(labels, ports, ep, env, user, platform)
196203
if err != nil {
197204
return "", errors.Wrap(err, "failed to get image config")
198205
}
@@ -335,3 +342,19 @@ func constructSolveOpt(ce []client.CacheOptionsEntry, entry client.ExportEntry,
335342
}
336343
return opt
337344
}
345+
346+
func parsePlatform(platform string) (*ocispecs.Platform, error) {
347+
os, arch, variant := "linux", "amd64", ""
348+
if platform == "" {
349+
return &ocispecs.Platform{Architecture: arch, OS: os, Variant: variant}, nil
350+
}
351+
arr := strings.Split(platform, "/")
352+
if len(arr) < 2 {
353+
return nil, errors.New("invalid platform format, expected `os/arch[/variant]`")
354+
}
355+
os, arch = arr[0], arr[1]
356+
if len(arr) >= 3 {
357+
variant = arr[2]
358+
}
359+
return &ocispecs.Platform{Architecture: arch, OS: os, Variant: variant}, nil
360+
}

pkg/builder/types.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ type Options struct {
4949
ImportCache string
5050
// UseHTTPProxy uses HTTPS_PROXY/HTTP_PROXY/NO_PROXY in the build process.
5151
UseHTTPProxy bool
52+
// Specify the target platform for the build output.
53+
// e.g. platform=linux/arm64,linux/amd64
54+
Platform string
5255
}
5356

5457
type generalBuilder struct {

pkg/builder/util.go

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import (
2525
"github.com/containerd/console"
2626
"github.com/moby/buildkit/client"
2727
gatewayclient "github.com/moby/buildkit/frontend/gateway/client"
28-
v1 "github.com/opencontainers/image-spec/specs-go/v1"
28+
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
2929
"github.com/sirupsen/logrus"
3030
)
3131

@@ -35,20 +35,20 @@ const (
3535
)
3636

3737
func ImageConfigStr(labels map[string]string, ports map[string]struct{},
38-
entrypoint []string, env []string, user string) (string, error) {
39-
img := v1.Image{
40-
Config: v1.ImageConfig{
38+
entrypoint []string, env []string, user string, platform *ocispecs.Platform) (string, error) {
39+
img := ocispecs.Image{
40+
Config: ocispecs.ImageConfig{
4141
Labels: labels,
4242
User: user,
4343
WorkingDir: "/",
4444
Env: env,
4545
ExposedPorts: ports,
4646
Entrypoint: entrypoint,
4747
},
48-
Architecture: "amd64",
48+
Architecture: platform.Architecture,
4949
// Refer to https://github.com/tensorchord/envd/issues/269#issuecomment-1152944914
50-
OS: "linux",
51-
RootFS: v1.RootFS{
50+
OS: platform.OS,
51+
RootFS: ocispecs.RootFS{
5252
Type: "layers",
5353
},
5454
}

pkg/editor/vscode/util.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import (
2222
"strings"
2323

2424
"github.com/cockroachdb/errors"
25-
v1 "github.com/opencontainers/image-spec/specs-go/v1"
25+
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
2626
"github.com/sirupsen/logrus"
2727
)
2828

@@ -38,7 +38,7 @@ const (
3838
PLATFORM_ALPINE_X64 = "alpine-x64"
3939
)
4040

41-
func ConvertLLBPlatform(platform *v1.Platform) (string, error) {
41+
func ConvertLLBPlatform(platform *ocispecs.Platform) (string, error) {
4242
// Convert opencontainers style platform to VSCode extension style platform.
4343
switch platform.OS {
4444
case "windows":

pkg/lang/ir/graph.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@ import (
1818
"context"
1919

2020
"github.com/moby/buildkit/client/llb"
21+
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
2122

2223
"github.com/tensorchord/envd/pkg/progress/compileui"
2324
)
2425

2526
type Graph interface {
26-
Compile(ctx context.Context, envName string, pub string, progressMode string) (*llb.Definition, error)
27+
Compile(ctx context.Context, envName string, pub string, platform *ocispecs.Platform, progressMode string) (*llb.Definition, error)
2728

2829
graphDebugger
2930
graphVisitor
@@ -56,4 +57,5 @@ type graphVisitor interface {
5657
GetHTTP() []HTTPInfo
5758
GetRuntimeCommands() map[string]string
5859
GetUser() string
60+
GetPlatform() *ocispecs.Platform
5961
}

pkg/lang/ir/util.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,16 @@ import (
2626
v1 "github.com/opencontainers/image-spec/specs-go/v1"
2727
)
2828

29-
func FetchImageConfig(ctx context.Context, imageName string) (config v1.ImageConfig, err error) {
29+
func FetchImageConfig(ctx context.Context, imageName string, platform *v1.Platform) (config v1.ImageConfig, err error) {
3030
ref, err := docker.ParseReference(fmt.Sprintf("//%s", imageName))
3131
if err != nil {
3232
return config, errors.Wrap(err, "failed to parse image reference")
3333
}
3434
sys := types.SystemContext{}
35+
if platform != nil {
36+
sys.ArchitectureChoice = platform.Architecture
37+
sys.OSChoice = platform.OS
38+
}
3539
src, err := ref.NewImageSource(ctx, &sys)
3640
if err != nil {
3741
return config, errors.Wrap(err, "failed to get image source from ref")

0 commit comments

Comments
 (0)