Skip to content

Commit e4da654

Browse files
tonistiigitonistiigi
authored
Merge pull request #5601 from tonistiigi/v0.18.2-picks
[v0.18] cherry-picks for v0.18.2
2 parents eb68885 + 987b409 commit e4da654

File tree

17 files changed

+254
-30
lines changed

17 files changed

+254
-30
lines changed

‎Dockerfile

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# syntax=docker/dockerfile-upstream:master
22

3-
ARG RUNC_VERSION=v1.2.2
3+
ARG RUNC_VERSION=v1.2.3
44
ARG CONTAINERD_VERSION=v2.0.0
55
# CONTAINERD_ALT_VERSION_... defines fallback containerd version for integration tests
66
ARG CONTAINERD_ALT_VERSION_17=v1.7.23
@@ -20,7 +20,7 @@ ARG DELVE_VERSION=v1.23.1
2020

2121
ARG GO_VERSION=1.23
2222
ARG ALPINE_VERSION=3.20
23-
ARG XX_VERSION=1.5.0
23+
ARG XX_VERSION=1.6.1
2424
ARG BUILDKIT_DEBUG
2525

2626
# minio for s3 integration tests

‎client/llb/definition.go

-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import (
1616
// For example, after marshalling a LLB state and sending over the wire, the
1717
// LLB state can be reconstructed from the definition.
1818
type DefinitionOp struct {
19-
MarshalCache
2019
mu sync.Mutex
2120
ops map[digest.Digest]*pb.Op
2221
defs map[digest.Digest][]byte

‎client/llb/diff.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
)
99

1010
type DiffOp struct {
11-
MarshalCache
11+
cache MarshalCache
1212
lower Output
1313
upper Output
1414
output Output
@@ -31,7 +31,10 @@ func (m *DiffOp) Validate(ctx context.Context, constraints *Constraints) error {
3131
}
3232

3333
func (m *DiffOp) Marshal(ctx context.Context, constraints *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
34-
if dgst, dt, md, srcs, err := m.Load(constraints); err == nil {
34+
cache := m.cache.Acquire()
35+
defer cache.Release()
36+
37+
if dgst, dt, md, srcs, err := cache.Load(constraints); err == nil {
3538
return dgst, dt, md, srcs, nil
3639
}
3740
if err := m.Validate(ctx, constraints); err != nil {
@@ -72,7 +75,7 @@ func (m *DiffOp) Marshal(ctx context.Context, constraints *Constraints) (digest.
7275
return "", nil, nil, nil, err
7376
}
7477

75-
return m.Store(dt, md, m.constraints.SourceLocations, constraints)
78+
return cache.Store(dt, md, m.constraints.SourceLocations, constraints)
7679
}
7780

7881
func (m *DiffOp) Output() Output {

‎client/llb/exec.go

+10-4
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ type mount struct {
5151
}
5252

5353
type ExecOp struct {
54-
MarshalCache
54+
cache MarshalCache
5555
proxyEnv *ProxyEnv
5656
root Output
5757
mounts []*mount
@@ -63,6 +63,9 @@ type ExecOp struct {
6363
}
6464

6565
func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Output {
66+
cache := e.cache.Acquire()
67+
defer cache.Release()
68+
6669
m := &mount{
6770
target: target,
6871
source: source,
@@ -84,7 +87,7 @@ func (e *ExecOp) AddMount(target string, source Output, opt ...MountOption) Outp
8487
}
8588
m.output = o
8689
}
87-
e.Store(nil, nil, nil, nil)
90+
cache.Store(nil, nil, nil, nil)
8891
e.isValidated = false
8992
return m.output
9093
}
@@ -128,7 +131,10 @@ func (e *ExecOp) Validate(ctx context.Context, c *Constraints) error {
128131
}
129132

130133
func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
131-
if dgst, dt, md, srcs, err := e.Load(c); err == nil {
134+
cache := e.cache.Acquire()
135+
defer cache.Release()
136+
137+
if dgst, dt, md, srcs, err := cache.Load(c); err == nil {
132138
return dgst, dt, md, srcs, nil
133139
}
134140

@@ -446,7 +452,7 @@ func (e *ExecOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
446452
if err != nil {
447453
return "", nil, nil, nil, err
448454
}
449-
return e.Store(dt, md, e.constraints.SourceLocations, c)
455+
return cache.Store(dt, md, e.constraints.SourceLocations, c)
450456
}
451457

452458
func (e *ExecOp) Output() Output {

‎client/llb/fileop.go

+5-2
Original file line numberDiff line numberDiff line change
@@ -746,7 +746,10 @@ func (ms *marshalState) add(fa *FileAction, c *Constraints) (*fileActionState, e
746746
}
747747

748748
func (f *FileOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
749-
if dgst, dt, md, srcs, err := f.Load(c); err == nil {
749+
cache := f.Acquire()
750+
defer cache.Release()
751+
752+
if dgst, dt, md, srcs, err := cache.Load(c); err == nil {
750753
return dgst, dt, md, srcs, nil
751754
}
752755

@@ -816,7 +819,7 @@ func (f *FileOp) Marshal(ctx context.Context, c *Constraints) (digest.Digest, []
816819
if err != nil {
817820
return "", nil, nil, nil, err
818821
}
819-
return f.Store(dt, md, f.constraints.SourceLocations, c)
822+
return cache.Store(dt, md, f.constraints.SourceLocations, c)
820823
}
821824

822825
func normalizePath(parent, p string, keepSlash bool) string {

‎client/llb/fileop_test.go

+13
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/moby/buildkit/solver/pb"
99
digest "github.com/opencontainers/go-digest"
1010
"github.com/stretchr/testify/require"
11+
"golang.org/x/sync/errgroup"
1112
)
1213

1314
func TestFileMkdir(t *testing.T) {
@@ -737,3 +738,15 @@ func TestFileOpMarshalConsistency(t *testing.T) {
737738
prevDef = def.Def
738739
}
739740
}
741+
742+
func TestParallelMarshal(t *testing.T) {
743+
st := Scratch().File(Mkfile("/tmp", 0644, []byte("tmp 1")))
744+
eg, ctx := errgroup.WithContext(context.Background())
745+
for i := 0; i < 100; i++ {
746+
eg.Go(func() error {
747+
_, err := st.Marshal(ctx)
748+
return err
749+
})
750+
}
751+
require.NoError(t, eg.Wait())
752+
}

‎client/llb/llbbuild/llbbuild.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func NewBuildOp(source llb.Output, opt ...BuildOption) llb.Vertex {
2424
}
2525

2626
type build struct {
27-
llb.MarshalCache
27+
cache llb.MarshalCache
2828
source llb.Output
2929
info *BuildInfo
3030
constraints llb.Constraints
@@ -47,7 +47,10 @@ func (b *build) Validate(context.Context, *llb.Constraints) error {
4747
}
4848

4949
func (b *build) Marshal(ctx context.Context, c *llb.Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*llb.SourceLocation, error) {
50-
if dgst, dt, md, srcs, err := b.Load(c); err == nil {
50+
cache := b.cache.Acquire()
51+
defer cache.Release()
52+
53+
if dgst, dt, md, srcs, err := cache.Load(c); err == nil {
5154
return dgst, dt, md, srcs, nil
5255
}
5356

@@ -85,7 +88,7 @@ func (b *build) Marshal(ctx context.Context, c *llb.Constraints) (digest.Digest,
8588
if err != nil {
8689
return "", nil, nil, nil, err
8790
}
88-
return b.Store(dt, md, b.constraints.SourceLocations, c)
91+
return cache.Store(dt, md, b.constraints.SourceLocations, c)
8992
}
9093

9194
func (b *build) Output() llb.Output {

‎client/llb/marshal.go

+22-7
Original file line numberDiff line numberDiff line change
@@ -117,30 +117,45 @@ func MarshalConstraints(base, override *Constraints) (*pb.Op, *pb.OpMetadata) {
117117
}
118118

119119
type MarshalCache struct {
120-
cache sync.Map
120+
mu sync.Mutex
121+
cache map[*Constraints]*marshalCacheResult
121122
}
122123

123-
func (mc *MarshalCache) Load(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
124-
v, ok := mc.cache.Load(c)
124+
type MarshalCacheInstance struct {
125+
*MarshalCache
126+
}
127+
128+
func (mc *MarshalCache) Acquire() *MarshalCacheInstance {
129+
mc.mu.Lock()
130+
return &MarshalCacheInstance{mc}
131+
}
132+
133+
func (mc *MarshalCacheInstance) Load(c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
134+
res, ok := mc.cache[c]
125135
if !ok {
126136
return "", nil, nil, nil, cerrdefs.ErrNotFound
127137
}
128-
129-
res := v.(*marshalCacheResult)
130138
return res.digest, res.dt, res.md, res.srcs, nil
131139
}
132140

133-
func (mc *MarshalCache) Store(dt []byte, md *pb.OpMetadata, srcs []*SourceLocation, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
141+
func (mc *MarshalCacheInstance) Store(dt []byte, md *pb.OpMetadata, srcs []*SourceLocation, c *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
134142
res := &marshalCacheResult{
135143
digest: digest.FromBytes(dt),
136144
dt: dt,
137145
md: md,
138146
srcs: srcs,
139147
}
140-
mc.cache.Store(c, res)
148+
if mc.cache == nil {
149+
mc.cache = make(map[*Constraints]*marshalCacheResult)
150+
}
151+
mc.cache[c] = res
141152
return res.digest, res.dt, res.md, res.srcs, nil
142153
}
143154

155+
func (mc *MarshalCacheInstance) Release() {
156+
mc.mu.Unlock()
157+
}
158+
144159
type marshalCacheResult struct {
145160
digest digest.Digest
146161
dt []byte

‎client/llb/merge.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
)
1010

1111
type MergeOp struct {
12-
MarshalCache
12+
cache MarshalCache
1313
inputs []Output
1414
output Output
1515
constraints Constraints
@@ -32,7 +32,10 @@ func (m *MergeOp) Validate(ctx context.Context, constraints *Constraints) error
3232
}
3333

3434
func (m *MergeOp) Marshal(ctx context.Context, constraints *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
35-
if dgst, dt, md, srcs, err := m.Load(constraints); err == nil {
35+
cache := m.cache.Acquire()
36+
defer cache.Release()
37+
38+
if dgst, dt, md, srcs, err := cache.Load(constraints); err == nil {
3639
return dgst, dt, md, srcs, nil
3740
}
3841

@@ -59,7 +62,7 @@ func (m *MergeOp) Marshal(ctx context.Context, constraints *Constraints) (digest
5962
return "", nil, nil, nil, err
6063
}
6164

62-
return m.Store(dt, md, m.constraints.SourceLocations, constraints)
65+
return cache.Store(dt, md, m.constraints.SourceLocations, constraints)
6366
}
6467

6568
func (m *MergeOp) Output() Output {

‎client/llb/source.go

+6-3
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020
)
2121

2222
type SourceOp struct {
23-
MarshalCache
23+
cache MarshalCache
2424
id string
2525
attrs map[string]string
2626
output Output
@@ -49,7 +49,10 @@ func (s *SourceOp) Validate(ctx context.Context, c *Constraints) error {
4949
}
5050

5151
func (s *SourceOp) Marshal(ctx context.Context, constraints *Constraints) (digest.Digest, []byte, *pb.OpMetadata, []*SourceLocation, error) {
52-
if dgst, dt, md, srcs, err := s.Load(constraints); err == nil {
52+
cache := s.cache.Acquire()
53+
defer cache.Release()
54+
55+
if dgst, dt, md, srcs, err := cache.Load(constraints); err == nil {
5356
return dgst, dt, md, srcs, nil
5457
}
5558

@@ -82,7 +85,7 @@ func (s *SourceOp) Marshal(ctx context.Context, constraints *Constraints) (diges
8285
return "", nil, nil, nil, err
8386
}
8487

85-
return s.Store(dt, md, s.constraints.SourceLocations, constraints)
88+
return cache.Store(dt, md, s.constraints.SourceLocations, constraints)
8689
}
8790

8891
func (s *SourceOp) Output() Output {

‎frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
ARG GO_VERSION=1.23
44
ARG ALPINE_VERSION=3.20
5-
ARG XX_VERSION=1.5.0
5+
ARG XX_VERSION=1.6.1
66

77
# xx is a helper for cross-compilation
88
FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx

‎frontend/dockerfile/dockerfile2llb/convert.go

+3
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,7 @@ func toDispatchState(ctx context.Context, dt []byte, opt ConvertOpt) (*dispatchS
376376
}
377377
}
378378
allDispatchStates.addState(ds)
379+
ds.base = nil // reset base set by addState
379380
continue
380381
}
381382
}
@@ -1065,6 +1066,8 @@ func (ds *dispatchState) init() {
10651066
ds.state = ds.base.state
10661067
ds.platform = ds.base.platform
10671068
ds.image = clone(ds.base.image)
1069+
// onbuild triggers to not carry over from base stage
1070+
ds.image.Config.OnBuild = nil
10681071
ds.baseImg = cloneX(ds.base.baseImg)
10691072
// Utilize the same path index as our base image so we propagate
10701073
// the paths we use back to the base image.

‎frontend/dockerfile/dockerfile2llb/image.go

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ func clone(src dockerspec.DockerOCIImage) dockerspec.DockerOCIImage {
1212
img.Config.Env = append([]string{}, src.Config.Env...)
1313
img.Config.Cmd = append([]string{}, src.Config.Cmd...)
1414
img.Config.Entrypoint = append([]string{}, src.Config.Entrypoint...)
15+
img.Config.OnBuild = append([]string{}, src.Config.OnBuild...)
1516
return img
1617
}
1718

‎frontend/dockerfile/dockerfile_mount_test.go

+41
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ var mountTests = integration.TestFuncs(
2828
testMountTmpfsSize,
2929
testMountDuplicate,
3030
testCacheMountUser,
31+
testCacheMountParallel,
3132
)
3233

3334
func init() {
@@ -536,3 +537,43 @@ COPY --from=base /combined.txt /
536537
test("foo\n")
537538
test("updated\n")
538539
}
540+
541+
// moby/buildkit#5566
542+
func testCacheMountParallel(t *testing.T, sb integration.Sandbox) {
543+
integration.SkipOnPlatform(t, "windows")
544+
f := getFrontend(t, sb)
545+
546+
dockerfile := []byte(`
547+
FROM alpine AS b1
548+
RUN --mount=type=cache,target=/foo/bar --mount=type=cache,target=/foo/bar/baz echo 1
549+
550+
FROM alpine AS b2
551+
RUN --mount=type=cache,target=/foo/bar --mount=type=cache,target=/foo/bar/baz echo 2
552+
553+
FROM scratch
554+
COPY --from=b1 /etc/passwd p1
555+
COPY --from=b2 /etc/passwd p2
556+
`)
557+
558+
dir := integration.Tmpdir(
559+
t,
560+
fstest.CreateFile("Dockerfile", dockerfile, 0600),
561+
)
562+
563+
c, err := client.New(sb.Context(), sb.Address())
564+
require.NoError(t, err)
565+
defer c.Close()
566+
567+
for i := 0; i < 20; i++ {
568+
_, err = f.Solve(sb.Context(), c, client.SolveOpt{
569+
FrontendAttrs: map[string]string{
570+
"no-cache": "",
571+
},
572+
LocalMounts: map[string]fsutil.FS{
573+
dockerui.DefaultLocalNameDockerfile: dir,
574+
dockerui.DefaultLocalNameContext: dir,
575+
},
576+
}, nil)
577+
require.NoError(t, err)
578+
}
579+
}

0 commit comments

Comments
 (0)