From 8b3982819b44c56879b1106c82674c00d3cfc62a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Feb 2025 18:18:47 +0000 Subject: [PATCH] build(deps): bump github.com/containers/storage from 1.56.0 to 1.57.1 Bumps [github.com/containers/storage](https://github.com/containers/storage) from 1.56.0 to 1.57.1. - [Release notes](https://github.com/containers/storage/releases) - [Changelog](https://github.com/containers/storage/blob/main/docs/containers-storage-changes.md) - [Commits](https://github.com/containers/storage/compare/v1.56.0...v1.57.1) --- updated-dependencies: - dependency-name: github.com/containers/storage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 12 +- go.sum | 24 +- .../stargz-snapshotter/estargz/testutil.go | 15 +- .../github.com/containers/storage/.cirrus.yml | 4 +- vendor/github.com/containers/storage/Makefile | 2 +- vendor/github.com/containers/storage/VERSION | 2 +- vendor/github.com/containers/storage/check.go | 8 +- .../containers/storage/drivers/aufs/aufs.go | 5 + .../containers/storage/drivers/btrfs/btrfs.go | 5 + .../storage/drivers/chown_darwin.go | 2 +- .../containers/storage/drivers/chown_unix.go | 2 +- .../storage/drivers/copy/copy_linux.go | 4 +- .../containers/storage/drivers/driver.go | 24 +- .../storage/drivers/overlay/check_116.go | 3 +- .../storage/drivers/overlay/composefs.go | 4 +- .../storage/drivers/overlay/overlay.go | 21 + .../storage/drivers/overlay/overlay_nocgo.go | 23 - .../drivers/register/register_overlay.go | 2 +- .../containers/storage/drivers/vfs/driver.go | 17 + .../storage/drivers/windows/windows.go | 5 + .../containers/storage/drivers/zfs/zfs.go | 5 + .../storage/internal/dedup/dedup.go | 163 +++++ .../storage/internal/dedup/dedup_linux.go | 139 ++++ .../internal/dedup/dedup_unsupported.go | 27 + .../github.com/containers/storage/layers.go | 37 +- .../containers/storage/pkg/archive/archive.go | 55 +- .../containers/storage/pkg/archive/changes.go | 3 + .../storage/pkg/archive/changes_linux.go | 11 +- .../storage/pkg/chrootarchive/archive.go | 2 +- .../storage/pkg/chunked/cache_linux.go | 12 +- .../storage/pkg/chunked/compression.go | 18 +- .../storage/pkg/chunked/compression_linux.go | 191 +++--- .../pkg/chunked/compressor/compressor.go | 24 +- .../storage/pkg/chunked/dump/dump.go | 66 +- .../storage/pkg/chunked/filesystem_linux.go | 73 ++- .../internal/{ => minimal}/compression.go | 2 +- .../storage/pkg/chunked/internal/path/path.go | 27 + .../storage/pkg/chunked/storage_linux.go | 612 +++++++++++++----- .../pkg/chunked/storage_unsupported.go | 2 +- .../containers/storage/pkg/chunked/toc/toc.go | 4 +- .../containers/storage/pkg/idtools/idtools.go | 169 ++++- .../containers/storage/pkg/ioutils/writers.go | 4 +- .../storage/pkg/loopback/attach_loopback.go | 2 +- .../containers/storage/pkg/loopback/ioctl.go | 2 +- .../storage/pkg/loopback/loop_wrapper.go | 38 +- .../storage/pkg/loopback/loopback.go | 2 +- .../storage/pkg/system/extattr_freebsd.go | 93 +++ .../storage/pkg/system/extattr_unsupported.go | 24 + .../storage/pkg/system/stat_netbsd.go | 13 + .../storage/pkg/system/xattrs_darwin.go | 2 +- .../storage/pkg/system/xattrs_freebsd.go | 85 +++ .../storage/pkg/system/xattrs_linux.go | 2 +- .../storage/pkg/system/xattrs_unsupported.go | 4 +- .../containers/storage/storage.conf | 19 + vendor/github.com/containers/storage/store.go | 64 ++ .../cyphar/filepath-securejoin/CHANGELOG.md | 35 +- .../cyphar/filepath-securejoin/VERSION | 2 +- .../gocompat_errors_go120.go | 18 + .../gocompat_errors_unsupported.go | 38 ++ .../gocompat_generics_go121.go | 32 + .../gocompat_generics_unsupported.go | 124 ++++ .../filepath-securejoin/lookup_linux.go | 3 +- .../cyphar/filepath-securejoin/mkdir_linux.go | 18 +- .../filepath-securejoin/openat2_linux.go | 3 +- .../filepath-securejoin/procfs_linux.go | 30 +- .../vbatts/tar-split/archive/tar/format.go | 4 + .../vbatts/tar-split/archive/tar/reader.go | 14 +- vendor/golang.org/x/exp/maps/maps.go | 58 +- vendor/modules.txt | 20 +- 69 files changed, 2013 insertions(+), 566 deletions(-) delete mode 100644 vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go create mode 100644 vendor/github.com/containers/storage/internal/dedup/dedup.go create mode 100644 vendor/github.com/containers/storage/internal/dedup/dedup_linux.go create mode 100644 vendor/github.com/containers/storage/internal/dedup/dedup_unsupported.go rename vendor/github.com/containers/storage/pkg/chunked/internal/{ => minimal}/compression.go (99%) create mode 100644 vendor/github.com/containers/storage/pkg/chunked/internal/path/path.go create mode 100644 vendor/github.com/containers/storage/pkg/system/extattr_freebsd.go create mode 100644 vendor/github.com/containers/storage/pkg/system/extattr_unsupported.go create mode 100644 vendor/github.com/containers/storage/pkg/system/stat_netbsd.go create mode 100644 vendor/github.com/containers/storage/pkg/system/xattrs_freebsd.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/gocompat_errors_go120.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/gocompat_errors_unsupported.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/gocompat_generics_go121.go create mode 100644 vendor/github.com/cyphar/filepath-securejoin/gocompat_generics_unsupported.go diff --git a/go.mod b/go.mod index b14e05f5..37795b83 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/spf13/pflag v1.0.6 // indirect ) -require github.com/containers/storage v1.56.0 +require github.com/containers/storage v1.57.1 require ( dario.cat/mergo v1.0.1 // indirect @@ -36,7 +36,7 @@ require ( github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect - github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/containerd/typeurl/v2 v2.2.0 // indirect github.com/containers/buildah v1.38.1-0.20241119213149-52437ef15d33 // indirect github.com/containers/gvisor-tap-vsock v0.8.3 // indirect @@ -49,7 +49,7 @@ require ( github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09 // indirect github.com/crc-org/vfkit v0.6.0 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f // indirect - github.com/cyphar/filepath-securejoin v0.3.4 // indirect + github.com/cyphar/filepath-securejoin v0.3.6 // indirect github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e // indirect github.com/digitalocean/go-qemu v0.0.0-20230711162256-2e3d0186973e // indirect github.com/disiqueira/gotree/v3 v3.0.2 // indirect @@ -148,14 +148,14 @@ require ( github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect github.com/sylabs/sif/v2 v2.19.1 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect - github.com/tchap/go-patricia/v2 v2.3.1 // indirect + github.com/tchap/go-patricia/v2 v2.3.2 // indirect github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.9.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/vbatts/tar-split v0.11.6 // indirect + github.com/vbatts/tar-split v0.11.7 // indirect github.com/vbauerster/mpb/v8 v8.8.3 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.mongodb.org/mongo-driver v1.14.0 // indirect @@ -167,7 +167,7 @@ require ( go.opentelemetry.io/otel/trace v1.28.0 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.33.0 // indirect - golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect + golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect golang.org/x/net v0.35.0 // indirect golang.org/x/sync v0.11.0 // indirect golang.org/x/sys v0.30.0 // indirect diff --git a/go.sum b/go.sum index df3e8fd5..b4c1381c 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= -github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk= +github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= +github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= github.com/containers/buildah v1.38.1-0.20241119213149-52437ef15d33 h1:Ih6KuyByK7ZGGzkS0M5rVBPLWIyeDvdL5klhsKBo8vA= @@ -82,8 +82,8 @@ github.com/containers/ocicrypt v1.2.0 h1:X14EgRK3xNFvJEfI5O4Qn4T3E25ANudSOZz/sir github.com/containers/ocicrypt v1.2.0/go.mod h1:ZNviigQajtdlxIZGibvblVuIFBKIuUI2M0QM12SD31U= github.com/containers/psgo v1.9.0 h1:eJ74jzSaCHnWt26OlKZROSyUyRcGDf+gYBdXnxrMW4g= github.com/containers/psgo v1.9.0/go.mod h1:0YoluUm43Mz2UnBIh1P+6V6NWcbpTL5uRtXyOcH0B5A= -github.com/containers/storage v1.56.0 h1:DZ9KSkj6M2tvj/4bBoaJu3QDHRl35BwsZ4kmLJS97ZI= -github.com/containers/storage v1.56.0/go.mod h1:c6WKowcAlED/DkWGNuL9bvGYqIWCVy7isRMdCSKWNjk= +github.com/containers/storage v1.57.1 h1:hKPoFsuBcB3qTzBxa4IFpZMRzUuL5Xhv/BE44W0XHx8= +github.com/containers/storage v1.57.1/go.mod h1:i/Hb4lu7YgFr9G0K6BMjqW0BLJO1sFsnWQwj2UoWCUM= github.com/containers/winquit v1.1.0 h1:jArun04BNDQvt2W0Y78kh9TazN2EIEMG5Im6/JY7+pE= github.com/containers/winquit v1.1.0/go.mod h1:PsPeZlnbkmGGIToMPHF1zhWjBUkd8aHjMOr/vFcPxw8= github.com/coreos/go-systemd/v22 v22.5.1-0.20231103132048-7d375ecc2b09 h1:OoRAFlvDGCUqDLampLQjk0yeeSGdF9zzst/3G9IkBbc= @@ -97,8 +97,8 @@ github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f h1:eHnXnuK47UlSTOQexbzxAZfekVz6i+LKRdj1CU5DPaM= github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw= -github.com/cyphar/filepath-securejoin v0.3.4 h1:VBWugsJh2ZxJmLFSM06/0qzQyiQX2Qs0ViKrUAcqdZ8= -github.com/cyphar/filepath-securejoin v0.3.4/go.mod h1:8s/MCNJREmFK0H02MF6Ihv1nakJe4L/w3WZLHNkvlYM= +github.com/cyphar/filepath-securejoin v0.3.6 h1:4d9N5ykBnSp5Xn2JkhocYDkOpURL/18CYMpo6xB9uWM= +github.com/cyphar/filepath-securejoin v0.3.6/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -403,8 +403,8 @@ github.com/sylabs/sif/v2 v2.19.1 h1:1eeMmFc8elqJe60ZiWwXgL3gMheb0IP4GmNZ4q0IEA0= github.com/sylabs/sif/v2 v2.19.1/go.mod h1:U1SUhvl8X1JIxAylC0DYz1fa/Xba6EMZD1dGPGBH83E= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes= -github.com/tchap/go-patricia/v2 v2.3.1/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= +github.com/tchap/go-patricia/v2 v2.3.2 h1:xTHFutuitO2zqKAQ5rCROYgUb7Or/+IC3fts9/Yc7nM= +github.com/tchap/go-patricia/v2 v2.3.2/go.mod h1:VZRHKAb53DLaG+nA9EaYYiaEx6YztwDlLElMsnSHD4k= github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 h1:e/5i7d4oYZ+C1wj2THlRK+oAhjeS/TRQwMfkIuet3w0= github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399/go.mod h1:LdwHTNJT99C5fTAzDz0ud328OgXz+gierycbcIx2fRs= github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= @@ -417,8 +417,8 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs= -github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI= +github.com/vbatts/tar-split v0.11.7 h1:ixZ93pO/GmvaZw4Vq9OwmfZK/kc2zKdPfu0B+gYqs3U= +github.com/vbatts/tar-split v0.11.7/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= github.com/vbauerster/mpb/v8 v8.8.3 h1:dTOByGoqwaTJYPubhVz3lO5O6MK553XVgUo33LdnNsQ= github.com/vbauerster/mpb/v8 v8.8.3/go.mod h1:JfCCrtcMsJwP6ZwMn9e5LMnNyp3TVNpUWWkN+nd4EWk= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= @@ -463,8 +463,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= -golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= +golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 h1:1UoZQm6f0P/ZO0w1Ri+f+ifG/gXhegadRdwBIXEFWDo= +golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67/go.mod h1:qj5a5QZpwLU2NLQudwIN5koi3beDhSAlJwa67PuM98c= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go index 0ca6fd75..ba650b4d 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go @@ -26,12 +26,13 @@ import ( "archive/tar" "bytes" "compress/gzip" + "crypto/rand" "crypto/sha256" "encoding/json" "errors" "fmt" "io" - "math/rand" + "math/big" "os" "path/filepath" "reflect" @@ -45,10 +46,6 @@ import ( digest "github.com/opencontainers/go-digest" ) -func init() { - rand.Seed(time.Now().UnixNano()) -} - // TestingController is Compression with some helper methods necessary for testing. type TestingController interface { Compression @@ -920,9 +917,11 @@ func checkVerifyInvalidTOCEntryFail(filename string) check { } if sampleEntry == nil { t.Fatalf("TOC must contain at least one regfile or chunk entry other than the rewrite target") + return } if targetEntry == nil { t.Fatalf("rewrite target not found") + return } targetEntry.Offset = sampleEntry.Offset }, @@ -2291,7 +2290,11 @@ var runes = []rune("1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWX func randomContents(n int) string { b := make([]rune, n) for i := range b { - b[i] = runes[rand.Intn(len(runes))] + bi, err := rand.Int(rand.Reader, big.NewInt(int64(len(runes)))) + if err != nil { + panic(err) + } + b[i] = runes[int(bi.Int64())] } return string(b) } diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index 0fed08c3..2102e938 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -17,13 +17,13 @@ env: #### #### Cache-image names to test with (double-quotes around names are critical) ### - FEDORA_NAME: "fedora-39" + FEDORA_NAME: "fedora-41" DEBIAN_NAME: "debian-13" # GCE project where images live IMAGE_PROJECT: "libpod-218412" # VM Image built in containers/automation_images - IMAGE_SUFFIX: "c20241010t105554z-f40f39d13" + IMAGE_SUFFIX: "c20250107t132430z-f41f40d13" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" DEBIAN_CACHE_IMAGE_NAME: "debian-${IMAGE_SUFFIX}" diff --git a/vendor/github.com/containers/storage/Makefile b/vendor/github.com/containers/storage/Makefile index 64c01b67..888fef84 100644 --- a/vendor/github.com/containers/storage/Makefile +++ b/vendor/github.com/containers/storage/Makefile @@ -35,7 +35,7 @@ TESTFLAGS := $(shell $(GO) test -race $(BUILDFLAGS) ./pkg/stringutils 2>&1 > /de # N/B: This value is managed by Renovate, manual changes are # possible, as long as they don't disturb the formatting # (i.e. DO NOT ADD A 'v' prefix!) -GOLANGCI_LINT_VERSION := 1.61.0 +GOLANGCI_LINT_VERSION := 1.63.4 default all: local-binary docs local-validate local-cross ## validate all checks, build and cross-build\nbinaries and docs diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 3ebf789f..b4cf7c0d 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.56.0 +1.57.1 diff --git a/vendor/github.com/containers/storage/check.go b/vendor/github.com/containers/storage/check.go index 396648e7..e8837ff9 100644 --- a/vendor/github.com/containers/storage/check.go +++ b/vendor/github.com/containers/storage/check.go @@ -80,7 +80,7 @@ type CheckOptions struct { // layer to the contents that we'd expect it to have to ignore certain // discrepancies type checkIgnore struct { - ownership, timestamps, permissions bool + ownership, timestamps, permissions, filetype bool } // CheckMost returns a CheckOptions with mostly just "quick" checks enabled. @@ -139,8 +139,10 @@ func (s *store) Check(options *CheckOptions) (CheckReport, error) { if strings.Contains(o, "ignore_chown_errors=true") { ignore.ownership = true } - if strings.HasPrefix(o, "force_mask=") { + if strings.Contains(o, "force_mask=") { + ignore.ownership = true ignore.permissions = true + ignore.filetype = true } } for o := range s.pullOptions { @@ -833,7 +835,7 @@ func (s *store) Repair(report CheckReport, options *RepairOptions) []error { // compareFileInfo returns a string summarizing what's different between the two checkFileInfos func compareFileInfo(a, b checkFileInfo, idmap *idtools.IDMappings, ignore checkIgnore) string { var comparison []string - if a.typeflag != b.typeflag { + if a.typeflag != b.typeflag && !ignore.filetype { comparison = append(comparison, fmt.Sprintf("filetype:%v→%v", a.typeflag, b.typeflag)) } if idmap != nil && !idmap.Empty() { diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index a8312811..d0393426 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -776,3 +776,8 @@ func (a *Driver) UpdateLayerIDMap(id string, toContainer, toHost *idtools.IDMapp func (a *Driver) SupportsShifting() bool { return false } + +// Dedup performs deduplication of the driver's storage. +func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) { + return graphdriver.DedupResult{}, nil +} diff --git a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go index c7f37d97..4a80339f 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go @@ -673,3 +673,8 @@ func (d *Driver) ListLayers() ([]string, error) { func (d *Driver) AdditionalImageStores() []string { return nil } + +// Dedup performs deduplication of the driver's storage. +func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) { + return graphdriver.DedupResult{}, nil +} diff --git a/vendor/github.com/containers/storage/drivers/chown_darwin.go b/vendor/github.com/containers/storage/drivers/chown_darwin.go index 882fa04b..4f275020 100644 --- a/vendor/github.com/containers/storage/drivers/chown_darwin.go +++ b/vendor/github.com/containers/storage/drivers/chown_darwin.go @@ -83,7 +83,7 @@ func (c *platformChowner) LChown(path string, info os.FileInfo, toHost, toContai } if uid != int(st.Uid) || gid != int(st.Gid) { capability, err := system.Lgetxattr(path, "security.capability") - if err != nil && !errors.Is(err, system.EOPNOTSUPP) && err != system.ErrNotSupportedPlatform { + if err != nil && !errors.Is(err, system.ENOTSUP) && err != system.ErrNotSupportedPlatform { return fmt.Errorf("%s: %w", os.Args[0], err) } diff --git a/vendor/github.com/containers/storage/drivers/chown_unix.go b/vendor/github.com/containers/storage/drivers/chown_unix.go index 808f4fea..b0c25cd9 100644 --- a/vendor/github.com/containers/storage/drivers/chown_unix.go +++ b/vendor/github.com/containers/storage/drivers/chown_unix.go @@ -101,7 +101,7 @@ func (c *platformChowner) LChown(path string, info os.FileInfo, toHost, toContai } if uid != int(st.Uid) || gid != int(st.Gid) { cap, err := system.Lgetxattr(path, "security.capability") - if err != nil && !errors.Is(err, system.EOPNOTSUPP) && !errors.Is(err, system.EOVERFLOW) && err != system.ErrNotSupportedPlatform { + if err != nil && !errors.Is(err, system.ENOTSUP) && !errors.Is(err, system.EOVERFLOW) && err != system.ErrNotSupportedPlatform { return fmt.Errorf("%s: %w", os.Args[0], err) } diff --git a/vendor/github.com/containers/storage/drivers/copy/copy_linux.go b/vendor/github.com/containers/storage/drivers/copy/copy_linux.go index 8c0bbed6..93fc0a32 100644 --- a/vendor/github.com/containers/storage/drivers/copy/copy_linux.go +++ b/vendor/github.com/containers/storage/drivers/copy/copy_linux.go @@ -106,7 +106,7 @@ func legacyCopy(srcFile io.Reader, dstFile io.Writer) error { func copyXattr(srcPath, dstPath, attr string) error { data, err := system.Lgetxattr(srcPath, attr) - if err != nil && !errors.Is(err, unix.EOPNOTSUPP) { + if err != nil && !errors.Is(err, system.ENOTSUP) { return err } if data != nil { @@ -279,7 +279,7 @@ func doCopyXattrs(srcPath, dstPath string) error { } xattrs, err := system.Llistxattr(srcPath) - if err != nil && !errors.Is(err, unix.EOPNOTSUPP) { + if err != nil && !errors.Is(err, system.ENOTSUP) { return err } diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index dc9b12c3..1f7ac5ff 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" + "github.com/containers/storage/internal/dedup" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/fileutils" @@ -81,6 +82,23 @@ type ApplyDiffWithDifferOpts struct { Flags map[string]interface{} } +// DedupArgs contains the information to perform storage deduplication. +type DedupArgs struct { + // Layers is the list of layers to deduplicate. + Layers []string + + // Options that are passed directly to the pkg/dedup.DedupDirs function. + Options dedup.DedupOptions +} + +// DedupResult contains the result of the Dedup() call. +type DedupResult struct { + // Deduped represents the total number of bytes saved by deduplication. + // This value accounts also for all previously deduplicated data, not only the savings + // from the last run. + Deduped uint64 +} + // InitFunc initializes the storage driver. type InitFunc func(homedir string, options Options) (Driver, error) @@ -139,6 +157,8 @@ type ProtoDriver interface { // AdditionalImageStores returns additional image stores supported by the driver // This API is experimental and can be changed without bumping the major version number. AdditionalImageStores() []string + // Dedup performs deduplication of the driver's storage. + Dedup(DedupArgs) (DedupResult, error) } // DiffDriver is the interface to use to implement graph diffs @@ -211,8 +231,8 @@ const ( // DifferOutputFormatDir means the output is a directory and it will // keep the original layout. DifferOutputFormatDir = iota - // DifferOutputFormatFlat will store the files by their checksum, in the form - // checksum[0:2]/checksum[2:] + // DifferOutputFormatFlat will store the files by their checksum, per + // pkg/chunked/internal/composefs.RegularFilePathForValidatedDigest. DifferOutputFormatFlat ) diff --git a/vendor/github.com/containers/storage/drivers/overlay/check_116.go b/vendor/github.com/containers/storage/drivers/overlay/check_116.go index 7867d500..5cbf5e1c 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/check_116.go +++ b/vendor/github.com/containers/storage/drivers/overlay/check_116.go @@ -10,7 +10,6 @@ import ( "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/system" - "golang.org/x/sys/unix" ) func scanForMountProgramIndicators(home string) (detected bool, err error) { @@ -28,7 +27,7 @@ func scanForMountProgramIndicators(home string) (detected bool, err error) { } if d.IsDir() { xattrs, err := system.Llistxattr(path) - if err != nil && !errors.Is(err, unix.EOPNOTSUPP) { + if err != nil && !errors.Is(err, system.ENOTSUP) { return err } for _, xattr := range xattrs { diff --git a/vendor/github.com/containers/storage/drivers/overlay/composefs.go b/vendor/github.com/containers/storage/drivers/overlay/composefs.go index 1fb3e62a..797e3646 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/composefs.go +++ b/vendor/github.com/containers/storage/drivers/overlay/composefs.go @@ -1,4 +1,4 @@ -//go:build linux && cgo +//go:build linux package overlay @@ -27,7 +27,7 @@ var ( composeFsHelperErr error // skipMountViaFile is used to avoid trying to mount EROFS directly via the file if we already know the current kernel - // does not support it. Mounting directly via a file will be supported in kernel 6.12. + // does not support it. Mounting directly via a file is supported from Linux 6.12. skipMountViaFile atomic.Bool ) diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index caf89646..56278805 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -22,6 +22,7 @@ import ( graphdriver "github.com/containers/storage/drivers" "github.com/containers/storage/drivers/overlayutils" "github.com/containers/storage/drivers/quota" + "github.com/containers/storage/internal/dedup" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/directory" @@ -1096,6 +1097,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, readOnl } if d.options.forceMask != nil { + st.Mode |= os.ModeDir if err := idtools.SetContainersOverrideXattr(diff, st); err != nil { return err } @@ -2740,3 +2742,22 @@ func getMappedMountRoot(path string) string { } return dirName } + +// Dedup performs deduplication of the driver's storage. +func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) { + var dirs []string + for _, layer := range req.Layers { + dir, _, inAdditionalStore := d.dir2(layer, false) + if inAdditionalStore { + continue + } + if err := fileutils.Exists(dir); err == nil { + dirs = append(dirs, filepath.Join(dir, "diff")) + } + } + r, err := dedup.DedupDirs(dirs, req.Options) + if err != nil { + return graphdriver.DedupResult{}, err + } + return graphdriver.DedupResult{Deduped: r.Deduped}, nil +} diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go b/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go deleted file mode 100644 index a6df21d0..00000000 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay_nocgo.go +++ /dev/null @@ -1,23 +0,0 @@ -//go:build linux && !cgo - -package overlay - -import ( - "fmt" -) - -func openComposefsMount(dataDir string) (int, error) { - return 0, fmt.Errorf("composefs not supported on this build") -} - -func getComposeFsHelper() (string, error) { - return "", fmt.Errorf("composefs not supported on this build") -} - -func mountComposefsBlob(dataDir, mountPoint string) error { - return fmt.Errorf("composefs not supported on this build") -} - -func generateComposeFsBlob(verityDigests map[string]string, toc interface{}, composefsDir string) error { - return fmt.Errorf("composefs not supported on this build") -} diff --git a/vendor/github.com/containers/storage/drivers/register/register_overlay.go b/vendor/github.com/containers/storage/drivers/register/register_overlay.go index 6e82a1ea..c4f96282 100644 --- a/vendor/github.com/containers/storage/drivers/register/register_overlay.go +++ b/vendor/github.com/containers/storage/drivers/register/register_overlay.go @@ -1,4 +1,4 @@ -//go:build !exclude_graphdriver_overlay && linux && cgo +//go:build !exclude_graphdriver_overlay && linux package register diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index f60ec17b..98dc55b0 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -10,6 +10,7 @@ import ( "strings" graphdriver "github.com/containers/storage/drivers" + "github.com/containers/storage/internal/dedup" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/fileutils" @@ -348,3 +349,19 @@ func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string, func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent string, parentMappings *idtools.IDMappings, mountLabel string) (size int64, err error) { return d.naiveDiff.DiffSize(id, idMappings, parent, parentMappings, mountLabel) } + +// Dedup performs deduplication of the driver's storage. +func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) { + var dirs []string + for _, layer := range req.Layers { + dir := d.dir2(layer, false) + if err := fileutils.Exists(dir); err == nil { + dirs = append(dirs, dir) + } + } + r, err := dedup.DedupDirs(dirs, req.Options) + if err != nil { + return graphdriver.DedupResult{}, err + } + return graphdriver.DedupResult{Deduped: r.Deduped}, nil +} diff --git a/vendor/github.com/containers/storage/drivers/windows/windows.go b/vendor/github.com/containers/storage/drivers/windows/windows.go index d38e7453..59ed9a75 100644 --- a/vendor/github.com/containers/storage/drivers/windows/windows.go +++ b/vendor/github.com/containers/storage/drivers/windows/windows.go @@ -975,6 +975,11 @@ func (d *Driver) AdditionalImageStores() []string { return nil } +// Dedup performs deduplication of the driver's storage. +func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) { + return graphdriver.DedupResult{}, nil +} + // UpdateLayerIDMap changes ownerships in the layer's filesystem tree from // matching those in toContainer to matching those in toHost. func (d *Driver) UpdateLayerIDMap(id string, toContainer, toHost *idtools.IDMappings, mountLabel string) error { diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs.go b/vendor/github.com/containers/storage/drivers/zfs/zfs.go index ef26c6c7..f53b0e1b 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs.go @@ -511,3 +511,8 @@ func (d *Driver) ListLayers() ([]string, error) { func (d *Driver) AdditionalImageStores() []string { return nil } + +// Dedup performs deduplication of the driver's storage. +func (d *Driver) Dedup(req graphdriver.DedupArgs) (graphdriver.DedupResult, error) { + return graphdriver.DedupResult{}, nil +} diff --git a/vendor/github.com/containers/storage/internal/dedup/dedup.go b/vendor/github.com/containers/storage/internal/dedup/dedup.go new file mode 100644 index 00000000..59fcd0d2 --- /dev/null +++ b/vendor/github.com/containers/storage/internal/dedup/dedup.go @@ -0,0 +1,163 @@ +package dedup + +import ( + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "hash/crc64" + "io/fs" + "sync" + + "github.com/opencontainers/selinux/pkg/pwalkdir" + "github.com/sirupsen/logrus" +) + +var notSupported = errors.New("reflinks are not supported on this platform") + +const ( + DedupHashInvalid DedupHashMethod = iota + DedupHashCRC + DedupHashFileSize + DedupHashSHA256 +) + +type DedupHashMethod int + +type DedupOptions struct { + // HashMethod is the hash function to use to find identical files + HashMethod DedupHashMethod +} + +type DedupResult struct { + // Deduped represents the total number of bytes saved by deduplication. + // This value accounts also for all previously deduplicated data, not only the savings + // from the last run. + Deduped uint64 +} + +func getFileChecksum(hashMethod DedupHashMethod, path string, info fs.FileInfo) (string, error) { + switch hashMethod { + case DedupHashInvalid: + return "", fmt.Errorf("invalid hash method: %v", hashMethod) + case DedupHashFileSize: + return fmt.Sprintf("%v", info.Size()), nil + case DedupHashSHA256: + return readAllFile(path, info, func(buf []byte) (string, error) { + h := sha256.New() + if _, err := h.Write(buf); err != nil { + return "", err + } + return string(h.Sum(nil)), nil + }) + case DedupHashCRC: + return readAllFile(path, info, func(buf []byte) (string, error) { + c := crc64.New(crc64.MakeTable(crc64.ECMA)) + if _, err := c.Write(buf); err != nil { + return "", err + } + bufRet := make([]byte, 8) + binary.BigEndian.PutUint64(bufRet, c.Sum64()) + return string(bufRet), nil + }) + default: + return "", fmt.Errorf("unknown hash method: %v", hashMethod) + } +} + +type pathsLocked struct { + paths []string + lock sync.Mutex +} + +func DedupDirs(dirs []string, options DedupOptions) (DedupResult, error) { + res := DedupResult{} + hashToPaths := make(map[string]*pathsLocked) + lock := sync.Mutex{} // protects `hashToPaths` and `res` + + dedup, err := newDedupFiles() + if err != nil { + return res, err + } + + for _, dir := range dirs { + logrus.Debugf("Deduping directory %s", dir) + if err := pwalkdir.Walk(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.Type().IsRegular() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + size := uint64(info.Size()) + if size == 0 { + // do not bother with empty files + return nil + } + + // the file was already deduplicated + if visited, err := dedup.isFirstVisitOf(info); err != nil { + return err + } else if visited { + return nil + } + + h, err := getFileChecksum(options.HashMethod, path, info) + if err != nil { + return err + } + + lock.Lock() + item, foundItem := hashToPaths[h] + if !foundItem { + item = &pathsLocked{paths: []string{path}} + hashToPaths[h] = item + lock.Unlock() + return nil + } + item.lock.Lock() + lock.Unlock() + + dedupBytes, err := func() (uint64, error) { // function to have a scope for the defer statement + defer item.lock.Unlock() + + var dedupBytes uint64 + for _, src := range item.paths { + deduped, err := dedup.dedup(src, path, info) + if err == nil && deduped > 0 { + logrus.Debugf("Deduped %q -> %q (%d bytes)", src, path, deduped) + dedupBytes += deduped + break + } + logrus.Debugf("Failed to deduplicate: %v", err) + if errors.Is(err, notSupported) { + return dedupBytes, err + } + } + if dedupBytes == 0 { + item.paths = append(item.paths, path) + } + return dedupBytes, nil + }() + if err != nil { + return err + } + + lock.Lock() + res.Deduped += dedupBytes + lock.Unlock() + return nil + }); err != nil { + // if reflinks are not supported, return immediately without errors + if errors.Is(err, notSupported) { + return res, nil + } + return res, err + } + } + return res, nil +} diff --git a/vendor/github.com/containers/storage/internal/dedup/dedup_linux.go b/vendor/github.com/containers/storage/internal/dedup/dedup_linux.go new file mode 100644 index 00000000..90ccb5f3 --- /dev/null +++ b/vendor/github.com/containers/storage/internal/dedup/dedup_linux.go @@ -0,0 +1,139 @@ +package dedup + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "sync" + "syscall" + + "golang.org/x/sys/unix" +) + +type deviceInodePair struct { + dev uint64 + ino uint64 +} + +type dedupFiles struct { + lock sync.Mutex + visitedInodes map[deviceInodePair]struct{} +} + +func newDedupFiles() (*dedupFiles, error) { + return &dedupFiles{ + visitedInodes: make(map[deviceInodePair]struct{}), + }, nil +} + +func (d *dedupFiles) recordInode(dev, ino uint64) (bool, error) { + d.lock.Lock() + defer d.lock.Unlock() + + di := deviceInodePair{ + dev: dev, + ino: ino, + } + + _, visited := d.visitedInodes[di] + d.visitedInodes[di] = struct{}{} + return visited, nil +} + +// isFirstVisitOf records that the file is being processed. Returns true if the file was already visited. +func (d *dedupFiles) isFirstVisitOf(fi fs.FileInfo) (bool, error) { + st, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return false, fmt.Errorf("unable to get raw syscall.Stat_t data") + } + return d.recordInode(uint64(st.Dev), st.Ino) +} + +// dedup deduplicates the file at src path to dst path +func (d *dedupFiles) dedup(src, dst string, fiDst fs.FileInfo) (uint64, error) { + srcFile, err := os.OpenFile(src, os.O_RDONLY, 0) + if err != nil { + return 0, fmt.Errorf("failed to open source file: %w", err) + } + defer srcFile.Close() + + dstFile, err := os.OpenFile(dst, os.O_WRONLY, 0) + if err != nil { + return 0, fmt.Errorf("failed to open destination file: %w", err) + } + defer dstFile.Close() + + stSrc, err := srcFile.Stat() + if err != nil { + return 0, fmt.Errorf("failed to stat source file: %w", err) + } + sSrc, ok := stSrc.Sys().(*syscall.Stat_t) + if !ok { + return 0, fmt.Errorf("unable to get raw syscall.Stat_t data") + } + sDest, ok := fiDst.Sys().(*syscall.Stat_t) + if !ok { + return 0, fmt.Errorf("unable to get raw syscall.Stat_t data") + } + if sSrc.Dev == sDest.Dev && sSrc.Ino == sDest.Ino { + // same inode, we are dealing with a hard link, no need to deduplicate + return 0, nil + } + + value := unix.FileDedupeRange{ + Src_offset: 0, + Src_length: uint64(stSrc.Size()), + Info: []unix.FileDedupeRangeInfo{ + { + Dest_fd: int64(dstFile.Fd()), + Dest_offset: 0, + }, + }, + } + err = unix.IoctlFileDedupeRange(int(srcFile.Fd()), &value) + if err == nil { + return uint64(value.Info[0].Bytes_deduped), nil + } + + if errors.Is(err, unix.ENOTSUP) { + return 0, notSupported + } + return 0, fmt.Errorf("failed to clone file %q: %w", src, err) +} + +func readAllFile(path string, info fs.FileInfo, fn func([]byte) (string, error)) (string, error) { + size := info.Size() + if size == 0 { + return fn(nil) + } + + file, err := os.Open(path) + if err != nil { + return "", err + } + defer file.Close() + + if size < 4096 { + // small file, read it all + data := make([]byte, size) + _, err = io.ReadFull(file, data) + if err != nil { + return "", err + } + return fn(data) + } + + mmap, err := unix.Mmap(int(file.Fd()), 0, int(size), unix.PROT_READ, unix.MAP_PRIVATE) + if err != nil { + return "", fmt.Errorf("failed to mmap file: %w", err) + } + defer func() { + _ = unix.Munmap(mmap) + }() + + _ = unix.Madvise(mmap, unix.MADV_SEQUENTIAL) + + return fn(mmap) +} diff --git a/vendor/github.com/containers/storage/internal/dedup/dedup_unsupported.go b/vendor/github.com/containers/storage/internal/dedup/dedup_unsupported.go new file mode 100644 index 00000000..cfadf832 --- /dev/null +++ b/vendor/github.com/containers/storage/internal/dedup/dedup_unsupported.go @@ -0,0 +1,27 @@ +//go:build !linux + +package dedup + +import ( + "io/fs" +) + +type dedupFiles struct{} + +func newDedupFiles() (*dedupFiles, error) { + return nil, notSupported +} + +// isFirstVisitOf records that the file is being processed. Returns true if the file was already visited. +func (d *dedupFiles) isFirstVisitOf(fi fs.FileInfo) (bool, error) { + return false, notSupported +} + +// dedup deduplicates the file at src path to dst path +func (d *dedupFiles) dedup(src, dst string, fiDst fs.FileInfo) (uint64, error) { + return 0, notSupported +} + +func readAllFile(path string, info fs.FileInfo, fn func([]byte) (string, error)) (string, error) { + return "", notSupported +} diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 6fe1a080..1f8203fb 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -336,6 +336,9 @@ type rwLayerStore interface { // Clean up unreferenced layers GarbageCollect() error + + // Dedup deduplicates layers in the store. + dedup(drivers.DedupArgs) (drivers.DedupResult, error) } type multipleLockFile struct { @@ -913,23 +916,32 @@ func (r *layerStore) load(lockedForWriting bool) (bool, error) { // user of this storage area marked for deletion but didn't manage to // actually delete. var incompleteDeletionErrors error // = nil + var layersToDelete []*Layer for _, layer := range r.layers { if layer.Flags == nil { layer.Flags = make(map[string]interface{}) } if layerHasIncompleteFlag(layer) { - logrus.Warnf("Found incomplete layer %#v, deleting it", layer.ID) - err := r.deleteInternal(layer.ID) - if err != nil { - // Don't return the error immediately, because deleteInternal does not saveLayers(); - // Even if deleting one incomplete layer fails, call saveLayers() so that other possible successfully - // deleted incomplete layers have their metadata correctly removed. - incompleteDeletionErrors = multierror.Append(incompleteDeletionErrors, - fmt.Errorf("deleting layer %#v: %w", layer.ID, err)) - } - modifiedLocations |= layerLocation(layer) + // Important: Do not call r.deleteInternal() here. It modifies r.layers + // which causes unexpected side effects while iterating over r.layers here. + // The range loop has no idea that the underlying elements where shifted + // around. + layersToDelete = append(layersToDelete, layer) } } + // Now actually delete the layers + for _, layer := range layersToDelete { + logrus.Warnf("Found incomplete layer %q, deleting it", layer.ID) + err := r.deleteInternal(layer.ID) + if err != nil { + // Don't return the error immediately, because deleteInternal does not saveLayers(); + // Even if deleting one incomplete layer fails, call saveLayers() so that other possible successfully + // deleted incomplete layers have their metadata correctly removed. + incompleteDeletionErrors = multierror.Append(incompleteDeletionErrors, + fmt.Errorf("deleting layer %#v: %w", layer.ID, err)) + } + modifiedLocations |= layerLocation(layer) + } if err := r.saveLayers(modifiedLocations); err != nil { return false, err } @@ -2592,6 +2604,11 @@ func (r *layerStore) LayersByTOCDigest(d digest.Digest) ([]Layer, error) { return r.layersByDigestMap(r.bytocsum, d) } +// Requires startWriting. +func (r *layerStore) dedup(req drivers.DedupArgs) (drivers.DedupResult, error) { + return r.driver.Dedup(req) +} + func closeAll(closes ...func() error) (rErr error) { for _, f := range closes { if err := f(); err != nil { diff --git a/vendor/github.com/containers/storage/pkg/archive/archive.go b/vendor/github.com/containers/storage/pkg/archive/archive.go index 13a45895..41daad85 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive.go @@ -78,6 +78,7 @@ const ( windows = "windows" darwin = "darwin" freebsd = "freebsd" + linux = "linux" ) var xattrsToIgnore = map[string]interface{}{ @@ -427,7 +428,7 @@ func readSecurityXattrToTarHeader(path string, hdr *tar.Header) error { } for _, xattr := range []string{"security.capability", "security.ima"} { capability, err := system.Lgetxattr(path, xattr) - if err != nil && !errors.Is(err, system.EOPNOTSUPP) && err != system.ErrNotSupportedPlatform { + if err != nil && !errors.Is(err, system.ENOTSUP) && err != system.ErrNotSupportedPlatform { return fmt.Errorf("failed to read %q attribute from %q: %w", xattr, path, err) } if capability != nil { @@ -440,7 +441,7 @@ func readSecurityXattrToTarHeader(path string, hdr *tar.Header) error { // readUserXattrToTarHeader reads user.* xattr from filesystem to a tar header func readUserXattrToTarHeader(path string, hdr *tar.Header) error { xattrs, err := system.Llistxattr(path) - if err != nil && !errors.Is(err, system.EOPNOTSUPP) && err != system.ErrNotSupportedPlatform { + if err != nil && !errors.Is(err, system.ENOTSUP) && err != system.ErrNotSupportedPlatform { return err } for _, key := range xattrs { @@ -655,12 +656,20 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L // so use hdrInfo.Mode() (they differ for e.g. setuid bits) hdrInfo := hdr.FileInfo() + typeFlag := hdr.Typeflag mask := hdrInfo.Mode() + + // update also the implementation of ForceMask in pkg/chunked if forceMask != nil { mask = *forceMask + // If we have a forceMask, force the real type to either be a directory, + // a link, or a regular file. + if typeFlag != tar.TypeDir && typeFlag != tar.TypeSymlink && typeFlag != tar.TypeLink { + typeFlag = tar.TypeReg + } } - switch hdr.Typeflag { + switch typeFlag { case tar.TypeDir: // Create directory unless it exists as a directory already. // In that case we just want to merge the two @@ -728,16 +737,6 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag) } - if forceMask != nil && (hdr.Typeflag != tar.TypeSymlink || runtime.GOOS == "darwin") { - value := idtools.Stat{ - IDs: idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}, - Mode: hdrInfo.Mode() & 0o7777, - } - if err := idtools.SetContainersOverrideXattr(path, value); err != nil { - return err - } - } - // Lchown is not supported on Windows. if Lchown && runtime.GOOS != windows { if chownOpts == nil { @@ -793,18 +792,30 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L continue } if err := system.Lsetxattr(path, xattrKey, []byte(value), 0); err != nil { - if errors.Is(err, syscall.ENOTSUP) || (inUserns && errors.Is(err, syscall.EPERM)) { - // We ignore errors here because not all graphdrivers support - // xattrs *cough* old versions of AUFS *cough*. However only - // ENOTSUP should be emitted in that case, otherwise we still - // bail. We also ignore EPERM errors if we are running in a - // user namespace. + if errors.Is(err, system.ENOTSUP) || (inUserns && errors.Is(err, syscall.EPERM)) { + // Ignore specific error cases: + // - ENOTSUP: Expected for graphdrivers lacking extended attribute support: + // - Legacy AUFS versions + // - FreeBSD with unsupported namespaces (trusted, security) + // - EPERM: Expected when operating within a user namespace + // All other errors will cause a failure. errs = append(errs, err.Error()) continue } return err } + } + if forceMask != nil && (typeFlag == tar.TypeReg || typeFlag == tar.TypeDir || runtime.GOOS == "darwin") { + value := idtools.Stat{ + IDs: idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}, + Mode: hdrInfo.Mode(), + Major: int(hdr.Devmajor), + Minor: int(hdr.Devminor), + } + if err := idtools.SetContainersOverrideXattr(path, value); err != nil { + return err + } } // We defer setting flags on directories until the end of @@ -1149,11 +1160,11 @@ loop: } if options.ForceMask != nil { - value := idtools.Stat{Mode: 0o755} + value := idtools.Stat{Mode: os.ModeDir | os.FileMode(0o755)} if rootHdr != nil { value.IDs.UID = rootHdr.Uid value.IDs.GID = rootHdr.Gid - value.Mode = os.FileMode(rootHdr.Mode) + value.Mode = os.ModeDir | os.FileMode(rootHdr.Mode) } if err := idtools.SetContainersOverrideXattr(dest, value); err != nil { return err @@ -1379,7 +1390,7 @@ func remapIDs(readIDMappings, writeIDMappings *idtools.IDMappings, chownOpts *id uid, gid = hdr.Uid, hdr.Gid if xstat, ok := hdr.PAXRecords[PaxSchilyXattr+idtools.ContainersOverrideXattr]; ok { attrs := strings.Split(string(xstat), ":") - if len(attrs) == 3 { + if len(attrs) >= 3 { val, err := strconv.ParseUint(attrs[0], 10, 32) if err != nil { uid = int(val) diff --git a/vendor/github.com/containers/storage/pkg/archive/changes.go b/vendor/github.com/containers/storage/pkg/archive/changes.go index 3075c27b..2b526549 100644 --- a/vendor/github.com/containers/storage/pkg/archive/changes.go +++ b/vendor/github.com/containers/storage/pkg/archive/changes.go @@ -270,6 +270,7 @@ type FileInfo struct { capability []byte added bool xattrs map[string]string + target string } // LookUp looks up the file information of a file. @@ -336,6 +337,7 @@ func (info *FileInfo) addChanges(oldInfo *FileInfo, changes *[]Change) { // back mtime if statDifferent(oldStat, oldInfo, newStat, info) || !bytes.Equal(oldChild.capability, newChild.capability) || + oldChild.target != newChild.target || !reflect.DeepEqual(oldChild.xattrs, newChild.xattrs) { change := Change{ Path: newChild.path(), @@ -390,6 +392,7 @@ func newRootFileInfo(idMappings *idtools.IDMappings) *FileInfo { name: string(os.PathSeparator), idMappings: idMappings, children: make(map[string]*FileInfo), + target: "", } return root } diff --git a/vendor/github.com/containers/storage/pkg/archive/changes_linux.go b/vendor/github.com/containers/storage/pkg/archive/changes_linux.go index dc308120..42e77c4d 100644 --- a/vendor/github.com/containers/storage/pkg/archive/changes_linux.go +++ b/vendor/github.com/containers/storage/pkg/archive/changes_linux.go @@ -79,6 +79,7 @@ func walkchunk(path string, fi os.FileInfo, dir string, root *FileInfo) error { children: make(map[string]*FileInfo), parent: parent, idMappings: root.idMappings, + target: "", } cpath := filepath.Join(dir, path) stat, err := system.FromStatT(fi.Sys().(*syscall.Stat_t)) @@ -87,11 +88,11 @@ func walkchunk(path string, fi os.FileInfo, dir string, root *FileInfo) error { } info.stat = stat info.capability, err = system.Lgetxattr(cpath, "security.capability") // lgetxattr(2): fs access - if err != nil && !errors.Is(err, system.EOPNOTSUPP) { + if err != nil && !errors.Is(err, system.ENOTSUP) { return err } xattrs, err := system.Llistxattr(cpath) - if err != nil && !errors.Is(err, system.EOPNOTSUPP) { + if err != nil && !errors.Is(err, system.ENOTSUP) { return err } for _, key := range xattrs { @@ -110,6 +111,12 @@ func walkchunk(path string, fi os.FileInfo, dir string, root *FileInfo) error { info.xattrs[key] = string(value) } } + if fi.Mode()&os.ModeSymlink != 0 { + info.target, err = os.Readlink(cpath) + if err != nil { + return err + } + } parent.children[info.name] = info return nil } diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go index 710167d2..5a161781 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go @@ -47,7 +47,7 @@ func Untar(tarArchive io.Reader, dest string, options *archive.TarOptions) error // This should be used to prevent a potential attacker from manipulating `dest` // such that it would provide access to files outside of `dest` through things // like symlinks. Normally `ResolveSymlinksInScope` would handle this, however -// sanitizing symlinks in this manner is inherrently racey: +// sanitizing symlinks in this manner is inherently racey: // ref: CVE-2018-15664 func UntarWithRoot(tarArchive io.Reader, dest string, options *archive.TarOptions, root string) error { return untarHandler(tarArchive, dest, options, true, root) diff --git a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go index 47742b05..0e49ddd8 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go @@ -16,7 +16,7 @@ import ( storage "github.com/containers/storage" graphdriver "github.com/containers/storage/drivers" - "github.com/containers/storage/pkg/chunked/internal" + "github.com/containers/storage/pkg/chunked/internal/minimal" "github.com/containers/storage/pkg/ioutils" "github.com/docker/go-units" jsoniter "github.com/json-iterator/go" @@ -710,7 +710,7 @@ func prepareCacheFile(manifest []byte, format graphdriver.DifferOutputFormat) ([ switch format { case graphdriver.DifferOutputFormatDir: case graphdriver.DifferOutputFormatFlat: - entries, err = makeEntriesFlat(entries) + entries, err = makeEntriesFlat(entries, nil) if err != nil { return nil, err } @@ -848,12 +848,12 @@ func (c *layersCache) findFileInOtherLayers(file *fileMetadata, useHardLinks boo return "", "", nil } -func (c *layersCache) findChunkInOtherLayers(chunk *internal.FileMetadata) (string, string, int64, error) { +func (c *layersCache) findChunkInOtherLayers(chunk *minimal.FileMetadata) (string, string, int64, error) { return c.findDigestInternal(chunk.ChunkDigest) } -func unmarshalToc(manifest []byte) (*internal.TOC, error) { - var toc internal.TOC +func unmarshalToc(manifest []byte) (*minimal.TOC, error) { + var toc minimal.TOC iter := jsoniter.ParseBytes(jsoniter.ConfigFastest, manifest) @@ -864,7 +864,7 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) { case "entries": for iter.ReadArray() { - var m internal.FileMetadata + var m minimal.FileMetadata for field := iter.ReadObject(); field != ""; field = iter.ReadObject() { switch strings.ToLower(field) { case "type": diff --git a/vendor/github.com/containers/storage/pkg/chunked/compression.go b/vendor/github.com/containers/storage/pkg/chunked/compression.go index e828d479..564efc8b 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/compression.go +++ b/vendor/github.com/containers/storage/pkg/chunked/compression.go @@ -4,18 +4,18 @@ import ( "io" "github.com/containers/storage/pkg/chunked/compressor" - "github.com/containers/storage/pkg/chunked/internal" + "github.com/containers/storage/pkg/chunked/internal/minimal" ) const ( - TypeReg = internal.TypeReg - TypeChunk = internal.TypeChunk - TypeLink = internal.TypeLink - TypeChar = internal.TypeChar - TypeBlock = internal.TypeBlock - TypeDir = internal.TypeDir - TypeFifo = internal.TypeFifo - TypeSymlink = internal.TypeSymlink + TypeReg = minimal.TypeReg + TypeChunk = minimal.TypeChunk + TypeLink = minimal.TypeLink + TypeChar = minimal.TypeChar + TypeBlock = minimal.TypeBlock + TypeDir = minimal.TypeDir + TypeFifo = minimal.TypeFifo + TypeSymlink = minimal.TypeSymlink ) // ZstdCompressor is a CompressorFunc for the zstd compression algorithm. diff --git a/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go b/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go index 2dac4635..67cc6cf0 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/compression_linux.go @@ -10,7 +10,7 @@ import ( "strconv" "time" - "github.com/containers/storage/pkg/chunked/internal" + "github.com/containers/storage/pkg/chunked/internal/minimal" "github.com/klauspost/compress/zstd" "github.com/klauspost/pgzip" digest "github.com/opencontainers/go-digest" @@ -20,6 +20,12 @@ import ( expMaps "golang.org/x/exp/maps" ) +const ( + // maxTocSize is the maximum size of a blob that we will attempt to process. + // It is used to prevent DoS attacks from layers that embed a very large TOC file. + maxTocSize = (1 << 20) * 150 +) + var typesToTar = map[string]byte{ TypeReg: tar.TypeReg, TypeLink: tar.TypeLink, @@ -38,31 +44,33 @@ func typeToTarType(t string) (byte, error) { return r, nil } +// readEstargzChunkedManifest reads the estargz manifest from the seekable stream blobStream. +// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert. func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, tocDigest digest.Digest) ([]byte, int64, error) { // information on the format here https://github.com/containerd/stargz-snapshotter/blob/main/docs/stargz-estargz.md footerSize := int64(51) if blobSize <= footerSize { return nil, 0, errors.New("blob too small") } - chunk := ImageSourceChunk{ - Offset: uint64(blobSize - footerSize), - Length: uint64(footerSize), - } - parts, errs, err := blobStream.GetBlobAt([]ImageSourceChunk{chunk}) + + footer := make([]byte, footerSize) + streamsOrErrors, err := getBlobAt(blobStream, ImageSourceChunk{Offset: uint64(blobSize - footerSize), Length: uint64(footerSize)}) if err != nil { + var badRequestErr ErrBadRequest + if errors.As(err, &badRequestErr) { + err = errFallbackCanConvert{newErrFallbackToOrdinaryLayerDownload(err)} + } return nil, 0, err } - var reader io.ReadCloser - select { - case r := <-parts: - reader = r - case err := <-errs: - return nil, 0, err - } - defer reader.Close() - footer := make([]byte, footerSize) - if _, err := io.ReadFull(reader, footer); err != nil { - return nil, 0, err + + for soe := range streamsOrErrors { + if soe.stream != nil { + _, err = io.ReadFull(soe.stream, footer) + _ = soe.stream.Close() + } + if soe.err != nil && err == nil { + err = soe.err + } } /* Read the ToC offset: @@ -81,48 +89,59 @@ func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, size := int64(blobSize - footerSize - tocOffset) // set a reasonable limit - if size > (1<<20)*50 { - return nil, 0, errors.New("manifest too big") - } - - chunk = ImageSourceChunk{ - Offset: uint64(tocOffset), - Length: uint64(size), - } - parts, errs, err = blobStream.GetBlobAt([]ImageSourceChunk{chunk}) - if err != nil { - return nil, 0, err - } - - var tocReader io.ReadCloser - select { - case r := <-parts: - tocReader = r - case err := <-errs: - return nil, 0, err + if size > maxTocSize { + // Not errFallbackCanConvert: we would still use too much memory. + return nil, 0, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("estargz manifest too big to process in memory (%d bytes)", size)) } - defer tocReader.Close() - r, err := pgzip.NewReader(tocReader) + streamsOrErrors, err = getBlobAt(blobStream, ImageSourceChunk{Offset: uint64(tocOffset), Length: uint64(size)}) if err != nil { + var badRequestErr ErrBadRequest + if errors.As(err, &badRequestErr) { + err = errFallbackCanConvert{newErrFallbackToOrdinaryLayerDownload(err)} + } return nil, 0, err } - defer r.Close() - - aTar := archivetar.NewReader(r) - header, err := aTar.Next() - if err != nil { - return nil, 0, err - } - // set a reasonable limit - if header.Size > (1<<20)*50 { - return nil, 0, errors.New("manifest too big") + var manifestUncompressed []byte + + for soe := range streamsOrErrors { + if soe.stream != nil { + err1 := func() error { + defer soe.stream.Close() + + r, err := pgzip.NewReader(soe.stream) + if err != nil { + return err + } + defer r.Close() + + aTar := archivetar.NewReader(r) + + header, err := aTar.Next() + if err != nil { + return err + } + // set a reasonable limit + if header.Size > maxTocSize { + return errors.New("manifest too big") + } + + manifestUncompressed = make([]byte, header.Size) + if _, err := io.ReadFull(aTar, manifestUncompressed); err != nil { + return err + } + return nil + }() + if err == nil { + err = err1 + } + } else if err == nil { + err = soe.err + } } - - manifestUncompressed := make([]byte, header.Size) - if _, err := io.ReadFull(aTar, manifestUncompressed); err != nil { - return nil, 0, err + if manifestUncompressed == nil { + return nil, 0, errors.New("manifest not found") } manifestDigester := digest.Canonical.Digester() @@ -140,10 +159,11 @@ func readEstargzChunkedManifest(blobStream ImageSourceSeekable, blobSize int64, // readZstdChunkedManifest reads the zstd:chunked manifest from the seekable stream blobStream. // Returns (manifest blob, parsed manifest, tar-split blob or nil, manifest offset). -func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Digest, annotations map[string]string) ([]byte, *internal.TOC, []byte, int64, error) { - offsetMetadata := annotations[internal.ManifestInfoKey] +// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert. +func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Digest, annotations map[string]string) (_ []byte, _ *minimal.TOC, _ []byte, _ int64, retErr error) { + offsetMetadata := annotations[minimal.ManifestInfoKey] if offsetMetadata == "" { - return nil, nil, nil, 0, fmt.Errorf("%q annotation missing", internal.ManifestInfoKey) + return nil, nil, nil, 0, fmt.Errorf("%q annotation missing", minimal.ManifestInfoKey) } var manifestChunk ImageSourceChunk var manifestLengthUncompressed, manifestType uint64 @@ -153,48 +173,59 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di // The tarSplit… values are valid if tarSplitChunk.Offset > 0 var tarSplitChunk ImageSourceChunk var tarSplitLengthUncompressed uint64 - if tarSplitInfoKeyAnnotation, found := annotations[internal.TarSplitInfoKey]; found { + if tarSplitInfoKeyAnnotation, found := annotations[minimal.TarSplitInfoKey]; found { if _, err := fmt.Sscanf(tarSplitInfoKeyAnnotation, "%d:%d:%d", &tarSplitChunk.Offset, &tarSplitChunk.Length, &tarSplitLengthUncompressed); err != nil { return nil, nil, nil, 0, err } } - if manifestType != internal.ManifestTypeCRFS { + if manifestType != minimal.ManifestTypeCRFS { return nil, nil, nil, 0, errors.New("invalid manifest type") } // set a reasonable limit - if manifestChunk.Length > (1<<20)*50 { - return nil, nil, nil, 0, errors.New("manifest too big") + if manifestChunk.Length > maxTocSize { + // Not errFallbackCanConvert: we would still use too much memory. + return nil, nil, nil, 0, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("zstd:chunked manifest too big to process in memory (%d bytes compressed)", manifestChunk.Length)) } - if manifestLengthUncompressed > (1<<20)*50 { - return nil, nil, nil, 0, errors.New("manifest too big") + if manifestLengthUncompressed > maxTocSize { + // Not errFallbackCanConvert: we would still use too much memory. + return nil, nil, nil, 0, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("zstd:chunked manifest too big to process in memory (%d bytes uncompressed)", manifestLengthUncompressed)) } chunks := []ImageSourceChunk{manifestChunk} if tarSplitChunk.Offset > 0 { chunks = append(chunks, tarSplitChunk) } - parts, errs, err := blobStream.GetBlobAt(chunks) + + streamsOrErrors, err := getBlobAt(blobStream, chunks...) if err != nil { + var badRequestErr ErrBadRequest + if errors.As(err, &badRequestErr) { + err = errFallbackCanConvert{newErrFallbackToOrdinaryLayerDownload(err)} + } return nil, nil, nil, 0, err } + defer func() { + err := ensureAllBlobsDone(streamsOrErrors) + if retErr == nil { + retErr = err + } + }() + readBlob := func(len uint64) ([]byte, error) { - var reader io.ReadCloser - select { - case r := <-parts: - reader = r - case err := <-errs: - return nil, err + soe, ok := <-streamsOrErrors + if !ok { + return nil, errors.New("stream closed") + } + if soe.err != nil { + return nil, soe.err } + defer soe.stream.Close() blob := make([]byte, len) - if _, err := io.ReadFull(reader, blob); err != nil { - reader.Close() - return nil, err - } - if err := reader.Close(); err != nil { + if _, err := io.ReadFull(soe.stream, blob); err != nil { return nil, err } return blob, nil @@ -217,7 +248,7 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di var decodedTarSplit []byte = nil if toc.TarSplitDigest != "" { if tarSplitChunk.Offset <= 0 { - return nil, nil, nil, 0, fmt.Errorf("TOC requires a tar-split, but the %s annotation does not describe a position", internal.TarSplitInfoKey) + return nil, nil, nil, 0, fmt.Errorf("TOC requires a tar-split, but the %s annotation does not describe a position", minimal.TarSplitInfoKey) } tarSplit, err := readBlob(tarSplitChunk.Length) if err != nil { @@ -247,11 +278,11 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di } // ensureTOCMatchesTarSplit validates that toc and tarSplit contain _exactly_ the same entries. -func ensureTOCMatchesTarSplit(toc *internal.TOC, tarSplit []byte) error { - pendingFiles := map[string]*internal.FileMetadata{} // Name -> an entry in toc.Entries +func ensureTOCMatchesTarSplit(toc *minimal.TOC, tarSplit []byte) error { + pendingFiles := map[string]*minimal.FileMetadata{} // Name -> an entry in toc.Entries for i := range toc.Entries { e := &toc.Entries[i] - if e.Type != internal.TypeChunk { + if e.Type != minimal.TypeChunk { if _, ok := pendingFiles[e.Name]; ok { return fmt.Errorf("TOC contains duplicate entries for path %q", e.Name) } @@ -266,7 +297,7 @@ func ensureTOCMatchesTarSplit(toc *internal.TOC, tarSplit []byte) error { return fmt.Errorf("tar-split contains an entry for %q missing in TOC", hdr.Name) } delete(pendingFiles, hdr.Name) - expected, err := internal.NewFileMetadata(hdr) + expected, err := minimal.NewFileMetadata(hdr) if err != nil { return fmt.Errorf("determining expected metadata for %q: %w", hdr.Name, err) } @@ -347,8 +378,8 @@ func ensureTimePointersMatch(a, b *time.Time) error { // ensureFileMetadataAttributesMatch ensures that a and b match in file attributes (it ignores entries relevant to locating data // in the tar stream or matching contents) -func ensureFileMetadataAttributesMatch(a, b *internal.FileMetadata) error { - // Keep this in sync with internal.FileMetadata! +func ensureFileMetadataAttributesMatch(a, b *minimal.FileMetadata) error { + // Keep this in sync with minimal.FileMetadata! if a.Type != b.Type { return fmt.Errorf("mismatch of Type: %q != %q", a.Type, b.Type) diff --git a/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go b/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go index 65496974..56ae4c77 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go +++ b/vendor/github.com/containers/storage/pkg/chunked/compressor/compressor.go @@ -9,7 +9,7 @@ import ( "bytes" "io" - "github.com/containers/storage/pkg/chunked/internal" + "github.com/containers/storage/pkg/chunked/internal/minimal" "github.com/containers/storage/pkg/ioutils" "github.com/klauspost/compress/zstd" "github.com/opencontainers/go-digest" @@ -213,7 +213,7 @@ func newTarSplitData(level int) (*tarSplitData, error) { compressed := bytes.NewBuffer(nil) digester := digest.Canonical.Digester() - zstdWriter, err := internal.ZstdWriterWithLevel(io.MultiWriter(compressed, digester.Hash()), level) + zstdWriter, err := minimal.ZstdWriterWithLevel(io.MultiWriter(compressed, digester.Hash()), level) if err != nil { return nil, err } @@ -254,7 +254,7 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r buf := make([]byte, 4096) - zstdWriter, err := internal.ZstdWriterWithLevel(dest, level) + zstdWriter, err := minimal.ZstdWriterWithLevel(dest, level) if err != nil { return err } @@ -276,7 +276,7 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r return offset, nil } - var metadata []internal.FileMetadata + var metadata []minimal.FileMetadata for { hdr, err := tr.Next() if err != nil { @@ -341,9 +341,9 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r chunkSize := rcReader.WrittenOut - lastChunkOffset if chunkSize > 0 { - chunkType := internal.ChunkTypeData + chunkType := minimal.ChunkTypeData if rcReader.IsLastChunkZeros { - chunkType = internal.ChunkTypeZeros + chunkType = minimal.ChunkTypeZeros } chunks = append(chunks, chunk{ @@ -368,17 +368,17 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r } } - mainEntry, err := internal.NewFileMetadata(hdr) + mainEntry, err := minimal.NewFileMetadata(hdr) if err != nil { return err } mainEntry.Digest = checksum mainEntry.Offset = startOffset mainEntry.EndOffset = lastOffset - entries := []internal.FileMetadata{mainEntry} + entries := []minimal.FileMetadata{mainEntry} for i := 1; i < len(chunks); i++ { - entries = append(entries, internal.FileMetadata{ - Type: internal.TypeChunk, + entries = append(entries, minimal.FileMetadata{ + Type: minimal.TypeChunk, Name: hdr.Name, ChunkOffset: chunks[i].ChunkOffset, }) @@ -424,13 +424,13 @@ func writeZstdChunkedStream(destFile io.Writer, outMetadata map[string]string, r } tarSplitData.zstd = nil - ts := internal.TarSplitData{ + ts := minimal.TarSplitData{ Data: tarSplitData.compressed.Bytes(), Digest: tarSplitData.digester.Digest(), UncompressedSize: tarSplitData.uncompressedCounter.Count, } - return internal.WriteZstdChunkedManifest(dest, outMetadata, uint64(dest.Count), &ts, metadata, level) + return minimal.WriteZstdChunkedManifest(dest, outMetadata, uint64(dest.Count), &ts, metadata, level) } type zstdChunkedWriter struct { diff --git a/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go b/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go index d98cee09..0e673f3f 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go +++ b/vendor/github.com/containers/storage/pkg/chunked/dump/dump.go @@ -9,10 +9,11 @@ import ( "io" "path/filepath" "reflect" - "strings" "time" - "github.com/containers/storage/pkg/chunked/internal" + "github.com/containers/storage/pkg/chunked/internal/minimal" + storagePath "github.com/containers/storage/pkg/chunked/internal/path" + "github.com/opencontainers/go-digest" "golang.org/x/sys/unix" ) @@ -85,17 +86,17 @@ func escapedOptional(val []byte, escape int) string { func getStMode(mode uint32, typ string) (uint32, error) { switch typ { - case internal.TypeReg, internal.TypeLink: + case minimal.TypeReg, minimal.TypeLink: mode |= unix.S_IFREG - case internal.TypeChar: + case minimal.TypeChar: mode |= unix.S_IFCHR - case internal.TypeBlock: + case minimal.TypeBlock: mode |= unix.S_IFBLK - case internal.TypeDir: + case minimal.TypeDir: mode |= unix.S_IFDIR - case internal.TypeFifo: + case minimal.TypeFifo: mode |= unix.S_IFIFO - case internal.TypeSymlink: + case minimal.TypeSymlink: mode |= unix.S_IFLNK default: return 0, fmt.Errorf("unknown type %s", typ) @@ -103,24 +104,14 @@ func getStMode(mode uint32, typ string) (uint32, error) { return mode, nil } -func sanitizeName(name string) string { - path := filepath.Clean(name) - if path == "." { - path = "/" - } else if path[0] != '/' { - path = "/" + path - } - return path -} - -func dumpNode(out io.Writer, added map[string]*internal.FileMetadata, links map[string]int, verityDigests map[string]string, entry *internal.FileMetadata) error { - path := sanitizeName(entry.Name) +func dumpNode(out io.Writer, added map[string]*minimal.FileMetadata, links map[string]int, verityDigests map[string]string, entry *minimal.FileMetadata) error { + path := storagePath.CleanAbsPath(entry.Name) parent := filepath.Dir(path) if _, found := added[parent]; !found && path != "/" { - parentEntry := &internal.FileMetadata{ + parentEntry := &minimal.FileMetadata{ Name: parent, - Type: internal.TypeDir, + Type: minimal.TypeDir, Mode: 0o755, } if err := dumpNode(out, added, links, verityDigests, parentEntry); err != nil { @@ -143,7 +134,7 @@ func dumpNode(out io.Writer, added map[string]*internal.FileMetadata, links map[ nlinks := links[entry.Name] + links[entry.Linkname] + 1 link := "" - if entry.Type == internal.TypeLink { + if entry.Type == minimal.TypeLink { link = "@" } @@ -169,16 +160,21 @@ func dumpNode(out io.Writer, added map[string]*internal.FileMetadata, links map[ var payload string if entry.Linkname != "" { - if entry.Type == internal.TypeSymlink { + if entry.Type == minimal.TypeSymlink { payload = entry.Linkname } else { - payload = sanitizeName(entry.Linkname) + payload = storagePath.CleanAbsPath(entry.Linkname) } - } else { - if len(entry.Digest) > 10 { - d := strings.Replace(entry.Digest, "sha256:", "", 1) - payload = d[:2] + "/" + d[2:] + } else if entry.Digest != "" { + d, err := digest.Parse(entry.Digest) + if err != nil { + return fmt.Errorf("invalid digest %q for %q: %w", entry.Digest, entry.Name, err) + } + path, err := storagePath.RegularFilePathForValidatedDigest(d) + if err != nil { + return fmt.Errorf("determining physical file path for %q: %w", entry.Name, err) } + payload = path } if _, err := fmt.Fprint(out, escapedOptional([]byte(payload), ESCAPE_LONE_DASH)); err != nil { @@ -219,7 +215,7 @@ func dumpNode(out io.Writer, added map[string]*internal.FileMetadata, links map[ // GenerateDump generates a dump of the TOC in the same format as `composefs-info dump` func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, error) { - toc, ok := tocI.(*internal.TOC) + toc, ok := tocI.(*minimal.TOC) if !ok { return nil, fmt.Errorf("invalid TOC type") } @@ -235,21 +231,21 @@ func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, }() links := make(map[string]int) - added := make(map[string]*internal.FileMetadata) + added := make(map[string]*minimal.FileMetadata) for _, e := range toc.Entries { if e.Linkname == "" { continue } - if e.Type == internal.TypeSymlink { + if e.Type == minimal.TypeSymlink { continue } links[e.Linkname] = links[e.Linkname] + 1 } if len(toc.Entries) == 0 { - root := &internal.FileMetadata{ + root := &minimal.FileMetadata{ Name: "/", - Type: internal.TypeDir, + Type: minimal.TypeDir, Mode: 0o755, } @@ -261,7 +257,7 @@ func GenerateDump(tocI interface{}, verityDigests map[string]string) (io.Reader, } for _, e := range toc.Entries { - if e.Type == internal.TypeChunk { + if e.Type == minimal.TypeChunk { continue } if err := dumpNode(w, added, links, verityDigests, &e); err != nil { diff --git a/vendor/github.com/containers/storage/pkg/chunked/filesystem_linux.go b/vendor/github.com/containers/storage/pkg/chunked/filesystem_linux.go index e26e5e5c..82685e9c 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/filesystem_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/filesystem_linux.go @@ -15,7 +15,8 @@ import ( driversCopy "github.com/containers/storage/drivers/copy" "github.com/containers/storage/pkg/archive" - "github.com/containers/storage/pkg/chunked/internal" + "github.com/containers/storage/pkg/chunked/internal/minimal" + storagePath "github.com/containers/storage/pkg/chunked/internal/path" securejoin "github.com/cyphar/filepath-securejoin" "github.com/vbatts/tar-split/archive/tar" "golang.org/x/sys/unix" @@ -34,14 +35,14 @@ func procPathForFd(fd int) string { return fmt.Sprintf("/proc/self/fd/%d", fd) } -// fileMetadata is a wrapper around internal.FileMetadata with additional private fields that +// fileMetadata is a wrapper around minimal.FileMetadata with additional private fields that // are not part of the TOC document. // Type: TypeChunk entries are stored in Chunks, the primary [fileMetadata] entries never use TypeChunk. type fileMetadata struct { - internal.FileMetadata + minimal.FileMetadata // chunks stores the TypeChunk entries relevant to this entry when FileMetadata.Type == TypeReg. - chunks []*internal.FileMetadata + chunks []*minimal.FileMetadata // skipSetAttrs is set when the file attributes must not be // modified, e.g. it is a hard link from a different source, @@ -49,10 +50,37 @@ type fileMetadata struct { skipSetAttrs bool } +// splitPath takes a file path as input and returns two components: dir and base. +// Differently than filepath.Split(), this function handles some edge cases. +// If the path refers to a file in the root directory, the returned dir is "/". +// The returned base value is never empty, it never contains any slash and the +// value "..". +func splitPath(path string) (string, string, error) { + path = storagePath.CleanAbsPath(path) + dir, base := filepath.Split(path) + if base == "" { + base = "." + } + // Remove trailing slashes from dir, but make sure that "/" is preserved. + dir = strings.TrimSuffix(dir, "/") + if dir == "" { + dir = "/" + } + + if strings.Contains(base, "/") { + // This should never happen, but be safe as the base is passed to *at syscalls. + return "", "", fmt.Errorf("internal error: splitPath(%q) contains a slash", path) + } + return dir, base, nil +} + func doHardLink(dirfd, srcFd int, destFile string) error { - destDir, destBase := filepath.Split(destFile) + destDir, destBase, err := splitPath(destFile) + if err != nil { + return err + } destDirFd := dirfd - if destDir != "" && destDir != "." { + if destDir != "/" { f, err := openOrCreateDirUnderRoot(dirfd, destDir, 0) if err != nil { return err @@ -72,7 +100,7 @@ func doHardLink(dirfd, srcFd int, destFile string) error { return nil } - err := doLink() + err = doLink() // if the destination exists, unlink it first and try again if err != nil && os.IsExist(err) { @@ -281,8 +309,11 @@ func openFileUnderRootFallback(dirfd int, name string, flags uint64, mode os.Fil // If O_NOFOLLOW is specified in the flags, then resolve only the parent directory and use the // last component as the path to openat(). if hasNoFollow { - dirName, baseName := filepath.Split(name) - if dirName != "" && dirName != "." { + dirName, baseName, err := splitPath(name) + if err != nil { + return -1, err + } + if dirName != "/" { newRoot, err := securejoin.SecureJoin(root, dirName) if err != nil { return -1, err @@ -409,7 +440,8 @@ func openOrCreateDirUnderRoot(dirfd int, name string, mode os.FileMode) (*os.Fil if errors.Is(err, unix.ENOENT) { parent := filepath.Dir(name) - if parent != "" { + // do not create the root directory, it should always exist + if parent != name { pDir, err2 := openOrCreateDirUnderRoot(dirfd, parent, mode) if err2 != nil { return nil, err @@ -448,9 +480,12 @@ func appendHole(fd int, name string, size int64) error { } func safeMkdir(dirfd int, mode os.FileMode, name string, metadata *fileMetadata, options *archive.TarOptions) error { - parent, base := filepath.Split(name) + parent, base, err := splitPath(name) + if err != nil { + return err + } parentFd := dirfd - if parent != "" && parent != "." { + if parent != "/" { parentFile, err := openOrCreateDirUnderRoot(dirfd, parent, 0) if err != nil { return err @@ -506,9 +541,12 @@ func safeLink(dirfd int, mode os.FileMode, metadata *fileMetadata, options *arch } func safeSymlink(dirfd int, metadata *fileMetadata) error { - destDir, destBase := filepath.Split(metadata.Name) + destDir, destBase, err := splitPath(metadata.Name) + if err != nil { + return err + } destDirFd := dirfd - if destDir != "" && destDir != "." { + if destDir != "/" { f, err := openOrCreateDirUnderRoot(dirfd, destDir, 0) if err != nil { return err @@ -542,9 +580,12 @@ func (d whiteoutHandler) Setxattr(path, name string, value []byte) error { } func (d whiteoutHandler) Mknod(path string, mode uint32, dev int) error { - dir, base := filepath.Split(path) + dir, base, err := splitPath(path) + if err != nil { + return err + } dirfd := d.Dirfd - if dir != "" && dir != "." { + if dir != "/" { dir, err := openOrCreateDirUnderRoot(d.Dirfd, dir, 0) if err != nil { return err diff --git a/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go b/vendor/github.com/containers/storage/pkg/chunked/internal/minimal/compression.go similarity index 99% rename from vendor/github.com/containers/storage/pkg/chunked/internal/compression.go rename to vendor/github.com/containers/storage/pkg/chunked/internal/minimal/compression.go index 2a24e4bf..377ece2e 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/internal/compression.go +++ b/vendor/github.com/containers/storage/pkg/chunked/internal/minimal/compression.go @@ -1,4 +1,4 @@ -package internal +package minimal // NOTE: This is used from github.com/containers/image by callers that // don't otherwise use containers/storage, so don't make this depend on any diff --git a/vendor/github.com/containers/storage/pkg/chunked/internal/path/path.go b/vendor/github.com/containers/storage/pkg/chunked/internal/path/path.go new file mode 100644 index 00000000..55ba7455 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/chunked/internal/path/path.go @@ -0,0 +1,27 @@ +package path + +import ( + "fmt" + "path/filepath" + + "github.com/opencontainers/go-digest" +) + +// CleanAbsPath removes any ".." and "." from the path +// and ensures it starts with a "/". If the path refers to the root +// directory, it returns "/". +func CleanAbsPath(path string) string { + return filepath.Clean("/" + path) +} + +// RegularFilePath returns the path used in the composefs backing store for a +// regular file with the provided content digest. +// +// The caller MUST ensure d is a valid digest (in particular, that it contains no path separators or .. entries) +func RegularFilePathForValidatedDigest(d digest.Digest) (string, error) { + if algo := d.Algorithm(); algo != digest.SHA256 { + return "", fmt.Errorf("unexpected digest algorithm %q", algo) + } + e := d.Encoded() + return e[0:2] + "/" + e[2:], nil +} diff --git a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go index 8ecbfb98..8f467985 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go @@ -2,6 +2,7 @@ package chunked import ( archivetar "archive/tar" + "bytes" "context" "encoding/base64" "errors" @@ -22,17 +23,21 @@ import ( graphdriver "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/chunked/compressor" - "github.com/containers/storage/pkg/chunked/internal" + "github.com/containers/storage/pkg/chunked/internal/minimal" + path "github.com/containers/storage/pkg/chunked/internal/path" "github.com/containers/storage/pkg/chunked/toc" "github.com/containers/storage/pkg/fsverity" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/system" + securejoin "github.com/cyphar/filepath-securejoin" jsoniter "github.com/json-iterator/go" "github.com/klauspost/compress/zstd" "github.com/klauspost/pgzip" digest "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" "github.com/vbatts/tar-split/archive/tar" + "github.com/vbatts/tar-split/tar/asm" + tsStorage "github.com/vbatts/tar-split/tar/storage" "golang.org/x/sys/unix" ) @@ -57,46 +62,53 @@ const ( type compressedFileType int type chunkedDiffer struct { + // Initial parameters, used throughout and never modified + // ========== + pullOptions pullOptions stream ImageSourceSeekable - manifest []byte - toc *internal.TOC // The parsed contents of manifest, or nil if not yet available - tarSplit []byte - layersCache *layersCache - tocOffset int64 - fileType compressedFileType - - copyBuffer []byte - - gzipReader *pgzip.Reader - zstdReader *zstd.Decoder - rawReader io.Reader - - // tocDigest is the digest of the TOC document when the layer - // is partially pulled. - tocDigest digest.Digest + // blobDigest is the digest of the whole compressed layer. It is used if + // convertToZstdChunked to validate a layer when it is converted since there + // is no TOC referenced by the manifest. + blobDigest digest.Digest + blobSize int64 + // Input format + // ========== + fileType compressedFileType // convertedToZstdChunked is set to true if the layer needs to // be converted to the zstd:chunked format before it can be // handled. convertToZstdChunked bool + // Chunked metadata + // This is usually set in GetDiffer, but if convertToZstdChunked, it is only computed in chunkedDiffer.ApplyDiff + // ========== + // tocDigest is the digest of the TOC document when the layer + // is partially pulled, or "" if not relevant to consumers. + tocDigest digest.Digest + tocOffset int64 + manifest []byte + toc *minimal.TOC // The parsed contents of manifest, or nil if not yet available + tarSplit []byte + uncompressedTarSize int64 // -1 if unknown // skipValidation is set to true if the individual files in // the layer are trusted and should not be validated. skipValidation bool - // blobDigest is the digest of the whole compressed layer. It is used if - // convertToZstdChunked to validate a layer when it is converted since there - // is no TOC referenced by the manifest. - blobDigest digest.Digest - - blobSize int64 - uncompressedTarSize int64 // -1 if unknown - - pullOptions map[string]string - - useFsVerity graphdriver.DifferFsVerity + // Long-term caches + // This is set in GetDiffer, when the caller must not hold any storage locks, and later consumed in .ApplyDiff() + // ========== + layersCache *layersCache + copyBuffer []byte + fsVerityMutex sync.Mutex // protects fsVerityDigests fsVerityDigests map[string]string - fsVerityMutex sync.Mutex + + // Private state of .ApplyDiff + // ========== + gzipReader *pgzip.Reader + zstdReader *zstd.Decoder + rawReader io.Reader + useFsVerity graphdriver.DifferFsVerity } var xattrsToIgnore = map[string]interface{}{ @@ -108,6 +120,42 @@ type chunkedLayerData struct { Format graphdriver.DifferOutputFormat `json:"format"` } +// pullOptions contains parsed data from storage.Store.PullOptions. +// TO DO: ideally this should be parsed along with the rest of the config file into StoreOptions directly +// (and then storage.Store.PullOptions would need to be somehow simulated). +type pullOptions struct { + enablePartialImages bool // enable_partial_images + convertImages bool // convert_images + useHardLinks bool // use_hard_links + insecureAllowUnpredictableImageContents bool // insecure_allow_unpredictable_image_contents + ostreeRepos []string // ostree_repos +} + +func parsePullOptions(store storage.Store) pullOptions { + options := store.PullOptions() + + res := pullOptions{} + for _, e := range []struct { + dest *bool + name string + defaultValue bool + }{ + {&res.enablePartialImages, "enable_partial_images", false}, + {&res.convertImages, "convert_images", false}, + {&res.useHardLinks, "use_hard_links", false}, + {&res.insecureAllowUnpredictableImageContents, "insecure_allow_unpredictable_image_contents", false}, + } { + if value, ok := options[e.name]; ok { + *e.dest = strings.ToLower(value) == "true" + } else { + *e.dest = e.defaultValue + } + } + res.ostreeRepos = strings.Split(options["ostree_repos"], ":") + + return res +} + func (c *chunkedDiffer) convertTarToZstdChunked(destDirectory string, payload *os.File) (int64, *seekableFile, digest.Digest, map[string]string, error) { diff, err := archive.DecompressStream(payload) if err != nil { @@ -144,127 +192,160 @@ func (c *chunkedDiffer) convertTarToZstdChunked(destDirectory string, payload *o } // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer. -// If it returns an error that implements IsErrFallbackToOrdinaryLayerDownload, the caller can +// If it returns an error that matches ErrFallbackToOrdinaryLayerDownload, the caller can // retry the operation with a different method. func GetDiffer(ctx context.Context, store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) { - pullOptions := store.PullOptions() + pullOptions := parsePullOptions(store) - if !parseBooleanPullOption(pullOptions, "enable_partial_images", false) { - // If convertImages is set, the two options disagree whether fallback is permissible. + if !pullOptions.enablePartialImages { + // If pullOptions.convertImages is set, the two options disagree whether fallback is permissible. // Right now, we enable it, but that’s not a promise; rather, such a configuration should ideally be rejected. return nil, newErrFallbackToOrdinaryLayerDownload(errors.New("partial images are disabled")) } - // convertImages also serves as a “must not fallback to non-partial pull” option (?!) - convertImages := parseBooleanPullOption(pullOptions, "convert_images", false) + // pullOptions.convertImages also serves as a “must not fallback to non-partial pull” option (?!) graphDriver, err := store.GraphDriver() if err != nil { return nil, err } if _, partialSupported := graphDriver.(graphdriver.DriverWithDiffer); !partialSupported { - if convertImages { + if pullOptions.convertImages { return nil, fmt.Errorf("graph driver %s does not support partial pull but convert_images requires that", graphDriver.String()) } return nil, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("graph driver %s does not support partial pull", graphDriver.String())) } - differ, canFallback, err := getProperDiffer(store, blobDigest, blobSize, annotations, iss, pullOptions) + differ, err := getProperDiffer(store, blobDigest, blobSize, annotations, iss, pullOptions) if err != nil { - if !canFallback { + var fallbackErr ErrFallbackToOrdinaryLayerDownload + if !errors.As(err, &fallbackErr) { return nil, err } // If convert_images is enabled, always attempt to convert it instead of returning an error or falling back to a different method. - if convertImages { - logrus.Debugf("Created differ to convert blob %q", blobDigest) - return makeConvertFromRawDiffer(store, blobDigest, blobSize, iss, pullOptions) + if !pullOptions.convertImages { + return nil, err } - return nil, newErrFallbackToOrdinaryLayerDownload(err) + var canConvertErr errFallbackCanConvert + if !errors.As(err, &canConvertErr) { + // We are supposed to use makeConvertFromRawDiffer, but that would not work. + // Fail, and make sure the error does _not_ match ErrFallbackToOrdinaryLayerDownload: use only the error text, + // discard all type information. + return nil, fmt.Errorf("neither a partial pull nor convert_images is possible: %s", err.Error()) + } + logrus.Debugf("Created differ to convert blob %q", blobDigest) + return makeConvertFromRawDiffer(store, blobDigest, blobSize, iss, pullOptions) } return differ, nil } +// errFallbackCanConvert is an an error type _accompanying_ ErrFallbackToOrdinaryLayerDownload +// within getProperDiffer, to mark that using makeConvertFromRawDiffer makes sense. +// This is used to distinguish between cases where the environment does not support partial pulls +// (e.g. a registry does not support range requests) and convert_images is still possible, +// from cases where the image content is unacceptable for partial pulls (e.g. exceeds memory limits) +// and convert_images would not help. +type errFallbackCanConvert struct { + err error +} + +func (e errFallbackCanConvert) Error() string { + return e.err.Error() +} + +func (e errFallbackCanConvert) Unwrap() error { + return e.err +} + // getProperDiffer is an implementation detail of GetDiffer. // It returns a “proper” differ (not a convert_images one) if possible. -// On error, the second parameter is true if a fallback to an alternative (either the makeConverToRaw differ, or a non-partial pull) -// is permissible. -func getProperDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, pullOptions map[string]string) (graphdriver.Differ, bool, error) { - zstdChunkedTOCDigestString, hasZstdChunkedTOC := annotations[internal.ManifestChecksumKey] +// May return an error matching ErrFallbackToOrdinaryLayerDownload if a fallback to an alternative +// (either makeConvertFromRawDiffer, or a non-partial pull) is permissible. +func getProperDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, pullOptions pullOptions) (graphdriver.Differ, error) { + zstdChunkedTOCDigestString, hasZstdChunkedTOC := annotations[minimal.ManifestChecksumKey] estargzTOCDigestString, hasEstargzTOC := annotations[estargz.TOCJSONDigestAnnotation] switch { case hasZstdChunkedTOC && hasEstargzTOC: - return nil, false, errors.New("both zstd:chunked and eStargz TOC found") + return nil, errors.New("both zstd:chunked and eStargz TOC found") case hasZstdChunkedTOC: zstdChunkedTOCDigest, err := digest.Parse(zstdChunkedTOCDigestString) if err != nil { - return nil, false, err + return nil, err } differ, err := makeZstdChunkedDiffer(store, blobSize, zstdChunkedTOCDigest, annotations, iss, pullOptions) if err != nil { logrus.Debugf("Could not create zstd:chunked differ for blob %q: %v", blobDigest, err) - // If the error is a bad request to the server, then signal to the caller that it can try a different method. - var badRequestErr ErrBadRequest - return nil, errors.As(err, &badRequestErr), err + return nil, err } logrus.Debugf("Created zstd:chunked differ for blob %q", blobDigest) - return differ, false, nil + return differ, nil case hasEstargzTOC: estargzTOCDigest, err := digest.Parse(estargzTOCDigestString) if err != nil { - return nil, false, err + return nil, err } differ, err := makeEstargzChunkedDiffer(store, blobSize, estargzTOCDigest, iss, pullOptions) if err != nil { logrus.Debugf("Could not create estargz differ for blob %q: %v", blobDigest, err) - // If the error is a bad request to the server, then signal to the caller that it can try a different method. - var badRequestErr ErrBadRequest - return nil, errors.As(err, &badRequestErr), err + return nil, err } logrus.Debugf("Created eStargz differ for blob %q", blobDigest) - return differ, false, nil + return differ, nil default: // no TOC - convertImages := parseBooleanPullOption(pullOptions, "convert_images", false) - if !convertImages { - return nil, true, errors.New("no TOC found and convert_images is not configured") + message := "no TOC found" + if !pullOptions.convertImages { + message = "no TOC found and convert_images is not configured" + } + return nil, errFallbackCanConvert{ + newErrFallbackToOrdinaryLayerDownload(errors.New(message)), } - return nil, true, errors.New("no TOC found") } } -func makeConvertFromRawDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) { +func makeConvertFromRawDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, error) { layersCache, err := getLayersCache(store) if err != nil { return nil, err } return &chunkedDiffer{ - fsVerityDigests: make(map[string]string), - blobDigest: blobDigest, - blobSize: blobSize, - uncompressedTarSize: -1, // Will be computed later + pullOptions: pullOptions, + stream: iss, + blobDigest: blobDigest, + blobSize: blobSize, + convertToZstdChunked: true, - copyBuffer: makeCopyBuffer(), - layersCache: layersCache, - pullOptions: pullOptions, - stream: iss, + + uncompressedTarSize: -1, // Will be computed later + + layersCache: layersCache, + copyBuffer: makeCopyBuffer(), + fsVerityDigests: make(map[string]string), }, nil } -func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, annotations map[string]string, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) { +// makeZstdChunkedDiffer sets up a chunkedDiffer for a zstd:chunked layer. +// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert. +func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, annotations map[string]string, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, error) { manifest, toc, tarSplit, tocOffset, err := readZstdChunkedManifest(iss, tocDigest, annotations) - if err != nil { + if err != nil { // May be ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert return nil, fmt.Errorf("read zstd:chunked manifest: %w", err) } + var uncompressedTarSize int64 = -1 if tarSplit != nil { uncompressedTarSize, err = tarSizeFromTarSplit(tarSplit) if err != nil { return nil, fmt.Errorf("computing size from tar-split: %w", err) } + } else if !pullOptions.insecureAllowUnpredictableImageContents { // With no tar-split, we can't compute the traditional UncompressedDigest. + return nil, errFallbackCanConvert{ + newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("zstd:chunked layers without tar-split data don't support partial pulls with guaranteed consistency with non-partial pulls")), + } } layersCache, err := getLayersCache(store) @@ -273,25 +354,36 @@ func makeZstdChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest } return &chunkedDiffer{ - fsVerityDigests: make(map[string]string), - blobSize: blobSize, - uncompressedTarSize: uncompressedTarSize, + pullOptions: pullOptions, + stream: iss, + blobSize: blobSize, + + fileType: fileTypeZstdChunked, + tocDigest: tocDigest, - copyBuffer: makeCopyBuffer(), - fileType: fileTypeZstdChunked, - layersCache: layersCache, + tocOffset: tocOffset, manifest: manifest, toc: toc, - pullOptions: pullOptions, - stream: iss, tarSplit: tarSplit, - tocOffset: tocOffset, + uncompressedTarSize: uncompressedTarSize, + + layersCache: layersCache, + copyBuffer: makeCopyBuffer(), + fsVerityDigests: make(map[string]string), }, nil } -func makeEstargzChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) { +// makeEstargzChunkedDiffer sets up a chunkedDiffer for an estargz layer. +// It may return an error matching ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert. +func makeEstargzChunkedDiffer(store storage.Store, blobSize int64, tocDigest digest.Digest, iss ImageSourceSeekable, pullOptions pullOptions) (*chunkedDiffer, error) { + if !pullOptions.insecureAllowUnpredictableImageContents { // With no tar-split, we can't compute the traditional UncompressedDigest. + return nil, errFallbackCanConvert{ + newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("estargz layers don't support partial pulls with guaranteed consistency with non-partial pulls")), + } + } + manifest, tocOffset, err := readEstargzChunkedManifest(iss, blobSize, tocDigest) - if err != nil { + if err != nil { // May be ErrFallbackToOrdinaryLayerDownload / errFallbackCanConvert return nil, fmt.Errorf("read zstd:chunked manifest: %w", err) } layersCache, err := getLayersCache(store) @@ -300,17 +392,20 @@ func makeEstargzChunkedDiffer(store storage.Store, blobSize int64, tocDigest dig } return &chunkedDiffer{ - fsVerityDigests: make(map[string]string), - blobSize: blobSize, - uncompressedTarSize: -1, // We would have to read and decompress the whole layer + pullOptions: pullOptions, + stream: iss, + blobSize: blobSize, + + fileType: fileTypeEstargz, + tocDigest: tocDigest, - copyBuffer: makeCopyBuffer(), - fileType: fileTypeEstargz, - layersCache: layersCache, - manifest: manifest, - pullOptions: pullOptions, - stream: iss, tocOffset: tocOffset, + manifest: manifest, + uncompressedTarSize: -1, // We would have to read and decompress the whole layer + + layersCache: layersCache, + copyBuffer: makeCopyBuffer(), + fsVerityDigests: make(map[string]string), }, nil } @@ -391,7 +486,7 @@ func canDedupFileWithHardLink(file *fileMetadata, fd int, s os.FileInfo) bool { } // fill only the attributes used by canDedupMetadataWithHardLink. otherFile := fileMetadata{ - FileMetadata: internal.FileMetadata{ + FileMetadata: minimal.FileMetadata{ UID: int(st.Uid), GID: int(st.Gid), Mode: int64(st.Mode), @@ -735,7 +830,12 @@ func (d *destinationFile) Close() (Err error) { } } - return setFileAttrs(d.dirfd, d.file, os.FileMode(d.metadata.Mode), d.metadata, d.options, false) + mode := os.FileMode(d.metadata.Mode) + if d.options.ForceMask != nil { + mode = *d.options.ForceMask + } + + return setFileAttrs(d.dirfd, d.file, mode, d.metadata, d.options, false) } func closeDestinationFiles(files chan *destinationFile, errors chan error) { @@ -1038,13 +1138,6 @@ type hardLinkToCreate struct { metadata *fileMetadata } -func parseBooleanPullOption(pullOptions map[string]string, name string, def bool) bool { - if value, ok := pullOptions[name]; ok { - return strings.ToLower(value) == "true" - } - return def -} - type findAndCopyFileOptions struct { useHardLinks bool ostreeRepos []string @@ -1111,10 +1204,13 @@ func (c *chunkedDiffer) findAndCopyFile(dirfd int, r *fileMetadata, copyOptions return false, nil } -func makeEntriesFlat(mergedEntries []fileMetadata) ([]fileMetadata, error) { +// makeEntriesFlat collects regular-file entries from mergedEntries, and produces a new list +// where each file content is only represented once, and uses composefs.RegularFilePathForValidatedDigest for its name. +// If flatPathNameMap is not nil, this function writes to it a mapping from filepath.Clean(originalName) to the composefs name. +func makeEntriesFlat(mergedEntries []fileMetadata, flatPathNameMap map[string]string) ([]fileMetadata, error) { var new []fileMetadata - hashes := make(map[string]string) + knownFlatPaths := make(map[string]struct{}) for i := range mergedEntries { if mergedEntries[i].Type != TypeReg { continue @@ -1124,16 +1220,22 @@ func makeEntriesFlat(mergedEntries []fileMetadata) ([]fileMetadata, error) { } digest, err := digest.Parse(mergedEntries[i].Digest) if err != nil { - return nil, err + return nil, fmt.Errorf("invalid digest %q for %q: %w", mergedEntries[i].Digest, mergedEntries[i].Name, err) + } + path, err := path.RegularFilePathForValidatedDigest(digest) + if err != nil { + return nil, fmt.Errorf("determining physical file path for %q: %w", mergedEntries[i].Name, err) + } + if flatPathNameMap != nil { + flatPathNameMap[filepath.Clean(mergedEntries[i].Name)] = path } - d := digest.Encoded() - if hashes[d] != "" { + if _, known := knownFlatPaths[path]; known { continue } - hashes[d] = d + knownFlatPaths[path] = struct{}{} - mergedEntries[i].Name = fmt.Sprintf("%s/%s", d[0:2], d[2:]) + mergedEntries[i].Name = path mergedEntries[i].skipSetAttrs = true new = append(new, mergedEntries[i]) @@ -1141,44 +1243,140 @@ func makeEntriesFlat(mergedEntries []fileMetadata) ([]fileMetadata, error) { return new, nil } -func (c *chunkedDiffer) copyAllBlobToFile(destination *os.File) (digest.Digest, error) { - var payload io.ReadCloser - var streams chan io.ReadCloser - var errs chan error - var err error +type streamOrErr struct { + stream io.ReadCloser + err error +} - chunksToRequest := []ImageSourceChunk{ - { - Offset: 0, - Length: uint64(c.blobSize), - }, +// ensureAllBlobsDone ensures that all blobs are closed and returns the first error encountered. +func ensureAllBlobsDone(streamsOrErrors chan streamOrErr) (retErr error) { + for soe := range streamsOrErrors { + if soe.stream != nil { + _ = soe.stream.Close() + } else if retErr == nil { + retErr = soe.err + } } + return +} - streams, errs, err = c.stream.GetBlobAt(chunksToRequest) - if err != nil { - return "", err +// getBlobAtConverterGoroutine reads from the streams and errs channels, then sends +// either a stream or an error to the stream channel. The streams channel is closed when +// there are no more streams and errors to read. +// It ensures that no more than maxStreams streams are returned, and that every item from the +// streams and errs channels is consumed. +func getBlobAtConverterGoroutine(stream chan streamOrErr, streams chan io.ReadCloser, errs chan error, maxStreams int) { + tooManyStreams := false + streamsSoFar := 0 + + err := errors.New("Unexpected error in getBlobAtGoroutine") + + defer func() { + if err != nil { + stream <- streamOrErr{err: err} + } + close(stream) + }() + +loop: + for { + select { + case p, ok := <-streams: + if !ok { + streams = nil + break loop + } + if streamsSoFar >= maxStreams { + tooManyStreams = true + _ = p.Close() + continue + } + streamsSoFar++ + stream <- streamOrErr{stream: p} + case err, ok := <-errs: + if !ok { + errs = nil + break loop + } + stream <- streamOrErr{err: err} + } } - select { - case p := <-streams: - payload = p - case err := <-errs: - return "", err + if streams != nil { + for p := range streams { + if streamsSoFar >= maxStreams { + tooManyStreams = true + _ = p.Close() + continue + } + streamsSoFar++ + stream <- streamOrErr{stream: p} + } + } + if errs != nil { + for err := range errs { + stream <- streamOrErr{err: err} + } } - if payload == nil { - return "", errors.New("invalid stream returned") + if tooManyStreams { + stream <- streamOrErr{err: fmt.Errorf("too many streams returned, got more than %d", maxStreams)} } - defer payload.Close() + err = nil +} - originalRawDigester := digest.Canonical.Digester() +// getBlobAt provides a much more convenient way to consume data returned by ImageSourceSeekable.GetBlobAt. +// GetBlobAt returns two channels, forcing a caller to `select` on both of them — and in Go, reading a closed channel +// always succeeds in select. +// Instead, getBlobAt provides a single channel with all events, which can be consumed conveniently using `range`. +func getBlobAt(is ImageSourceSeekable, chunksToRequest ...ImageSourceChunk) (chan streamOrErr, error) { + streams, errs, err := is.GetBlobAt(chunksToRequest) + if err != nil { + return nil, err + } + stream := make(chan streamOrErr) + go getBlobAtConverterGoroutine(stream, streams, errs, len(chunksToRequest)) + return stream, nil +} - r := io.TeeReader(payload, originalRawDigester.Hash()) +func (c *chunkedDiffer) copyAllBlobToFile(destination *os.File) (digest.Digest, error) { + streamsOrErrors, err := getBlobAt(c.stream, ImageSourceChunk{Offset: 0, Length: uint64(c.blobSize)}) + if err != nil { + return "", err + } - // copy the entire tarball and compute its digest - _, err = io.CopyBuffer(destination, r, c.copyBuffer) + originalRawDigester := digest.Canonical.Digester() + for soe := range streamsOrErrors { + if soe.stream != nil { + r := io.TeeReader(soe.stream, originalRawDigester.Hash()) + // copy the entire tarball and compute its digest + _, err = io.CopyBuffer(destination, r, c.copyBuffer) + _ = soe.stream.Close() + } + if soe.err != nil && err == nil { + err = soe.err + } + } return originalRawDigester.Digest(), err } +func typeToOsMode(typ string) (os.FileMode, error) { + switch typ { + case TypeReg, TypeLink: + return 0, nil + case TypeSymlink: + return os.ModeSymlink, nil + case TypeDir: + return os.ModeDir, nil + case TypeChar: + return os.ModeDevice | os.ModeCharDevice, nil + case TypeBlock: + return os.ModeDevice, nil + case TypeFifo: + return os.ModeNamedPipe, nil + } + return 0, fmt.Errorf("unknown file type %q", typ) +} + func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, differOpts *graphdriver.DifferOptions) (graphdriver.DriverWithDifferOutput, error) { defer c.layersCache.release() defer func() { @@ -1298,13 +1496,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff Size: c.uncompressedTarSize, } - // When the hard links deduplication is used, file attributes are ignored because setting them - // modifies the source file as well. - useHardLinks := parseBooleanPullOption(c.pullOptions, "use_hard_links", false) - - // List of OSTree repositories to use for deduplication - ostreeRepos := strings.Split(c.pullOptions["ostree_repos"], ":") - whiteoutConverter := archive.GetWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData) var missingParts []missingPart @@ -1325,7 +1516,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff if err == nil { value := idtools.Stat{ IDs: idtools.IDPair{UID: int(uid), GID: int(gid)}, - Mode: os.FileMode(mode), + Mode: os.ModeDir | os.FileMode(mode), } if err := idtools.SetContainersOverrideXattr(dest, value); err != nil { return output, err @@ -1337,16 +1528,20 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff if err != nil { return output, &fs.PathError{Op: "open", Path: dest, Err: err} } - defer unix.Close(dirfd) + dirFile := os.NewFile(uintptr(dirfd), dest) + defer dirFile.Close() + var flatPathNameMap map[string]string // = nil if differOpts != nil && differOpts.Format == graphdriver.DifferOutputFormatFlat { - mergedEntries, err = makeEntriesFlat(mergedEntries) + flatPathNameMap = map[string]string{} + mergedEntries, err = makeEntriesFlat(mergedEntries, flatPathNameMap) if err != nil { return output, err } createdDirs := make(map[string]struct{}) for _, e := range mergedEntries { - d := e.Name[0:2] + // This hard-codes an assumption that RegularFilePathForValidatedDigest creates paths with exactly one directory component. + d := filepath.Dir(e.Name) if _, found := createdDirs[d]; !found { if err := unix.Mkdirat(dirfd, d, 0o755); err != nil { return output, &fs.PathError{Op: "mkdirat", Path: d, Err: err} @@ -1363,8 +1558,10 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff missingPartsSize, totalChunksSize := int64(0), int64(0) copyOptions := findAndCopyFileOptions{ - useHardLinks: useHardLinks, - ostreeRepos: ostreeRepos, + // When the hard links deduplication is used, file attributes are ignored because setting them + // modifies the source file as well. + useHardLinks: c.pullOptions.useHardLinks, + ostreeRepos: c.pullOptions.ostreeRepos, // List of OSTree repositories to use for deduplication options: options, } @@ -1408,13 +1605,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff filesToWaitFor := 0 for i := range mergedEntries { r := &mergedEntries[i] - if options.ForceMask != nil { - value := idtools.FormatContainersOverrideXattr(r.UID, r.GID, int(r.Mode)) - if r.Xattrs == nil { - r.Xattrs = make(map[string]string) - } - r.Xattrs[idtools.ContainersOverrideXattr] = base64.StdEncoding.EncodeToString([]byte(value)) - } mode := os.FileMode(r.Mode) @@ -1423,10 +1613,37 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff return output, err } - r.Name = filepath.Clean(r.Name) + size := r.Size + + // update also the implementation of ForceMask in pkg/archive + if options.ForceMask != nil { + mode = *options.ForceMask + + // special files will be stored as regular files + if t != tar.TypeDir && t != tar.TypeSymlink && t != tar.TypeReg && t != tar.TypeLink { + t = tar.TypeReg + size = 0 + } + + // if the entry will be stored as a directory or a regular file, store in a xattr the original + // owner and mode. + if t == tar.TypeDir || t == tar.TypeReg { + typeMode, err := typeToOsMode(r.Type) + if err != nil { + return output, err + } + value := idtools.FormatContainersOverrideXattrDevice(r.UID, r.GID, typeMode|fs.FileMode(r.Mode), int(r.Devmajor), int(r.Devminor)) + if r.Xattrs == nil { + r.Xattrs = make(map[string]string) + } + r.Xattrs[idtools.ContainersOverrideXattr] = base64.StdEncoding.EncodeToString([]byte(value)) + } + } + + r.Name = path.CleanAbsPath(r.Name) // do not modify the value of symlinks if r.Linkname != "" && t != tar.TypeSymlink { - r.Linkname = filepath.Clean(r.Linkname) + r.Linkname = path.CleanAbsPath(r.Linkname) } if whiteoutConverter != nil { @@ -1434,8 +1651,8 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff Typeflag: t, Name: r.Name, Linkname: r.Linkname, - Size: r.Size, - Mode: r.Mode, + Size: size, + Mode: int64(mode), Uid: r.UID, Gid: r.GID, } @@ -1454,7 +1671,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff switch t { case tar.TypeReg: // Create directly empty files. - if r.Size == 0 { + if size == 0 { // Used to have a scope for cleanup. createEmptyFile := func() error { file, err := openFileUnderRoot(dirfd, r.Name, newFileFlags, 0) @@ -1474,7 +1691,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff } case tar.TypeDir: - if r.Name == "" || r.Name == "." { + if r.Name == "/" { output.RootDirMode = &mode } if err := safeMkdir(dirfd, mode, r.Name, r, options); err != nil { @@ -1509,7 +1726,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff return output, fmt.Errorf("invalid type %q", t) } - totalChunksSize += r.Size + totalChunksSize += size if t == tar.TypeReg { index := i @@ -1572,7 +1789,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff } switch chunk.ChunkType { - case internal.ChunkTypeData: + case minimal.ChunkTypeData: root, path, offset, err := c.layersCache.findChunkInOtherLayers(chunk) if err != nil { return output, err @@ -1585,7 +1802,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff Offset: offset, } } - case internal.ChunkTypeZeros: + case minimal.ChunkTypeZeros: missingPartsSize -= size mp.Hole = true // Mark all chunks belonging to the missing part as holes @@ -1609,6 +1826,39 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff } } + // To ensure that consumers of the layer who decompress and read the full tar stream, + // and consumers who consume the data via the TOC, both see exactly the same data and metadata, + // compute the UncompressedDigest. + // c/image will then ensure that this value matches the value in the image config’s RootFS.DiffID, i.e. the image must commit + // to one UncompressedDigest value for each layer, and that will avoid the ambiguity (in consumers who validate layers against DiffID). + // + // c/image also uses the UncompressedDigest as a layer ID, allowing it to use the traditional layer and image IDs. + // + // This is, sadly, quite costly: Up to now we might have only have had to write, and digest, only the new/modified files. + // Here we need to read, and digest, the whole layer, even if almost all of it was already present locally previously. + // So, really specialized (EXTREMELY RARE) users can opt out of this check using insecureAllowUnpredictableImageContents . + // + // Layers without a tar-split (estargz layers and old zstd:chunked layers) can't produce an UncompressedDigest that + // matches the expected RootFS.DiffID; we always fall back to full pulls, again unless the user opts out + // via insecureAllowUnpredictableImageContents . + if output.UncompressedDigest == "" { + switch { + case c.pullOptions.insecureAllowUnpredictableImageContents: + // Oh well. Skip the costly digest computation. + case output.TarSplit != nil: + metadata := tsStorage.NewJSONUnpacker(bytes.NewReader(output.TarSplit)) + fg := newStagedFileGetter(dirFile, flatPathNameMap) + digester := digest.Canonical.Digester() + if err := asm.WriteOutputTarStream(fg, metadata, digester.Hash()); err != nil { + return output, fmt.Errorf("digesting staged uncompressed stream: %w", err) + } + output.UncompressedDigest = digester.Digest() + default: + // We are checking for this earlier in GetDiffer, so this should not be reachable. + return output, fmt.Errorf(`internal error: layer's UncompressedDigest is unknown and "insecure_allow_unpredictable_image_contents" is not set`) + } + } + if totalChunksSize > 0 { logrus.Debugf("Missing %d bytes out of %d (%.2f %%)", missingPartsSize, totalChunksSize, float32(missingPartsSize*100.0)/float32(totalChunksSize)) } @@ -1618,7 +1868,7 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions, diff return output, nil } -func mustSkipFile(fileType compressedFileType, e internal.FileMetadata) bool { +func mustSkipFile(fileType compressedFileType, e minimal.FileMetadata) bool { // ignore the metadata files for the estargz format. if fileType != fileTypeEstargz { return false @@ -1631,7 +1881,7 @@ func mustSkipFile(fileType compressedFileType, e internal.FileMetadata) bool { return false } -func (c *chunkedDiffer) mergeTocEntries(fileType compressedFileType, entries []internal.FileMetadata) ([]fileMetadata, error) { +func (c *chunkedDiffer) mergeTocEntries(fileType compressedFileType, entries []minimal.FileMetadata) ([]fileMetadata, error) { countNextChunks := func(start int) int { count := 0 for _, e := range entries[start:] { @@ -1668,7 +1918,7 @@ func (c *chunkedDiffer) mergeTocEntries(fileType compressedFileType, entries []i if e.Type == TypeReg { nChunks := countNextChunks(i + 1) - e.chunks = make([]*internal.FileMetadata, nChunks+1) + e.chunks = make([]*minimal.FileMetadata, nChunks+1) for j := 0; j <= nChunks; j++ { // we need a copy here, otherwise we override the // .Size later @@ -1703,7 +1953,7 @@ func (c *chunkedDiffer) mergeTocEntries(fileType compressedFileType, entries []i // validateChunkChecksum checks if the file at $root/$path[offset:chunk.ChunkSize] has the // same digest as chunk.ChunkDigest -func validateChunkChecksum(chunk *internal.FileMetadata, root, path string, offset int64, copyBuffer []byte) bool { +func validateChunkChecksum(chunk *minimal.FileMetadata, root, path string, offset int64, copyBuffer []byte) bool { parentDirfd, err := unix.Open(root, unix.O_PATH|unix.O_CLOEXEC, 0) if err != nil { return false @@ -1734,3 +1984,33 @@ func validateChunkChecksum(chunk *internal.FileMetadata, root, path string, offs return digester.Digest() == digest } + +// newStagedFileGetter returns an object usable as storage.FileGetter for rootDir. +// if flatPathNameMap is not nil, it must be used to map logical file names into the backing file paths. +func newStagedFileGetter(rootDir *os.File, flatPathNameMap map[string]string) *stagedFileGetter { + return &stagedFileGetter{ + rootDir: rootDir, + flatPathNameMap: flatPathNameMap, + } +} + +type stagedFileGetter struct { + rootDir *os.File + flatPathNameMap map[string]string // nil, or a map from filepath.Clean()ed tar file names to expected on-filesystem names +} + +func (fg *stagedFileGetter) Get(filename string) (io.ReadCloser, error) { + if fg.flatPathNameMap != nil { + path, ok := fg.flatPathNameMap[filepath.Clean(filename)] + if !ok { + return nil, fmt.Errorf("no path mapping exists for tar entry %q", filename) + } + filename = path + } + pathFD, err := securejoin.OpenatInRoot(fg.rootDir, filename) + if err != nil { + return nil, err + } + defer pathFD.Close() + return securejoin.Reopen(pathFD, unix.O_RDONLY) +} diff --git a/vendor/github.com/containers/storage/pkg/chunked/storage_unsupported.go b/vendor/github.com/containers/storage/pkg/chunked/storage_unsupported.go index 7be0adeb..fe3d36c7 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/storage_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/chunked/storage_unsupported.go @@ -13,5 +13,5 @@ import ( // GetDiffer returns a differ than can be used with ApplyDiffWithDiffer. func GetDiffer(ctx context.Context, store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable) (graphdriver.Differ, error) { - return nil, errors.New("format not supported on this system") + return nil, newErrFallbackToOrdinaryLayerDownload(errors.New("format not supported on this system")) } diff --git a/vendor/github.com/containers/storage/pkg/chunked/toc/toc.go b/vendor/github.com/containers/storage/pkg/chunked/toc/toc.go index 6fbaa41b..6f39b2ae 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/toc/toc.go +++ b/vendor/github.com/containers/storage/pkg/chunked/toc/toc.go @@ -3,7 +3,7 @@ package toc import ( "errors" - "github.com/containers/storage/pkg/chunked/internal" + "github.com/containers/storage/pkg/chunked/internal/minimal" digest "github.com/opencontainers/go-digest" ) @@ -19,7 +19,7 @@ const tocJSONDigestAnnotation = "containerd.io/snapshot/stargz/toc.digest" // This is an experimental feature and may be changed/removed in the future. func GetTOCDigest(annotations map[string]string) (*digest.Digest, error) { d1, ok1 := annotations[tocJSONDigestAnnotation] - d2, ok2 := annotations[internal.ManifestChecksumKey] + d2, ok2 := annotations[minimal.ManifestChecksumKey] switch { case ok1 && ok2: return nil, errors.New("both zstd:chunked and eStargz TOC found") diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools.go b/vendor/github.com/containers/storage/pkg/idtools/idtools.go index 248594e9..299bdbef 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools.go @@ -4,6 +4,7 @@ import ( "bufio" "errors" "fmt" + "io/fs" "os" "os/user" "runtime" @@ -369,27 +370,66 @@ func checkChownErr(err error, name string, uid, gid int) error { // Stat contains file states that can be overridden with ContainersOverrideXattr. type Stat struct { - IDs IDPair - Mode os.FileMode + IDs IDPair + Mode os.FileMode + Major int + Minor int } // FormatContainersOverrideXattr will format the given uid, gid, and mode into a string // that can be used as the value for the ContainersOverrideXattr xattr. func FormatContainersOverrideXattr(uid, gid, mode int) string { - return fmt.Sprintf("%d:%d:0%o", uid, gid, mode&0o7777) + return FormatContainersOverrideXattrDevice(uid, gid, fs.FileMode(mode), 0, 0) +} + +// FormatContainersOverrideXattrDevice will format the given uid, gid, and mode into a string +// that can be used as the value for the ContainersOverrideXattr xattr. For devices, it also +// needs the major and minor numbers. +func FormatContainersOverrideXattrDevice(uid, gid int, mode fs.FileMode, major, minor int) string { + typ := "" + switch mode & os.ModeType { + case os.ModeDir: + typ = "dir" + case os.ModeSymlink: + typ = "symlink" + case os.ModeNamedPipe: + typ = "pipe" + case os.ModeSocket: + typ = "socket" + case os.ModeDevice: + typ = fmt.Sprintf("block-%d-%d", major, minor) + case os.ModeDevice | os.ModeCharDevice: + typ = fmt.Sprintf("char-%d-%d", major, minor) + default: + typ = "file" + } + unixMode := mode & os.ModePerm + if mode&os.ModeSetuid != 0 { + unixMode |= 0o4000 + } + if mode&os.ModeSetgid != 0 { + unixMode |= 0o2000 + } + if mode&os.ModeSticky != 0 { + unixMode |= 0o1000 + } + return fmt.Sprintf("%d:%d:%04o:%s", uid, gid, unixMode, typ) } // GetContainersOverrideXattr will get and decode ContainersOverrideXattr. func GetContainersOverrideXattr(path string) (Stat, error) { - var stat Stat xstat, err := system.Lgetxattr(path, ContainersOverrideXattr) if err != nil { - return stat, err + return Stat{}, err } + return parseOverrideXattr(xstat) // This will fail if (xstat, err) == (nil, nil), i.e. the xattr does not exist. +} +func parseOverrideXattr(xstat []byte) (Stat, error) { + var stat Stat attrs := strings.Split(string(xstat), ":") - if len(attrs) != 3 { - return stat, fmt.Errorf("The number of clons in %s does not equal to 3", + if len(attrs) < 3 { + return stat, fmt.Errorf("The number of parts in %s is less than 3", ContainersOverrideXattr) } @@ -397,47 +437,105 @@ func GetContainersOverrideXattr(path string) (Stat, error) { if err != nil { return stat, fmt.Errorf("Failed to parse UID: %w", err) } - stat.IDs.UID = int(value) - value, err = strconv.ParseUint(attrs[0], 10, 32) + value, err = strconv.ParseUint(attrs[1], 10, 32) if err != nil { return stat, fmt.Errorf("Failed to parse GID: %w", err) } - stat.IDs.GID = int(value) value, err = strconv.ParseUint(attrs[2], 8, 32) if err != nil { return stat, fmt.Errorf("Failed to parse mode: %w", err) } + stat.Mode = os.FileMode(value) & os.ModePerm + if value&0o1000 != 0 { + stat.Mode |= os.ModeSticky + } + if value&0o2000 != 0 { + stat.Mode |= os.ModeSetgid + } + if value&0o4000 != 0 { + stat.Mode |= os.ModeSetuid + } - stat.Mode = os.FileMode(value) - + if len(attrs) > 3 { + typ := attrs[3] + if strings.HasPrefix(typ, "file") { + } else if strings.HasPrefix(typ, "dir") { + stat.Mode |= os.ModeDir + } else if strings.HasPrefix(typ, "symlink") { + stat.Mode |= os.ModeSymlink + } else if strings.HasPrefix(typ, "pipe") { + stat.Mode |= os.ModeNamedPipe + } else if strings.HasPrefix(typ, "socket") { + stat.Mode |= os.ModeSocket + } else if strings.HasPrefix(typ, "block") { + stat.Mode |= os.ModeDevice + stat.Major, stat.Minor, err = parseDevice(typ) + if err != nil { + return stat, err + } + } else if strings.HasPrefix(typ, "char") { + stat.Mode |= os.ModeDevice | os.ModeCharDevice + stat.Major, stat.Minor, err = parseDevice(typ) + if err != nil { + return stat, err + } + } else { + return stat, fmt.Errorf("Invalid file type %s", typ) + } + } return stat, nil } +func parseDevice(typ string) (int, int, error) { + parts := strings.Split(typ, "-") + // If there are more than 3 parts, just ignore them to be forward compatible + if len(parts) < 3 { + return 0, 0, fmt.Errorf("Invalid device type %s", typ) + } + if parts[0] != "block" && parts[0] != "char" { + return 0, 0, fmt.Errorf("Invalid device type %s", typ) + } + major, err := strconv.Atoi(parts[1]) + if err != nil { + return 0, 0, fmt.Errorf("Failed to parse major number: %w", err) + } + minor, err := strconv.Atoi(parts[2]) + if err != nil { + return 0, 0, fmt.Errorf("Failed to parse minor number: %w", err) + } + return major, minor, nil +} + // SetContainersOverrideXattr will encode and set ContainersOverrideXattr. func SetContainersOverrideXattr(path string, stat Stat) error { - value := FormatContainersOverrideXattr(stat.IDs.UID, stat.IDs.GID, int(stat.Mode)) + value := FormatContainersOverrideXattrDevice(stat.IDs.UID, stat.IDs.GID, stat.Mode, stat.Major, stat.Minor) return system.Lsetxattr(path, ContainersOverrideXattr, []byte(value), 0) } func SafeChown(name string, uid, gid int) error { if runtime.GOOS == "darwin" { - var mode os.FileMode = 0o0700 + stat := Stat{ + Mode: os.FileMode(0o0700), + } xstat, err := system.Lgetxattr(name, ContainersOverrideXattr) - if err == nil { - attrs := strings.Split(string(xstat), ":") - if len(attrs) == 3 { - val, err := strconv.ParseUint(attrs[2], 8, 32) - if err == nil { - mode = os.FileMode(val) - } + if err == nil && xstat != nil { + stat, err = parseOverrideXattr(xstat) + if err != nil { + return err + } + } else { + st, err := os.Stat(name) // Ideally we would share this with system.Stat below, but then we would need to convert Mode. + if err != nil { + return err } + stat.Mode = st.Mode() } - value := Stat{IDPair{uid, gid}, mode} - if err = SetContainersOverrideXattr(name, value); err != nil { + stat.IDs = IDPair{UID: uid, GID: gid} + if err = SetContainersOverrideXattr(name, stat); err != nil { return err } uid = os.Getuid() @@ -453,19 +551,24 @@ func SafeChown(name string, uid, gid int) error { func SafeLchown(name string, uid, gid int) error { if runtime.GOOS == "darwin" { - var mode os.FileMode = 0o0700 + stat := Stat{ + Mode: os.FileMode(0o0700), + } xstat, err := system.Lgetxattr(name, ContainersOverrideXattr) - if err == nil { - attrs := strings.Split(string(xstat), ":") - if len(attrs) == 3 { - val, err := strconv.ParseUint(attrs[2], 8, 32) - if err == nil { - mode = os.FileMode(val) - } + if err == nil && xstat != nil { + stat, err = parseOverrideXattr(xstat) + if err != nil { + return err + } + } else { + st, err := os.Lstat(name) // Ideally we would share this with system.Stat below, but then we would need to convert Mode. + if err != nil { + return err } + stat.Mode = st.Mode() } - value := Stat{IDPair{uid, gid}, mode} - if err = SetContainersOverrideXattr(name, value); err != nil { + stat.IDs = IDPair{UID: uid, GID: gid} + if err = SetContainersOverrideXattr(name, stat); err != nil { return err } uid = os.Getuid() diff --git a/vendor/github.com/containers/storage/pkg/ioutils/writers.go b/vendor/github.com/containers/storage/pkg/ioutils/writers.go index ccc7f9c2..0b6d0a7a 100644 --- a/vendor/github.com/containers/storage/pkg/ioutils/writers.go +++ b/vendor/github.com/containers/storage/pkg/ioutils/writers.go @@ -36,9 +36,9 @@ func (r *writeCloserWrapper) Close() error { } // NewWriteCloserWrapper returns a new io.WriteCloser. -func NewWriteCloserWrapper(r io.Writer, closer func() error) io.WriteCloser { +func NewWriteCloserWrapper(w io.Writer, closer func() error) io.WriteCloser { return &writeCloserWrapper{ - Writer: r, + Writer: w, closer: closer, } } diff --git a/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go b/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go index 4ad63451..40cb22b0 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go +++ b/vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go @@ -1,4 +1,4 @@ -//go:build linux && cgo +//go:build linux package loopback diff --git a/vendor/github.com/containers/storage/pkg/loopback/ioctl.go b/vendor/github.com/containers/storage/pkg/loopback/ioctl.go index cf77e523..9acc519c 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/ioctl.go +++ b/vendor/github.com/containers/storage/pkg/loopback/ioctl.go @@ -1,4 +1,4 @@ -//go:build linux && cgo +//go:build linux package loopback diff --git a/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go b/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go index 2a4f5cbc..2cd8fc8b 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go +++ b/vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go @@ -1,21 +1,7 @@ -//go:build linux && cgo +//go:build linux package loopback -/* -#include // FIXME: present only for defines, maybe we can remove it? - -#ifndef LOOP_CTL_GET_FREE - #define LOOP_CTL_GET_FREE 0x4C82 -#endif - -#ifndef LO_FLAGS_PARTSCAN - #define LO_FLAGS_PARTSCAN 8 -#endif - -*/ -import "C" - type loopInfo64 struct { loDevice uint64 /* ioctl r/o */ loInode uint64 /* ioctl r/o */ @@ -34,19 +20,19 @@ type loopInfo64 struct { // IOCTL consts const ( - LoopSetFd = C.LOOP_SET_FD - LoopCtlGetFree = C.LOOP_CTL_GET_FREE - LoopGetStatus64 = C.LOOP_GET_STATUS64 - LoopSetStatus64 = C.LOOP_SET_STATUS64 - LoopClrFd = C.LOOP_CLR_FD - LoopSetCapacity = C.LOOP_SET_CAPACITY + LoopSetFd = 0x4C00 + LoopCtlGetFree = 0x4C82 + LoopGetStatus64 = 0x4C05 + LoopSetStatus64 = 0x4C04 + LoopClrFd = 0x4C01 + LoopSetCapacity = 0x4C07 ) // LOOP consts. const ( - LoFlagsAutoClear = C.LO_FLAGS_AUTOCLEAR - LoFlagsReadOnly = C.LO_FLAGS_READ_ONLY - LoFlagsPartScan = C.LO_FLAGS_PARTSCAN - LoKeySize = C.LO_KEY_SIZE - LoNameSize = C.LO_NAME_SIZE + LoFlagsAutoClear = 0x4C07 + LoFlagsReadOnly = 1 + LoFlagsPartScan = 8 + LoKeySize = 32 + LoNameSize = 64 ) diff --git a/vendor/github.com/containers/storage/pkg/loopback/loopback.go b/vendor/github.com/containers/storage/pkg/loopback/loopback.go index 2d882127..b3527e3a 100644 --- a/vendor/github.com/containers/storage/pkg/loopback/loopback.go +++ b/vendor/github.com/containers/storage/pkg/loopback/loopback.go @@ -1,4 +1,4 @@ -//go:build linux && cgo +//go:build linux package loopback diff --git a/vendor/github.com/containers/storage/pkg/system/extattr_freebsd.go b/vendor/github.com/containers/storage/pkg/system/extattr_freebsd.go new file mode 100644 index 00000000..1314058f --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/system/extattr_freebsd.go @@ -0,0 +1,93 @@ +//go:build freebsd + +package system + +import ( + "os" + "unsafe" + + "golang.org/x/sys/unix" +) + +const ( + EXTATTR_NAMESPACE_EMPTY = unix.EXTATTR_NAMESPACE_EMPTY + EXTATTR_NAMESPACE_USER = unix.EXTATTR_NAMESPACE_USER + EXTATTR_NAMESPACE_SYSTEM = unix.EXTATTR_NAMESPACE_SYSTEM +) + +// ExtattrGetLink retrieves the value of the extended attribute identified by attrname +// in the given namespace and associated with the given path in the file system. +// If the path is a symbolic link, the extended attribute is retrieved from the link itself. +// Returns a []byte slice if the extattr is set and nil otherwise. +func ExtattrGetLink(path string, attrnamespace int, attrname string) ([]byte, error) { + size, errno := unix.ExtattrGetLink(path, attrnamespace, attrname, + uintptr(unsafe.Pointer(nil)), 0) + if errno != nil { + if errno == unix.ENOATTR { + return nil, nil + } + return nil, &os.PathError{Op: "extattr_get_link", Path: path, Err: errno} + } + if size == 0 { + return []byte{}, nil + } + + dest := make([]byte, size) + size, errno = unix.ExtattrGetLink(path, attrnamespace, attrname, + uintptr(unsafe.Pointer(&dest[0])), size) + if errno != nil { + return nil, &os.PathError{Op: "extattr_get_link", Path: path, Err: errno} + } + + return dest[:size], nil +} + +// ExtattrSetLink sets the value of extended attribute identified by attrname +// in the given namespace and associated with the given path in the file system. +// If the path is a symbolic link, the extended attribute is set on the link itself. +func ExtattrSetLink(path string, attrnamespace int, attrname string, data []byte) error { + if len(data) == 0 { + data = []byte{} // ensure non-nil for empty data + } + if _, errno := unix.ExtattrSetLink(path, attrnamespace, attrname, + uintptr(unsafe.Pointer(&data[0])), len(data)); errno != nil { + return &os.PathError{Op: "extattr_set_link", Path: path, Err: errno} + } + + return nil +} + +// ExtattrListLink lists extended attributes associated with the given path +// in the specified namespace. If the path is a symbolic link, the attributes +// are listed from the link itself. +func ExtattrListLink(path string, attrnamespace int) ([]string, error) { + size, errno := unix.ExtattrListLink(path, attrnamespace, + uintptr(unsafe.Pointer(nil)), 0) + if errno != nil { + return nil, &os.PathError{Op: "extattr_list_link", Path: path, Err: errno} + } + if size == 0 { + return []string{}, nil + } + + dest := make([]byte, size) + size, errno = unix.ExtattrListLink(path, attrnamespace, + uintptr(unsafe.Pointer(&dest[0])), size) + if errno != nil { + return nil, &os.PathError{Op: "extattr_list_link", Path: path, Err: errno} + } + + var attrs []string + for i := 0; i < size; { + // Each attribute is preceded by a single byte length + length := int(dest[i]) + i++ + if i+length > size { + break + } + attrs = append(attrs, string(dest[i:i+length])) + i += length + } + + return attrs, nil +} diff --git a/vendor/github.com/containers/storage/pkg/system/extattr_unsupported.go b/vendor/github.com/containers/storage/pkg/system/extattr_unsupported.go new file mode 100644 index 00000000..07b67357 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/system/extattr_unsupported.go @@ -0,0 +1,24 @@ +//go:build !freebsd + +package system + +const ( + EXTATTR_NAMESPACE_EMPTY = 0 + EXTATTR_NAMESPACE_USER = 0 + EXTATTR_NAMESPACE_SYSTEM = 0 +) + +// ExtattrGetLink is not supported on platforms other than FreeBSD. +func ExtattrGetLink(path string, attrnamespace int, attrname string) ([]byte, error) { + return nil, ErrNotSupportedPlatform +} + +// ExtattrSetLink is not supported on platforms other than FreeBSD. +func ExtattrSetLink(path string, attrnamespace int, attrname string, data []byte) error { + return ErrNotSupportedPlatform +} + +// ExtattrListLink is not supported on platforms other than FreeBSD. +func ExtattrListLink(path string, attrnamespace int) ([]string, error) { + return nil, ErrNotSupportedPlatform +} diff --git a/vendor/github.com/containers/storage/pkg/system/stat_netbsd.go b/vendor/github.com/containers/storage/pkg/system/stat_netbsd.go new file mode 100644 index 00000000..715f05b9 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/system/stat_netbsd.go @@ -0,0 +1,13 @@ +package system + +import "syscall" + +// fromStatT converts a syscall.Stat_t type to a system.Stat_t type +func fromStatT(s *syscall.Stat_t) (*StatT, error) { + return &StatT{size: s.Size, + mode: uint32(s.Mode), + uid: s.Uid, + gid: s.Gid, + rdev: uint64(s.Rdev), + mtim: s.Mtimespec}, nil +} diff --git a/vendor/github.com/containers/storage/pkg/system/xattrs_darwin.go b/vendor/github.com/containers/storage/pkg/system/xattrs_darwin.go index 75275b96..27ada208 100644 --- a/vendor/github.com/containers/storage/pkg/system/xattrs_darwin.go +++ b/vendor/github.com/containers/storage/pkg/system/xattrs_darwin.go @@ -12,7 +12,7 @@ const ( E2BIG unix.Errno = unix.E2BIG // Operation not supported - EOPNOTSUPP unix.Errno = unix.EOPNOTSUPP + ENOTSUP unix.Errno = unix.ENOTSUP ) // Lgetxattr retrieves the value of the extended attribute identified by attr diff --git a/vendor/github.com/containers/storage/pkg/system/xattrs_freebsd.go b/vendor/github.com/containers/storage/pkg/system/xattrs_freebsd.go new file mode 100644 index 00000000..5d653976 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/system/xattrs_freebsd.go @@ -0,0 +1,85 @@ +package system + +import ( + "strings" + + "golang.org/x/sys/unix" +) + +const ( + // Value is larger than the maximum size allowed + E2BIG unix.Errno = unix.E2BIG + + // Operation not supported + ENOTSUP unix.Errno = unix.ENOTSUP + + // Value is too small or too large for maximum size allowed + EOVERFLOW unix.Errno = unix.EOVERFLOW +) + +var ( + namespaceMap = map[string]int{ + "user": EXTATTR_NAMESPACE_USER, + "system": EXTATTR_NAMESPACE_SYSTEM, + } +) + +func xattrToExtattr(xattr string) (namespace int, extattr string, err error) { + namespaceName, extattr, found := strings.Cut(xattr, ".") + if !found { + return -1, "", ENOTSUP + } + + namespace, ok := namespaceMap[namespaceName] + if !ok { + return -1, "", ENOTSUP + } + return namespace, extattr, nil +} + +// Lgetxattr retrieves the value of the extended attribute identified by attr +// and associated with the given path in the file system. +// Returns a []byte slice if the xattr is set and nil otherwise. +func Lgetxattr(path string, attr string) ([]byte, error) { + namespace, extattr, err := xattrToExtattr(attr) + if err != nil { + return nil, err + } + return ExtattrGetLink(path, namespace, extattr) +} + +// Lsetxattr sets the value of the extended attribute identified by attr +// and associated with the given path in the file system. +func Lsetxattr(path string, attr string, value []byte, flags int) error { + if flags != 0 { + // FIXME: Flags are not supported on FreeBSD, but we can implement + // them mimicking the behavior of the Linux implementation. + // See lsetxattr(2) on Linux for more information. + return ENOTSUP + } + + namespace, extattr, err := xattrToExtattr(attr) + if err != nil { + return err + } + return ExtattrSetLink(path, namespace, extattr, value) +} + +// Llistxattr lists extended attributes associated with the given path +// in the file system. +func Llistxattr(path string) ([]string, error) { + attrs := []string{} + + for namespaceName, namespace := range namespaceMap { + namespaceAttrs, err := ExtattrListLink(path, namespace) + if err != nil { + return nil, err + } + + for _, attr := range namespaceAttrs { + attrs = append(attrs, namespaceName+"."+attr) + } + } + + return attrs, nil +} diff --git a/vendor/github.com/containers/storage/pkg/system/xattrs_linux.go b/vendor/github.com/containers/storage/pkg/system/xattrs_linux.go index 6b47c4e7..12462cca 100644 --- a/vendor/github.com/containers/storage/pkg/system/xattrs_linux.go +++ b/vendor/github.com/containers/storage/pkg/system/xattrs_linux.go @@ -12,7 +12,7 @@ const ( E2BIG unix.Errno = unix.E2BIG // Operation not supported - EOPNOTSUPP unix.Errno = unix.EOPNOTSUPP + ENOTSUP unix.Errno = unix.ENOTSUP // Value is too small or too large for maximum size allowed EOVERFLOW unix.Errno = unix.EOVERFLOW diff --git a/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go b/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go index 0ab61f3d..66bf5858 100644 --- a/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go @@ -1,4 +1,4 @@ -//go:build !linux && !darwin +//go:build !linux && !darwin && !freebsd package system @@ -9,7 +9,7 @@ const ( E2BIG syscall.Errno = syscall.Errno(0) // Operation not supported - EOPNOTSUPP syscall.Errno = syscall.Errno(0) + ENOTSUP syscall.Errno = syscall.Errno(0) // Value is too small or too large for maximum size allowed EOVERFLOW syscall.Errno = syscall.Errno(0) diff --git a/vendor/github.com/containers/storage/storage.conf b/vendor/github.com/containers/storage/storage.conf index 96232523..2fff0cec 100644 --- a/vendor/github.com/containers/storage/storage.conf +++ b/vendor/github.com/containers/storage/storage.conf @@ -80,6 +80,25 @@ additionalimagestores = [ # This is a "string bool": "false" | "true" (cannot be native TOML boolean) # convert_images = "false" +# This should ALMOST NEVER be set. +# It allows partial pulls of images without guaranteeing that "partial +# pulls" and non-partial pulls both result in consistent image contents. +# This allows pulling estargz images and early versions of zstd:chunked images; +# otherwise, these layers always use the traditional non-partial pull path. +# +# This option should be enabled EXTREMELY rarely, only if ALL images that could +# EVER be conceivably pulled on this system are GUARANTEED (e.g. using a signature policy) +# to come from a build system trusted to never attack image integrity. +# +# If this consistency enforcement were disabled, malicious images could be built +# in a way designed to evade other audit mechanisms, so presence of most other audit +# mechanisms is not a replacement for the above-mentioned need for all images to come +# from a trusted build system. +# +# As a side effect, enabling this option will also make image IDs unpredictable +# (usually not equal to the traditional value matching the config digest). +# insecure_allow_unpredictable_image_contents = "false" + # Root-auto-userns-user is a user name which can be used to look up one or more UID/GID # ranges in the /etc/subuid and /etc/subgid file. These ranges will be partitioned # to containers configured to create automatically a user namespace. Containers diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index a2d38500..cd1cf861 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -20,6 +20,7 @@ import ( _ "github.com/containers/storage/drivers/register" drivers "github.com/containers/storage/drivers" + "github.com/containers/storage/internal/dedup" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" @@ -166,6 +167,26 @@ type flaggableStore interface { type StoreOptions = types.StoreOptions +type DedupHashMethod = dedup.DedupHashMethod + +const ( + DedupHashInvalid = dedup.DedupHashInvalid + DedupHashCRC = dedup.DedupHashCRC + DedupHashFileSize = dedup.DedupHashFileSize + DedupHashSHA256 = dedup.DedupHashSHA256 +) + +type ( + DedupOptions = dedup.DedupOptions + DedupResult = dedup.DedupResult +) + +// DedupArgs is used to pass arguments to the Dedup command. +type DedupArgs struct { + // Options that are passed directly to the internal/dedup.DedupDirs function. + Options DedupOptions +} + // Store wraps up the various types of file-based stores that we use into a // singleton object that initializes and manages them all together. type Store interface { @@ -589,6 +610,9 @@ type Store interface { // MultiList returns consistent values as of a single point in time. // WARNING: The values may already be out of date by the time they are returned to the caller. MultiList(MultiListOptions) (MultiListResult, error) + + // Dedup deduplicates layers in the store. + Dedup(DedupArgs) (drivers.DedupResult, error) } // AdditionalLayer represents a layer that is contained in the additional layer store @@ -3843,3 +3867,43 @@ func (s *store) MultiList(options MultiListOptions) (MultiListResult, error) { } return out, nil } + +// Dedup deduplicates layers in the store. +func (s *store) Dedup(req DedupArgs) (drivers.DedupResult, error) { + imgs, err := s.Images() + if err != nil { + return drivers.DedupResult{}, err + } + var topLayers []string + for _, i := range imgs { + topLayers = append(topLayers, i.TopLayer) + topLayers = append(topLayers, i.MappedTopLayers...) + } + return writeToLayerStore(s, func(rlstore rwLayerStore) (drivers.DedupResult, error) { + layers := make(map[string]struct{}) + for _, i := range topLayers { + cur := i + for cur != "" { + if _, visited := layers[cur]; visited { + break + } + l, err := rlstore.Get(cur) + if err != nil { + if err == ErrLayerUnknown { + break + } + return drivers.DedupResult{}, err + } + layers[cur] = struct{}{} + cur = l.Parent + } + } + r := drivers.DedupArgs{ + Options: req.Options, + } + for l := range layers { + r.Layers = append(r.Layers, l) + } + return rlstore.dedup(r) + }) +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/CHANGELOG.md b/vendor/github.com/cyphar/filepath-securejoin/CHANGELOG.md index 04b5685a..cb1252b5 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/CHANGELOG.md +++ b/vendor/github.com/cyphar/filepath-securejoin/CHANGELOG.md @@ -6,6 +6,35 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ## +## [0.3.6] - 2024-12-17 ## + +### Compatibility ### +- The minimum Go version requirement for `filepath-securejoin` is now Go 1.18 + (we use generics internally). + + For reference, `filepath-securejoin@v0.3.0` somewhat-arbitrarily bumped the + Go version requirement to 1.21. + + While we did make some use of Go 1.21 stdlib features (and in principle Go + versions <= 1.21 are no longer even supported by upstream anymore), some + downstreams have complained that the version bump has meant that they have to + do workarounds when backporting fixes that use the new `filepath-securejoin` + API onto old branches. This is not an ideal situation, but since using this + library is probably better for most downstreams than a hand-rolled + workaround, we now have compatibility shims that allow us to build on older + Go versions. +- Lower minimum version requirement for `golang.org/x/sys` to `v0.18.0` (we + need the wrappers for `fsconfig(2)`), which should also make backporting + patches to older branches easier. + +## [0.3.5] - 2024-12-06 ## + +### Fixed ### +- `MkdirAll` will now no longer return an `EEXIST` error if two racing + processes are creating the same directory. We will still verify that the path + is a directory, but this will avoid spurious errors when multiple threads or + programs are trying to `MkdirAll` the same path. opencontainers/runc#4543 + ## [0.3.4] - 2024-10-09 ## ### Fixed ### @@ -164,8 +193,10 @@ This is our first release of `github.com/cyphar/filepath-securejoin`, containing a full implementation with a coverage of 93.5% (the only missing cases are the error cases, which are hard to mocktest at the moment). -[Unreleased]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.4...HEAD -[0.3.3]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.3...v0.3.4 +[Unreleased]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.6...HEAD +[0.3.6]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.5...v0.3.6 +[0.3.5]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.4...v0.3.5 +[0.3.4]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.3...v0.3.4 [0.3.3]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.2...v0.3.3 [0.3.2]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.1...v0.3.2 [0.3.1]: https://github.com/cyphar/filepath-securejoin/compare/v0.3.0...v0.3.1 diff --git a/vendor/github.com/cyphar/filepath-securejoin/VERSION b/vendor/github.com/cyphar/filepath-securejoin/VERSION index 42045aca..449d7e73 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/VERSION +++ b/vendor/github.com/cyphar/filepath-securejoin/VERSION @@ -1 +1 @@ -0.3.4 +0.3.6 diff --git a/vendor/github.com/cyphar/filepath-securejoin/gocompat_errors_go120.go b/vendor/github.com/cyphar/filepath-securejoin/gocompat_errors_go120.go new file mode 100644 index 00000000..42452bbf --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/gocompat_errors_go120.go @@ -0,0 +1,18 @@ +//go:build linux && go1.20 + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "fmt" +) + +// wrapBaseError is a helper that is equivalent to fmt.Errorf("%w: %w"), except +// that on pre-1.20 Go versions only errors.Is() works properly (errors.Unwrap) +// is only guaranteed to give you baseErr. +func wrapBaseError(baseErr, extraErr error) error { + return fmt.Errorf("%w: %w", extraErr, baseErr) +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/gocompat_errors_unsupported.go b/vendor/github.com/cyphar/filepath-securejoin/gocompat_errors_unsupported.go new file mode 100644 index 00000000..e7adca3f --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/gocompat_errors_unsupported.go @@ -0,0 +1,38 @@ +//go:build linux && !go1.20 + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "fmt" +) + +type wrappedError struct { + inner error + isError error +} + +func (err wrappedError) Is(target error) bool { + return err.isError == target +} + +func (err wrappedError) Unwrap() error { + return err.inner +} + +func (err wrappedError) Error() string { + return fmt.Sprintf("%v: %v", err.isError, err.inner) +} + +// wrapBaseError is a helper that is equivalent to fmt.Errorf("%w: %w"), except +// that on pre-1.20 Go versions only errors.Is() works properly (errors.Unwrap) +// is only guaranteed to give you baseErr. +func wrapBaseError(baseErr, extraErr error) error { + return wrappedError{ + inner: baseErr, + isError: extraErr, + } +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/gocompat_generics_go121.go b/vendor/github.com/cyphar/filepath-securejoin/gocompat_generics_go121.go new file mode 100644 index 00000000..ddd6fa9a --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/gocompat_generics_go121.go @@ -0,0 +1,32 @@ +//go:build linux && go1.21 + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "slices" + "sync" +) + +func slices_DeleteFunc[S ~[]E, E any](slice S, delFn func(E) bool) S { + return slices.DeleteFunc(slice, delFn) +} + +func slices_Contains[S ~[]E, E comparable](slice S, val E) bool { + return slices.Contains(slice, val) +} + +func slices_Clone[S ~[]E, E any](slice S) S { + return slices.Clone(slice) +} + +func sync_OnceValue[T any](f func() T) func() T { + return sync.OnceValue(f) +} + +func sync_OnceValues[T1, T2 any](f func() (T1, T2)) func() (T1, T2) { + return sync.OnceValues(f) +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/gocompat_generics_unsupported.go b/vendor/github.com/cyphar/filepath-securejoin/gocompat_generics_unsupported.go new file mode 100644 index 00000000..f1e6fe7e --- /dev/null +++ b/vendor/github.com/cyphar/filepath-securejoin/gocompat_generics_unsupported.go @@ -0,0 +1,124 @@ +//go:build linux && !go1.21 + +// Copyright (C) 2024 SUSE LLC. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package securejoin + +import ( + "sync" +) + +// These are very minimal implementations of functions that appear in Go 1.21's +// stdlib, included so that we can build on older Go versions. Most are +// borrowed directly from the stdlib, and a few are modified to be "obviously +// correct" without needing to copy too many other helpers. + +// clearSlice is equivalent to the builtin clear from Go 1.21. +// Copied from the Go 1.24 stdlib implementation. +func clearSlice[S ~[]E, E any](slice S) { + var zero E + for i := range slice { + slice[i] = zero + } +} + +// Copied from the Go 1.24 stdlib implementation. +func slices_IndexFunc[S ~[]E, E any](s S, f func(E) bool) int { + for i := range s { + if f(s[i]) { + return i + } + } + return -1 +} + +// Copied from the Go 1.24 stdlib implementation. +func slices_DeleteFunc[S ~[]E, E any](s S, del func(E) bool) S { + i := slices_IndexFunc(s, del) + if i == -1 { + return s + } + // Don't start copying elements until we find one to delete. + for j := i + 1; j < len(s); j++ { + if v := s[j]; !del(v) { + s[i] = v + i++ + } + } + clearSlice(s[i:]) // zero/nil out the obsolete elements, for GC + return s[:i] +} + +// Similar to the stdlib slices.Contains, except that we don't have +// slices.Index so we need to use slices.IndexFunc for this non-Func helper. +func slices_Contains[S ~[]E, E comparable](s S, v E) bool { + return slices_IndexFunc(s, func(e E) bool { return e == v }) >= 0 +} + +// Copied from the Go 1.24 stdlib implementation. +func slices_Clone[S ~[]E, E any](s S) S { + // Preserve nil in case it matters. + if s == nil { + return nil + } + return append(S([]E{}), s...) +} + +// Copied from the Go 1.24 stdlib implementation. +func sync_OnceValue[T any](f func() T) func() T { + var ( + once sync.Once + valid bool + p any + result T + ) + g := func() { + defer func() { + p = recover() + if !valid { + panic(p) + } + }() + result = f() + f = nil + valid = true + } + return func() T { + once.Do(g) + if !valid { + panic(p) + } + return result + } +} + +// Copied from the Go 1.24 stdlib implementation. +func sync_OnceValues[T1, T2 any](f func() (T1, T2)) func() (T1, T2) { + var ( + once sync.Once + valid bool + p any + r1 T1 + r2 T2 + ) + g := func() { + defer func() { + p = recover() + if !valid { + panic(p) + } + }() + r1, r2 = f() + f = nil + valid = true + } + return func() (T1, T2) { + once.Do(g) + if !valid { + panic(p) + } + return r1, r2 + } +} diff --git a/vendor/github.com/cyphar/filepath-securejoin/lookup_linux.go b/vendor/github.com/cyphar/filepath-securejoin/lookup_linux.go index 290befa1..be81e498 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/lookup_linux.go +++ b/vendor/github.com/cyphar/filepath-securejoin/lookup_linux.go @@ -12,7 +12,6 @@ import ( "os" "path" "path/filepath" - "slices" "strings" "golang.org/x/sys/unix" @@ -113,7 +112,7 @@ func (s *symlinkStack) push(dir *os.File, remainingPath, linkTarget string) erro return nil } // Split the link target and clean up any "" parts. - linkTargetParts := slices.DeleteFunc( + linkTargetParts := slices_DeleteFunc( strings.Split(linkTarget, "/"), func(part string) bool { return part == "" || part == "." }) diff --git a/vendor/github.com/cyphar/filepath-securejoin/mkdir_linux.go b/vendor/github.com/cyphar/filepath-securejoin/mkdir_linux.go index b5f67452..5e559bb7 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/mkdir_linux.go +++ b/vendor/github.com/cyphar/filepath-securejoin/mkdir_linux.go @@ -11,7 +11,6 @@ import ( "fmt" "os" "path/filepath" - "slices" "strings" "golang.org/x/sys/unix" @@ -93,7 +92,7 @@ func MkdirAllHandle(root *os.File, unsafePath string, mode int) (_ *os.File, Err } remainingParts := strings.Split(remainingPath, string(filepath.Separator)) - if slices.Contains(remainingParts, "..") { + if slices_Contains(remainingParts, "..") { // The path contained ".." components after the end of the "real" // components. We could try to safely resolve ".." here but that would // add a bunch of extra logic for something that it's not clear even @@ -119,11 +118,20 @@ func MkdirAllHandle(root *os.File, unsafePath string, mode int) (_ *os.File, Err // NOTE: mkdir(2) will not follow trailing symlinks, so we can safely // create the final component without worrying about symlink-exchange // attacks. - if err := unix.Mkdirat(int(currentDir.Fd()), part, uint32(mode)); err != nil { + // + // If we get -EEXIST, it's possible that another program created the + // directory at the same time as us. In that case, just continue on as + // if we created it (if the created inode is not a directory, the + // following open call will fail). + if err := unix.Mkdirat(int(currentDir.Fd()), part, uint32(mode)); err != nil && !errors.Is(err, unix.EEXIST) { err = &os.PathError{Op: "mkdirat", Path: currentDir.Name() + "/" + part, Err: err} // Make the error a bit nicer if the directory is dead. - if err2 := isDeadInode(currentDir); err2 != nil { - err = fmt.Errorf("%w (%w)", err, err2) + if deadErr := isDeadInode(currentDir); deadErr != nil { + // TODO: Once we bump the minimum Go version to 1.20, we can use + // multiple %w verbs for this wrapping. For now we need to use a + // compatibility shim for older Go versions. + //err = fmt.Errorf("%w (%w)", err, deadErr) + err = wrapBaseError(err, deadErr) } return nil, err } diff --git a/vendor/github.com/cyphar/filepath-securejoin/openat2_linux.go b/vendor/github.com/cyphar/filepath-securejoin/openat2_linux.go index ae3b381e..f7a13e69 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/openat2_linux.go +++ b/vendor/github.com/cyphar/filepath-securejoin/openat2_linux.go @@ -12,12 +12,11 @@ import ( "os" "path/filepath" "strings" - "sync" "golang.org/x/sys/unix" ) -var hasOpenat2 = sync.OnceValue(func() bool { +var hasOpenat2 = sync_OnceValue(func() bool { fd, err := unix.Openat2(unix.AT_FDCWD, ".", &unix.OpenHow{ Flags: unix.O_PATH | unix.O_CLOEXEC, Resolve: unix.RESOLVE_NO_SYMLINKS | unix.RESOLVE_IN_ROOT, diff --git a/vendor/github.com/cyphar/filepath-securejoin/procfs_linux.go b/vendor/github.com/cyphar/filepath-securejoin/procfs_linux.go index 8cc827d7..809a579c 100644 --- a/vendor/github.com/cyphar/filepath-securejoin/procfs_linux.go +++ b/vendor/github.com/cyphar/filepath-securejoin/procfs_linux.go @@ -12,7 +12,6 @@ import ( "os" "runtime" "strconv" - "sync" "golang.org/x/sys/unix" ) @@ -54,7 +53,7 @@ func verifyProcRoot(procRoot *os.File) error { return nil } -var hasNewMountApi = sync.OnceValue(func() bool { +var hasNewMountApi = sync_OnceValue(func() bool { // All of the pieces of the new mount API we use (fsopen, fsconfig, // fsmount, open_tree) were added together in Linux 5.1[1,2], so we can // just check for one of the syscalls and the others should also be @@ -192,11 +191,11 @@ func doGetProcRoot() (*os.File, error) { return procRoot, err } -var getProcRoot = sync.OnceValues(func() (*os.File, error) { +var getProcRoot = sync_OnceValues(func() (*os.File, error) { return doGetProcRoot() }) -var hasProcThreadSelf = sync.OnceValue(func() bool { +var hasProcThreadSelf = sync_OnceValue(func() bool { return unix.Access("/proc/thread-self/", unix.F_OK) == nil }) @@ -265,12 +264,20 @@ func procThreadSelf(procRoot *os.File, subpath string) (_ *os.File, _ procThread Resolve: unix.RESOLVE_BENEATH | unix.RESOLVE_NO_XDEV | unix.RESOLVE_NO_MAGICLINKS, }) if err != nil { - return nil, nil, fmt.Errorf("%w: %w", errUnsafeProcfs, err) + // TODO: Once we bump the minimum Go version to 1.20, we can use + // multiple %w verbs for this wrapping. For now we need to use a + // compatibility shim for older Go versions. + //err = fmt.Errorf("%w: %w", errUnsafeProcfs, err) + return nil, nil, wrapBaseError(err, errUnsafeProcfs) } } else { handle, err = openatFile(procRoot, threadSelf+subpath, unix.O_PATH|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0) if err != nil { - return nil, nil, fmt.Errorf("%w: %w", errUnsafeProcfs, err) + // TODO: Once we bump the minimum Go version to 1.20, we can use + // multiple %w verbs for this wrapping. For now we need to use a + // compatibility shim for older Go versions. + //err = fmt.Errorf("%w: %w", errUnsafeProcfs, err) + return nil, nil, wrapBaseError(err, errUnsafeProcfs) } defer func() { if Err != nil { @@ -289,12 +296,17 @@ func procThreadSelf(procRoot *os.File, subpath string) (_ *os.File, _ procThread return handle, runtime.UnlockOSThread, nil } -var hasStatxMountId = sync.OnceValue(func() bool { +// STATX_MNT_ID_UNIQUE is provided in golang.org/x/sys@v0.20.0, but in order to +// avoid bumping the requirement for a single constant we can just define it +// ourselves. +const STATX_MNT_ID_UNIQUE = 0x4000 + +var hasStatxMountId = sync_OnceValue(func() bool { var ( stx unix.Statx_t // We don't care which mount ID we get. The kernel will give us the // unique one if it is supported. - wantStxMask uint32 = unix.STATX_MNT_ID_UNIQUE | unix.STATX_MNT_ID + wantStxMask uint32 = STATX_MNT_ID_UNIQUE | unix.STATX_MNT_ID ) err := unix.Statx(-int(unix.EBADF), "/", 0, int(wantStxMask), &stx) return err == nil && stx.Mask&wantStxMask != 0 @@ -310,7 +322,7 @@ func getMountId(dir *os.File, path string) (uint64, error) { stx unix.Statx_t // We don't care which mount ID we get. The kernel will give us the // unique one if it is supported. - wantStxMask uint32 = unix.STATX_MNT_ID_UNIQUE | unix.STATX_MNT_ID + wantStxMask uint32 = STATX_MNT_ID_UNIQUE | unix.STATX_MNT_ID ) err := unix.Statx(int(dir.Fd()), path, unix.AT_EMPTY_PATH|unix.AT_SYMLINK_NOFOLLOW, int(wantStxMask), &stx) diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/format.go b/vendor/github.com/vbatts/tar-split/archive/tar/format.go index 1f89d0c5..60977980 100644 --- a/vendor/github.com/vbatts/tar-split/archive/tar/format.go +++ b/vendor/github.com/vbatts/tar-split/archive/tar/format.go @@ -143,6 +143,10 @@ const ( blockSize = 512 // Size of each block in a tar stream nameSize = 100 // Max length of the name field in USTAR format prefixSize = 155 // Max length of the prefix field in USTAR format + + // Max length of a special file (PAX header, GNU long name or link). + // This matches the limit used by libarchive. + maxSpecialFileSize = 1 << 20 ) // blockPadding computes the number of bytes needed to pad offset up to the diff --git a/vendor/github.com/vbatts/tar-split/archive/tar/reader.go b/vendor/github.com/vbatts/tar-split/archive/tar/reader.go index 6a6b3e01..248a7ccb 100644 --- a/vendor/github.com/vbatts/tar-split/archive/tar/reader.go +++ b/vendor/github.com/vbatts/tar-split/archive/tar/reader.go @@ -144,7 +144,7 @@ func (tr *Reader) next() (*Header, error) { continue // This is a meta header affecting the next header case TypeGNULongName, TypeGNULongLink: format.mayOnlyBe(FormatGNU) - realname, err := io.ReadAll(tr) + realname, err := readSpecialFile(tr) if err != nil { return nil, err } @@ -338,7 +338,7 @@ func mergePAX(hdr *Header, paxHdrs map[string]string) (err error) { // parsePAX parses PAX headers. // If an extended header (type 'x') is invalid, ErrHeader is returned func parsePAX(r io.Reader) (map[string]string, error) { - buf, err := io.ReadAll(r) + buf, err := readSpecialFile(r) if err != nil { return nil, err } @@ -889,6 +889,16 @@ func tryReadFull(r io.Reader, b []byte) (n int, err error) { return n, err } +// readSpecialFile is like io.ReadAll except it returns +// ErrFieldTooLong if more than maxSpecialFileSize is read. +func readSpecialFile(r io.Reader) ([]byte, error) { + buf, err := io.ReadAll(io.LimitReader(r, maxSpecialFileSize+1)) + if len(buf) > maxSpecialFileSize { + return nil, ErrFieldTooLong + } + return buf, err +} + // discard skips n bytes in r, reporting an error if unable to do so. func discard(tr *Reader, n int64) error { var seekSkipped, copySkipped int64 diff --git a/vendor/golang.org/x/exp/maps/maps.go b/vendor/golang.org/x/exp/maps/maps.go index ecc0dabb..c25939b9 100644 --- a/vendor/golang.org/x/exp/maps/maps.go +++ b/vendor/golang.org/x/exp/maps/maps.go @@ -5,9 +5,20 @@ // Package maps defines various functions useful with maps of any type. package maps +import "maps" + +// TODO(adonovan): when https://go.dev/issue/32816 is accepted, all of +// these functions except Keys and Values should be annotated +// (provisionally with "//go:fix inline") so that tools can safely and +// automatically replace calls to exp/maps with calls to std maps by +// inlining them. + // Keys returns the keys of the map m. // The keys will be in an indeterminate order. func Keys[M ~map[K]V, K comparable, V any](m M) []K { + // The simplest true equivalent using std is: + // return slices.AppendSeq(make([]K, 0, len(m)), maps.Keys(m)). + r := make([]K, 0, len(m)) for k := range m { r = append(r, k) @@ -18,6 +29,9 @@ func Keys[M ~map[K]V, K comparable, V any](m M) []K { // Values returns the values of the map m. // The values will be in an indeterminate order. func Values[M ~map[K]V, K comparable, V any](m M) []V { + // The simplest true equivalent using std is: + // return slices.AppendSeq(make([]V, 0, len(m)), maps.Values(m)). + r := make([]V, 0, len(m)) for _, v := range m { r = append(r, v) @@ -28,50 +42,24 @@ func Values[M ~map[K]V, K comparable, V any](m M) []V { // Equal reports whether two maps contain the same key/value pairs. // Values are compared using ==. func Equal[M1, M2 ~map[K]V, K, V comparable](m1 M1, m2 M2) bool { - if len(m1) != len(m2) { - return false - } - for k, v1 := range m1 { - if v2, ok := m2[k]; !ok || v1 != v2 { - return false - } - } - return true + return maps.Equal(m1, m2) } // EqualFunc is like Equal, but compares values using eq. // Keys are still compared with ==. func EqualFunc[M1 ~map[K]V1, M2 ~map[K]V2, K comparable, V1, V2 any](m1 M1, m2 M2, eq func(V1, V2) bool) bool { - if len(m1) != len(m2) { - return false - } - for k, v1 := range m1 { - if v2, ok := m2[k]; !ok || !eq(v1, v2) { - return false - } - } - return true + return maps.EqualFunc(m1, m2, eq) } // Clear removes all entries from m, leaving it empty. func Clear[M ~map[K]V, K comparable, V any](m M) { - for k := range m { - delete(m, k) - } + clear(m) } // Clone returns a copy of m. This is a shallow clone: // the new keys and values are set using ordinary assignment. func Clone[M ~map[K]V, K comparable, V any](m M) M { - // Preserve nil in case it matters. - if m == nil { - return nil - } - r := make(M, len(m)) - for k, v := range m { - r[k] = v - } - return r + return maps.Clone(m) } // Copy copies all key/value pairs in src adding them to dst. @@ -79,16 +67,10 @@ func Clone[M ~map[K]V, K comparable, V any](m M) M { // the value in dst will be overwritten by the value associated // with the key in src. func Copy[M1 ~map[K]V, M2 ~map[K]V, K comparable, V any](dst M1, src M2) { - for k, v := range src { - dst[k] = v - } + maps.Copy(dst, src) } // DeleteFunc deletes any key/value pairs from m for which del returns true. func DeleteFunc[M ~map[K]V, K comparable, V any](m M, del func(K, V) bool) { - for k, v := range m { - if del(k, v) { - delete(m, k) - } - } + maps.DeleteFunc(m, del) } diff --git a/vendor/modules.txt b/vendor/modules.txt index cfe6cf03..4d15ef72 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -111,8 +111,8 @@ github.com/containerd/log # github.com/containerd/platforms v0.2.1 ## explicit; go 1.20 github.com/containerd/platforms -# github.com/containerd/stargz-snapshotter/estargz v0.15.1 -## explicit; go 1.19 +# github.com/containerd/stargz-snapshotter/estargz v0.16.3 +## explicit; go 1.22.0 github.com/containerd/stargz-snapshotter/estargz github.com/containerd/stargz-snapshotter/estargz/errorutil # github.com/containerd/typeurl/v2 v2.2.0 @@ -303,7 +303,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.56.0 +# github.com/containers/storage v1.57.1 ## explicit; go 1.22.0 github.com/containers/storage github.com/containers/storage/drivers @@ -317,12 +317,14 @@ github.com/containers/storage/drivers/register github.com/containers/storage/drivers/vfs github.com/containers/storage/drivers/windows github.com/containers/storage/drivers/zfs +github.com/containers/storage/internal/dedup github.com/containers/storage/pkg/archive github.com/containers/storage/pkg/chrootarchive github.com/containers/storage/pkg/chunked github.com/containers/storage/pkg/chunked/compressor github.com/containers/storage/pkg/chunked/dump -github.com/containers/storage/pkg/chunked/internal +github.com/containers/storage/pkg/chunked/internal/minimal +github.com/containers/storage/pkg/chunked/internal/path github.com/containers/storage/pkg/chunked/toc github.com/containers/storage/pkg/config github.com/containers/storage/pkg/directory @@ -375,8 +377,8 @@ github.com/crc-org/vfkit/pkg/util # github.com/cyberphone/json-canonicalization v0.0.0-20231217050601-ba74d44ecf5f ## explicit github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer -# github.com/cyphar/filepath-securejoin v0.3.4 -## explicit; go 1.21 +# github.com/cyphar/filepath-securejoin v0.3.6 +## explicit; go 1.18 github.com/cyphar/filepath-securejoin # github.com/digitalocean/go-libvirt v0.0.0-20220804181439-8648fbde413e ## explicit; go 1.15 @@ -804,7 +806,7 @@ github.com/sylabs/sif/v2/pkg/sif # github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 ## explicit github.com/syndtr/gocapability/capability -# github.com/tchap/go-patricia/v2 v2.3.1 +# github.com/tchap/go-patricia/v2 v2.3.2 ## explicit; go 1.16 github.com/tchap/go-patricia/v2/patricia # github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399 @@ -844,7 +846,7 @@ github.com/ulikunitz/xz github.com/ulikunitz/xz/internal/hash github.com/ulikunitz/xz/internal/xlog github.com/ulikunitz/xz/lzma -# github.com/vbatts/tar-split v0.11.6 +# github.com/vbatts/tar-split v0.11.7 ## explicit; go 1.17 github.com/vbatts/tar-split/archive/tar github.com/vbatts/tar-split/tar/asm @@ -929,7 +931,7 @@ golang.org/x/crypto/sha3 golang.org/x/crypto/ssh golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/internal/bcrypt_pbkdf -# golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c +# golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 ## explicit; go 1.22.0 golang.org/x/exp/maps # golang.org/x/net v0.35.0