Skip to content

Commit e48b706

Browse files
tianontianon
committed
Move oci-import (and SBOM) shell code out of meta.jq into an explicit shell script
Also, add a *lot* more validation. This new script/code assumes we're setting `BASHBREW_META_SCRIPTS` to an appropriate value (pointing to the directory of this repository on-disk).
1 parent 4e45b5e commit e48b706

16 files changed

+727
-303
lines changed

‎.test/meta-commands/out.sh

Lines changed: 6 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -97,84 +97,14 @@ docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1
9797

9898
# </pull>
9999
# <build>
100-
export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}"
101-
gitCache="$BASHBREW_CACHE/git"
102-
git init --bare "$gitCache"
103-
_git() { git -C "$gitCache" "$@"; }
104-
_git config gc.auto 0
105-
_commit() { _git rev-parse 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592^{commit}'; }
106-
if ! _commit &> /dev/null; then _git fetch 'https://github.com/docker-library/busybox.git' 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:' || _git fetch 'refs/heads/dist-amd64:'; fi
107-
_commit
108-
mkdir temp
109-
_git archive --format=tar 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:latest/glibc/amd64/' | tar -xvC temp
110-
jq -s '
111-
if length != 1 then
112-
error("unexpected '\''oci-layout'\'' document count: " + length)
113-
else .[0] end
114-
| if .imageLayoutVersion != "1.0.0" then
115-
error("unsupported imageLayoutVersion: " + .imageLayoutVersion)
116-
else . end
117-
' temp/oci-layout > /dev/null
118-
jq -s --tab '
119-
if length != 1 then
120-
error("unexpected '\''index.json'\'' document count: " + length)
121-
else .[0] end
122-
| if .schemaVersion != 2 then
123-
error("unsupported schemaVersion: " + .schemaVersion)
124-
else . end
125-
| if .manifests | length != 1 then
126-
error("expected only one manifests entry, not " + (.manifests | length))
127-
else . end
128-
| .manifests[0] |= (
129-
if .mediaType != "application/vnd.oci.image.manifest.v1+json" then
130-
error("unsupported descriptor mediaType: " + .mediaType)
131-
else . end
132-
| if .size < 0 then
133-
error("invalid descriptor size: " + .size)
134-
else . end
135-
| del(.annotations, .urls)
136-
| .annotations = {"org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.version":"1.36.1","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch"}
137-
)
138-
' temp/index.json > temp/index.json.new
139-
mv temp/index.json.new temp/index.json
100+
build='{"buildId":"191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","build":{"img":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","resolved":{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","size":610,"annotations":{"com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","org.opencontainers.image.version":"1.36.1-glibc"},"platform":{"architecture":"amd64","os":"linux"}}],"annotations":{"org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:70a227928672dffb7d24880bad1a705b527fab650f7503c191e48a209c4a0d10"}},"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","arch":"amd64","parents":{},"resolvedParents":{}},"source":{"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","reproducibleGitChecksum":"17e76ce3a5b47357c5724738db231ed2477c94d43df69ce34ae0871c99f7de78","entries":[{"GitRepo":"https://github.com/docker-library/busybox.git","GitFetch":"refs/heads/dist-amd64","GitCommit":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","Directory":"latest/glibc/amd64","File":"index.json","Builder":"oci-import","SOURCE_DATE_EPOCH":1709081058}],"arches":{"amd64":{"tags":["busybox:1.36.1","busybox:1.36","busybox:1","busybox:stable","busybox:latest","busybox:1.36.1-glibc","busybox:1.36-glibc","busybox:1-glibc","busybox:stable-glibc","busybox:glibc"],"archTags":["amd64/busybox:1.36.1","amd64/busybox:1.36","amd64/busybox:1","amd64/busybox:stable","amd64/busybox:latest","amd64/busybox:1.36.1-glibc","amd64/busybox:1.36-glibc","amd64/busybox:1-glibc","amd64/busybox:stable-glibc","amd64/busybox:glibc"],"froms":["scratch"],"lastStageFrom":"scratch","platformString":"linux/amd64","platform":{"architecture":"amd64","os":"linux"},"parents":{"scratch":{"sourceId":null,"pin":null}}}}}}'
101+
"$BASHBREW_META_SCRIPTS/helpers/oci-import.sh" <<<"$build" temp
140102
# SBOM
141-
originalImageManifest="$(jq -r '.manifests[0].digest' temp/index.json)"
142-
SOURCE_DATE_EPOCH=1709081058 \
143-
docker buildx build --progress=plain \
144-
--load=false \
145-
--provenance=false \
146-
--build-arg BUILDKIT_DOCKERFILE_CHECK=skip=all \
147-
--sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" \
148-
--output 'type=oci,tar=false,dest=sbom' \
149-
--platform 'linux/amd64' \
150-
--build-context "fake=oci-layout://$PWD/temp@$originalImageManifest" \
151-
- <<<'FROM fake'
152-
sbomIndex="$(jq -r '.manifests[0].digest' sbom/index.json)"
153-
shell="$(jq -r --arg originalImageManifest "$originalImageManifest" '
154-
first(
155-
.manifests[]
156-
| select(.annotations["vnd.docker.reference.type"] == "attestation-manifest")
157-
) as $attDesc
158-
| @sh "sbomManifest=\($attDesc.digest)",
159-
@sh "sbomManifestDesc=\(
160-
$attDesc
161-
| .annotations["vnd.docker.reference.digest"] = $originalImageManifest
162-
| tojson
163-
)"
164-
' "sbom/blobs/${sbomIndex/://}")"
165-
eval "$shell"
166-
shell="$(jq -r '
167-
"copyBlobs=( \([ .config.digest, .layers[].digest | @sh ] | join(" ")) )"
168-
' "sbom/blobs/${sbomManifest/://}")"
169-
eval "$shell"
170-
copyBlobs+=( "$sbomManifest" )
171-
for blob in "${copyBlobs[@]}"; do
172-
cp "sbom/blobs/${blob/://}" "temp/blobs/${blob/://}"
173-
done
174-
jq -r --argjson sbomManifestDesc "$sbomManifestDesc" '.manifests += [ $sbomManifestDesc ]' temp/index.json > temp/index.json.new
175-
mv temp/index.json.new temp/index.json
103+
mv temp temp.orig
104+
"$BASHBREW_META_SCRIPTS/helpers/oci-sbom.sh" <<<"$build" temp.orig temp
105+
rm -rf temp.orig
176106
# </build>
177107
# <push>
178-
crane push --index temp 'oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f'
108+
crane push temp 'oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f'
179109
rm -rf temp
180110
# </push>

‎.test/oci-import/out.sh

Lines changed: 2 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,2 @@
1-
export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}"
2-
gitCache="$BASHBREW_CACHE/git"
3-
git init --bare "$gitCache"
4-
_git() { git -C "$gitCache" "$@"; }
5-
_git config gc.auto 0
6-
_commit() { _git rev-parse 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592^{commit}'; }
7-
if ! _commit &> /dev/null; then _git fetch 'https://github.com/docker-library/busybox.git' 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:' || _git fetch 'refs/heads/dist-amd64:'; fi
8-
_commit
9-
mkdir temp
10-
_git archive --format=tar 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:latest/glibc/amd64/' | tar -xvC temp
11-
jq -s '
12-
if length != 1 then
13-
error("unexpected '\''oci-layout'\'' document count: " + length)
14-
else .[0] end
15-
| if .imageLayoutVersion != "1.0.0" then
16-
error("unsupported imageLayoutVersion: " + .imageLayoutVersion)
17-
else . end
18-
' temp/oci-layout > /dev/null
19-
jq -s --tab '
20-
if length != 1 then
21-
error("unexpected '\''index.json'\'' document count: " + length)
22-
else .[0] end
23-
| if .schemaVersion != 2 then
24-
error("unsupported schemaVersion: " + .schemaVersion)
25-
else . end
26-
| if .manifests | length != 1 then
27-
error("expected only one manifests entry, not " + (.manifests | length))
28-
else . end
29-
| .manifests[0] |= (
30-
if .mediaType != "application/vnd.oci.image.manifest.v1+json" then
31-
error("unsupported descriptor mediaType: " + .mediaType)
32-
else . end
33-
| if .size < 0 then
34-
error("invalid descriptor size: " + .size)
35-
else . end
36-
| del(.annotations, .urls)
37-
| .annotations = {"org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.version":"1.36.1","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch"}
38-
)
39-
' temp/index.json > temp/index.json.new
40-
mv temp/index.json.new temp/index.json
1+
build='{"buildId":"191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","build":{"img":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","resolved":{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","size":610,"annotations":{"com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","org.opencontainers.image.version":"1.36.1-glibc"},"platform":{"architecture":"amd64","os":"linux"}}],"annotations":{"org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:70a227928672dffb7d24880bad1a705b527fab650f7503c191e48a209c4a0d10"}},"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","arch":"amd64","parents":{},"resolvedParents":{}},"source":{"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","reproducibleGitChecksum":"17e76ce3a5b47357c5724738db231ed2477c94d43df69ce34ae0871c99f7de78","entries":[{"GitRepo":"https://github.com/docker-library/busybox.git","GitFetch":"refs/heads/dist-amd64","GitCommit":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","Directory":"latest/glibc/amd64","File":"index.json","Builder":"oci-import","SOURCE_DATE_EPOCH":1709081058}],"arches":{"amd64":{"tags":["busybox:1.36.1","busybox:1.36","busybox:1","busybox:stable","busybox:latest","busybox:1.36.1-glibc","busybox:1.36-glibc","busybox:1-glibc","busybox:stable-glibc","busybox:glibc"],"archTags":["amd64/busybox:1.36.1","amd64/busybox:1.36","amd64/busybox:1","amd64/busybox:stable","amd64/busybox:latest","amd64/busybox:1.36.1-glibc","amd64/busybox:1.36-glibc","amd64/busybox:1-glibc","amd64/busybox:stable-glibc","amd64/busybox:glibc"],"froms":["scratch"],"lastStageFrom":"scratch","platformString":"linux/amd64","platform":{"architecture":"amd64","os":"linux"},"parents":{"scratch":{"sourceId":null,"pin":null}}}}}}'
2+
"$BASHBREW_META_SCRIPTS/helpers/oci-import.sh" <<<"$build" temp

‎.test/oci-import/temp/blobs/sha256/166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
{
2+
"schemaVersion": 2,
3+
"mediaType": "application/vnd.oci.image.index.v1+json",
4+
"manifests": [
5+
{
6+
"mediaType": "application/vnd.oci.image.manifest.v1+json",
7+
"digest": "sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0",
8+
"size": 610,
9+
"platform": {
10+
"os": "linux",
11+
"architecture": "amd64"
12+
},
13+
"annotations": {
14+
"com.docker.official-images.bashbrew.arch": "amd64",
15+
"org.opencontainers.image.base.name": "scratch",
16+
"org.opencontainers.image.created": "2024-02-28T00:44:18Z",
17+
"org.opencontainers.image.revision": "d0b7d566eb4f1fa9933984e6fc04ab11f08f4592",
18+
"org.opencontainers.image.source": "https://github.com/docker-library/busybox.git",
19+
"org.opencontainers.image.url": "https://hub.docker.com/_/busybox",
20+
"org.opencontainers.image.version": "1.36.1"
21+
}
22+
}
23+
]
24+
}

‎.test/oci-import/temp/index.json

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,9 @@
33
"mediaType": "application/vnd.oci.image.index.v1+json",
44
"manifests": [
55
{
6-
"mediaType": "application/vnd.oci.image.manifest.v1+json",
7-
"digest": "sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0",
8-
"size": 610,
9-
"platform": {
10-
"architecture": "amd64",
11-
"os": "linux"
12-
},
13-
"annotations": {
14-
"org.opencontainers.image.source": "https://github.com/docker-library/busybox.git",
15-
"org.opencontainers.image.revision": "d0b7d566eb4f1fa9933984e6fc04ab11f08f4592",
16-
"org.opencontainers.image.created": "2024-02-28T00:44:18Z",
17-
"org.opencontainers.image.version": "1.36.1",
18-
"org.opencontainers.image.url": "https://hub.docker.com/_/busybox",
19-
"com.docker.official-images.bashbrew.arch": "amd64",
20-
"org.opencontainers.image.base.name": "scratch"
21-
}
6+
"mediaType": "application/vnd.oci.image.index.v1+json",
7+
"digest": "sha256:166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e",
8+
"size": 838
229
}
2310
]
2411
}

‎.test/oci-import/test.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,9 @@ export BASHBREW_META_SCRIPTS=../..
1212
rm -rf temp
1313
source out.sh
1414

15+
# TODO this should be part of "oci-import.sh"
16+
"$BASHBREW_META_SCRIPTS/helpers/oci-validate.sh" temp
17+
1518
# make sure we don't commit the rootfs tarballs
1619
find temp -type f -size '+1k' -print -delete
20+
# TODO rely on .gitignore instead so that when the test finishes, we have a valid + complete OCI layout locally (that we can test push code against, for example)?

‎.test/oci-sort-platforms/test.jq

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ include "oci";
4343
},
4444

4545
# buildkit attestations
46-
# https://github.com/moby/buildkit/blob/5e0fe2793d529209ad52e811129f644d972ea094/docs/attestations/attestation-storage.md#attestation-manifest-descriptor
46+
# https://github.com/moby/buildkit/blob/c6145c2423de48f891862ac02f9b2653864d3c9e/docs/attestations/attestation-storage.md#attestation-manifest-descriptor
4747
{
4848
architecture: "unknown",
4949
os: "unknown",

‎Jenkinsfile.build

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
4747
))
4848
}
4949

50+
env.BASHBREW_META_SCRIPTS = env.WORKSPACE + '/meta/.scripts'
51+
5052
dir('.bin') {
5153
deleteDir()
5254

@@ -80,7 +82,7 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
8082
obj = sh(returnStdout: true, script: '''
8183
[ -n "$BUILD_ID" ]
8284
shell="$(
83-
jq -L.scripts -r '
85+
jq -L"$BASHBREW_META_SCRIPTS" -r '
8486
include "meta";
8587
.[env.BUILD_ID]
8688
| select(needs_build and .build.arch == env.BASHBREW_ARCH) # sanity check

‎deploy.jq

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,10 +57,10 @@ def deploy_objects:
5757
data: {
5858
schemaVersion: 2,
5959
mediaType: (
60-
if $manifests[0]?.mediaType == "application/vnd.docker.distribution.manifest.v2+json" then
61-
"application/vnd.docker.distribution.manifest.list.v2+json"
60+
if $manifests[0].mediaType == media_type_dockerv2_image then
61+
media_type_dockerv2_list
6262
else
63-
"application/vnd.oci.image.index.v1+json"
63+
media_type_oci_index
6464
end
6565
),
6666
manifests: (

0 commit comments

Comments
 (0)