diff --git a/.test/meta-commands/out.sh b/.test/meta-commands/out.sh
index cf8cc9a..bdc4008 100644
--- a/.test/meta-commands/out.sh
+++ b/.test/meta-commands/out.sh
@@ -97,84 +97,14 @@ docker push 'oisupport/staging-windows-amd64:9b405cfa5b88ba65121aabdb95ae90fd2e1
#
#
-export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}"
-gitCache="$BASHBREW_CACHE/git"
-git init --bare "$gitCache"
-_git() { git -C "$gitCache" "$@"; }
-_git config gc.auto 0
-_commit() { _git rev-parse 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592^{commit}'; }
-if ! _commit &> /dev/null; then _git fetch 'https://github.com/docker-library/busybox.git' 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:' || _git fetch 'refs/heads/dist-amd64:'; fi
-_commit
-mkdir temp
-_git archive --format=tar 'd0b7d566eb4f1fa9933984e6fc04ab11f08f4592:latest/glibc/amd64/' | tar -xvC temp
-jq -s '
- if length != 1 then
- error("unexpected '\''oci-layout'\'' document count: " + length)
- else .[0] end
- | if .imageLayoutVersion != "1.0.0" then
- error("unsupported imageLayoutVersion: " + .imageLayoutVersion)
- else . end
-' temp/oci-layout > /dev/null
-jq -s '
- if length != 1 then
- error("unexpected '\''index.json'\'' document count: " + length)
- else .[0] end
- | if .schemaVersion != 2 then
- error("unsupported schemaVersion: " + .schemaVersion)
- else . end
- | if .manifests | length != 1 then
- error("expected only one manifests entry, not " + (.manifests | length))
- else . end
- | .manifests[0] |= (
- if .mediaType != "application/vnd.oci.image.manifest.v1+json" then
- error("unsupported descriptor mediaType: " + .mediaType)
- else . end
- | if .size < 0 then
- error("invalid descriptor size: " + .size)
- else . end
- | del(.annotations, .urls)
- | .annotations = {"org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.version":"1.36.1","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch"}
- )
-' temp/index.json > temp/index.json.new
-mv temp/index.json.new temp/index.json
+build='{"buildId":"191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","build":{"img":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","resolved":{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","size":610,"annotations":{"com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","org.opencontainers.image.version":"1.36.1-glibc"},"platform":{"architecture":"amd64","os":"linux"}}],"annotations":{"org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:70a227928672dffb7d24880bad1a705b527fab650f7503c191e48a209c4a0d10"}},"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","arch":"amd64","parents":{},"resolvedParents":{}},"source":{"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","reproducibleGitChecksum":"17e76ce3a5b47357c5724738db231ed2477c94d43df69ce34ae0871c99f7de78","entries":[{"GitRepo":"https://github.com/docker-library/busybox.git","GitFetch":"refs/heads/dist-amd64","GitCommit":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","Directory":"latest/glibc/amd64","File":"index.json","Builder":"oci-import","SOURCE_DATE_EPOCH":1709081058}],"arches":{"amd64":{"tags":["busybox:1.36.1","busybox:1.36","busybox:1","busybox:stable","busybox:latest","busybox:1.36.1-glibc","busybox:1.36-glibc","busybox:1-glibc","busybox:stable-glibc","busybox:glibc"],"archTags":["amd64/busybox:1.36.1","amd64/busybox:1.36","amd64/busybox:1","amd64/busybox:stable","amd64/busybox:latest","amd64/busybox:1.36.1-glibc","amd64/busybox:1.36-glibc","amd64/busybox:1-glibc","amd64/busybox:stable-glibc","amd64/busybox:glibc"],"froms":["scratch"],"lastStageFrom":"scratch","platformString":"linux/amd64","platform":{"architecture":"amd64","os":"linux"},"parents":{"scratch":{"sourceId":null,"pin":null}}}}}}'
+"$BASHBREW_META_SCRIPTS/helpers/oci-import.sh" <<<"$build" temp
# SBOM
-originalImageManifest="$(jq -r '.manifests[0].digest' temp/index.json)"
-SOURCE_DATE_EPOCH=1709081058 \
- docker buildx build --progress=plain \
- --load=false \
- --provenance=false \
- --build-arg BUILDKIT_DOCKERFILE_CHECK=skip=all \
- --sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR" \
- --output 'type=oci,tar=false,dest=sbom' \
- --platform 'linux/amd64' \
- --build-context "fake=oci-layout://$PWD/temp@$originalImageManifest" \
- - <<<'FROM fake'
-sbomIndex="$(jq -r '.manifests[0].digest' sbom/index.json)"
-shell="$(jq -r --arg originalImageManifest "$originalImageManifest" '
- first(
- .manifests[]
- | select(.annotations["vnd.docker.reference.type"] == "attestation-manifest")
- ) as $attDesc
- | @sh "sbomManifest=\($attDesc.digest)",
- @sh "sbomManifestDesc=\(
- $attDesc
- | .annotations["vnd.docker.reference.digest"] = $originalImageManifest
- | tojson
- )"
-' "sbom/blobs/${sbomIndex/://}")"
-eval "$shell"
-shell="$(jq -r '
- "copyBlobs=( \([ .config.digest, .layers[].digest | @sh ] | join(" ")) )"
-' "sbom/blobs/${sbomManifest/://}")"
-eval "$shell"
-copyBlobs+=( "$sbomManifest" )
-for blob in "${copyBlobs[@]}"; do
- cp "sbom/blobs/${blob/://}" "temp/blobs/${blob/://}"
-done
-jq -r --argjson sbomManifestDesc "$sbomManifestDesc" '.manifests += [ $sbomManifestDesc ]' temp/index.json > temp/index.json.new
-mv temp/index.json.new temp/index.json
+mv temp temp.orig
+"$BASHBREW_META_SCRIPTS/helpers/oci-sbom.sh" <<<"$build" temp.orig temp
+rm -rf temp.orig
#
#
-crane push --index temp 'oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f'
+crane push temp 'oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f'
rm -rf temp
#
diff --git a/.test/oci-import/in.json b/.test/oci-import/in.json
new file mode 120000
index 0000000..06b614e
--- /dev/null
+++ b/.test/oci-import/in.json
@@ -0,0 +1 @@
+../builds.json
\ No newline at end of file
diff --git a/.test/oci-import/out.sh b/.test/oci-import/out.sh
new file mode 100644
index 0000000..cda1d7c
--- /dev/null
+++ b/.test/oci-import/out.sh
@@ -0,0 +1,2 @@
+build='{"buildId":"191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","build":{"img":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f","resolved":{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","size":610,"annotations":{"com.docker.official-images.bashbrew.arch":"amd64","org.opencontainers.image.base.name":"scratch","org.opencontainers.image.created":"2024-02-28T00:44:18Z","org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0","org.opencontainers.image.revision":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","org.opencontainers.image.source":"https://github.com/docker-library/busybox.git","org.opencontainers.image.url":"https://hub.docker.com/_/busybox","org.opencontainers.image.version":"1.36.1-glibc"},"platform":{"architecture":"amd64","os":"linux"}}],"annotations":{"org.opencontainers.image.ref.name":"oisupport/staging-amd64:191402ad0feacf03daf9d52a492207e73ef08b0bd17265043aea13aa27e2bb3f@sha256:70a227928672dffb7d24880bad1a705b527fab650f7503c191e48a209c4a0d10"}},"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","arch":"amd64","parents":{},"resolvedParents":{}},"source":{"sourceId":"df39fa95e66c7e19e56af0f9dfb8b79b15a0422a9b44eb0f16274d3f1f8939a2","reproducibleGitChecksum":"17e76ce3a5b47357c5724738db231ed2477c94d43df69ce34ae0871c99f7de78","entries":[{"GitRepo":"https://github.com/docker-library/busybox.git","GitFetch":"refs/heads/dist-amd64","GitCommit":"d0b7d566eb4f1fa9933984e6fc04ab11f08f4592","Directory":"latest/glibc/amd64","File":"index.json","Builder":"oci-import","SOURCE_DATE_EPOCH":1709081058}],"arches":{"amd64":{"tags":["busybox:1.36.1","busybox:1.36","busybox:1","busybox:stable","busybox:latest","busybox:1.36.1-glibc","busybox:1.36-glibc","busybox:1-glibc","busybox:stable-glibc","busybox:glibc"],"archTags":["amd64/busybox:1.36.1","amd64/busybox:1.36","amd64/busybox:1","amd64/busybox:stable","amd64/busybox:latest","amd64/busybox:1.36.1-glibc","amd64/busybox:1.36-glibc","amd64/busybox:1-glibc","amd64/busybox:stable-glibc","amd64/busybox:glibc"],"froms":["scratch"],"lastStageFrom":"scratch","platformString":"linux/amd64","platform":{"architecture":"amd64","os":"linux"},"parents":{"scratch":{"sourceId":null,"pin":null}}}}}}'
+"$BASHBREW_META_SCRIPTS/helpers/oci-import.sh" <<<"$build" temp
diff --git a/.test/oci-import/temp/blobs/sha256/166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e b/.test/oci-import/temp/blobs/sha256/166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e
new file mode 100644
index 0000000..ce314a6
--- /dev/null
+++ b/.test/oci-import/temp/blobs/sha256/166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e
@@ -0,0 +1,24 @@
+{
+ "schemaVersion": 2,
+ "mediaType": "application/vnd.oci.image.index.v1+json",
+ "manifests": [
+ {
+ "mediaType": "application/vnd.oci.image.manifest.v1+json",
+ "digest": "sha256:4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0",
+ "size": 610,
+ "platform": {
+ "os": "linux",
+ "architecture": "amd64"
+ },
+ "annotations": {
+ "com.docker.official-images.bashbrew.arch": "amd64",
+ "org.opencontainers.image.base.name": "scratch",
+ "org.opencontainers.image.created": "2024-02-28T00:44:18Z",
+ "org.opencontainers.image.revision": "d0b7d566eb4f1fa9933984e6fc04ab11f08f4592",
+ "org.opencontainers.image.source": "https://github.com/docker-library/busybox.git",
+ "org.opencontainers.image.url": "https://hub.docker.com/_/busybox",
+ "org.opencontainers.image.version": "1.36.1"
+ }
+ }
+ ]
+}
diff --git a/.test/oci-import/temp/blobs/sha256/4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0 b/.test/oci-import/temp/blobs/sha256/4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0
new file mode 120000
index 0000000..9530c85
--- /dev/null
+++ b/.test/oci-import/temp/blobs/sha256/4be429a5fbb2e71ae7958bfa558bc637cf3a61baf40a708cb8fff532b39e52d0
@@ -0,0 +1 @@
+../../image-manifest.json
\ No newline at end of file
diff --git a/.test/oci-import/temp/blobs/sha256/7b2699543f22d5b8dc8d66a5873eb246767bca37232dee1e7a3b8c9956bceb0c b/.test/oci-import/temp/blobs/sha256/7b2699543f22d5b8dc8d66a5873eb246767bca37232dee1e7a3b8c9956bceb0c
new file mode 120000
index 0000000..980db35
--- /dev/null
+++ b/.test/oci-import/temp/blobs/sha256/7b2699543f22d5b8dc8d66a5873eb246767bca37232dee1e7a3b8c9956bceb0c
@@ -0,0 +1 @@
+../../rootfs.tar.gz
\ No newline at end of file
diff --git a/.test/oci-import/temp/blobs/sha256/ba5dc23f65d4cc4a4535bce55cf9e63b068eb02946e3422d3587e8ce803b6aab b/.test/oci-import/temp/blobs/sha256/ba5dc23f65d4cc4a4535bce55cf9e63b068eb02946e3422d3587e8ce803b6aab
new file mode 120000
index 0000000..c8bac38
--- /dev/null
+++ b/.test/oci-import/temp/blobs/sha256/ba5dc23f65d4cc4a4535bce55cf9e63b068eb02946e3422d3587e8ce803b6aab
@@ -0,0 +1 @@
+../../image-config.json
\ No newline at end of file
diff --git a/.test/oci-import/temp/image-config.json b/.test/oci-import/temp/image-config.json
new file mode 100644
index 0000000..2a022c0
--- /dev/null
+++ b/.test/oci-import/temp/image-config.json
@@ -0,0 +1,22 @@
+{
+ "config": {
+ "Cmd": [
+ "sh"
+ ]
+ },
+ "created": "2023-05-18T22:34:17Z",
+ "history": [
+ {
+ "created": "2023-05-18T22:34:17Z",
+ "created_by": "BusyBox 1.36.1 (glibc), Debian 12"
+ }
+ ],
+ "rootfs": {
+ "type": "layers",
+ "diff_ids": [
+ "sha256:95c4a60383f7b6eb6f7b8e153a07cd6e896de0476763bef39d0f6cf3400624bd"
+ ]
+ },
+ "architecture": "amd64",
+ "os": "linux"
+}
diff --git a/.test/oci-import/temp/image-manifest.json b/.test/oci-import/temp/image-manifest.json
new file mode 100644
index 0000000..daf4303
--- /dev/null
+++ b/.test/oci-import/temp/image-manifest.json
@@ -0,0 +1,20 @@
+{
+ "schemaVersion": 2,
+ "mediaType": "application/vnd.oci.image.manifest.v1+json",
+ "config": {
+ "mediaType": "application/vnd.oci.image.config.v1+json",
+ "digest": "sha256:ba5dc23f65d4cc4a4535bce55cf9e63b068eb02946e3422d3587e8ce803b6aab",
+ "size": 372
+ },
+ "layers": [
+ {
+ "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+ "digest": "sha256:7b2699543f22d5b8dc8d66a5873eb246767bca37232dee1e7a3b8c9956bceb0c",
+ "size": 2152262
+ }
+ ],
+ "annotations": {
+ "org.opencontainers.image.url": "https://github.com/docker-library/busybox",
+ "org.opencontainers.image.version": "1.36.1-glibc"
+ }
+}
diff --git a/.test/oci-import/temp/index.json b/.test/oci-import/temp/index.json
new file mode 100644
index 0000000..38c5e11
--- /dev/null
+++ b/.test/oci-import/temp/index.json
@@ -0,0 +1,11 @@
+{
+ "schemaVersion": 2,
+ "mediaType": "application/vnd.oci.image.index.v1+json",
+ "manifests": [
+ {
+ "mediaType": "application/vnd.oci.image.index.v1+json",
+ "digest": "sha256:166d2948c01a6ec70e44b073b0a4c56a3d7c4a4b8fd390d9ebfcb16a3ecf658e",
+ "size": 838
+ }
+ ]
+}
diff --git a/.test/oci-import/temp/oci-layout b/.test/oci-import/temp/oci-layout
new file mode 100644
index 0000000..d5e10fd
--- /dev/null
+++ b/.test/oci-import/temp/oci-layout
@@ -0,0 +1 @@
+{"imageLayoutVersion":"1.0.0"}
diff --git a/.test/oci-import/test.jq b/.test/oci-import/test.jq
new file mode 100644
index 0000000..24291fc
--- /dev/null
+++ b/.test/oci-import/test.jq
@@ -0,0 +1,8 @@
+include "meta";
+
+first(.[] | select(normalized_builder == "oci-import"))
+
+| build_command
+
+# TODO find a better way to stop the SBOM bits from being included here
+| sub("(?s)\n+# SBOM.*"; "")
diff --git a/.test/oci-import/test.sh b/.test/oci-import/test.sh
new file mode 100755
index 0000000..24fd8dc
--- /dev/null
+++ b/.test/oci-import/test.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+dir="$(dirname "$BASH_SOURCE")"
+
+set -x
+
+cd "$dir"
+
+export BASHBREW_META_SCRIPTS=../..
+
+rm -rf temp
+source out.sh
+
+# TODO this should be part of "oci-import.sh"
+"$BASHBREW_META_SCRIPTS/helpers/oci-validate.sh" temp
+
+# make sure we don't commit the rootfs tarballs
+find temp -type f -size '+1k' -print -delete
+# TODO rely on .gitignore instead so that when the test finishes, we have a valid + complete OCI layout locally (that we can test push code against, for example)?
diff --git a/.test/oci-sort-platforms/test.jq b/.test/oci-sort-platforms/test.jq
index 7d1f7db..17c45c4 100644
--- a/.test/oci-sort-platforms/test.jq
+++ b/.test/oci-sort-platforms/test.jq
@@ -43,7 +43,7 @@ include "oci";
},
# buildkit attestations
- # https://github.com/moby/buildkit/blob/5e0fe2793d529209ad52e811129f644d972ea094/docs/attestations/attestation-storage.md#attestation-manifest-descriptor
+ # https://github.com/moby/buildkit/blob/c6145c2423de48f891862ac02f9b2653864d3c9e/docs/attestations/attestation-storage.md#attestation-manifest-descriptor
{
architecture: "unknown",
os: "unknown",
diff --git a/.test/test.sh b/.test/test.sh
index e466101..27a60d4 100755
--- a/.test/test.sh
+++ b/.test/test.sh
@@ -279,3 +279,10 @@ fi
# also run our "jq" tests (like generating example commands from the "builds.json" we just generated)
"$dir/jq.sh"
+
+# TODO a new helper to run these by themselves?
+for t in "$dir/"*"/test.sh"; do
+ if [ -x "$t" ]; then
+ "$t"
+ fi
+done
diff --git a/Jenkinsfile.build b/Jenkinsfile.build
index 686f928..216fd55 100644
--- a/Jenkinsfile.build
+++ b/Jenkinsfile.build
@@ -47,6 +47,8 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
))
}
+ env.BASHBREW_META_SCRIPTS = env.WORKSPACE + '/meta/.scripts'
+
dir('.bin') {
deleteDir()
@@ -80,7 +82,7 @@ node('multiarch-' + env.BASHBREW_ARCH) { ansiColor('xterm') {
obj = sh(returnStdout: true, script: '''
[ -n "$BUILD_ID" ]
shell="$(
- jq -L.scripts -r '
+ jq -L"$BASHBREW_META_SCRIPTS" -r '
include "meta";
.[env.BUILD_ID]
| select(needs_build and .build.arch == env.BASHBREW_ARCH) # sanity check
diff --git a/deploy.jq b/deploy.jq
index 64ffdc6..d470b69 100644
--- a/deploy.jq
+++ b/deploy.jq
@@ -57,10 +57,10 @@ def deploy_objects:
data: {
schemaVersion: 2,
mediaType: (
- if $manifests[0]?.mediaType == "application/vnd.docker.distribution.manifest.v2+json" then
- "application/vnd.docker.distribution.manifest.list.v2+json"
+ if $manifests[0].mediaType == media_type_dockerv2_image then
+ media_type_dockerv2_list
else
- "application/vnd.oci.image.index.v1+json"
+ media_type_oci_index
end
),
manifests: (
diff --git a/helpers/oci-import.sh b/helpers/oci-import.sh
new file mode 100755
index 0000000..81661e7
--- /dev/null
+++ b/helpers/oci-import.sh
@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# this is "docker build" but for "Builder: oci-import"
+# https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L90-L91
+
+# usage:
+# .../oci-import.sh temp <<<'{"buildId":"...","build":{...},"source":{"entries":[{"Builder":"oci-import","GitCommit":...},...],...}}'
+
+target="$1"; shift # target directory to put OCI layout into (must not exist!)
+# stdin: JSON of the full "builds.json" object
+
+[ ! -e "$target" ]
+[ -d "$BASHBREW_META_SCRIPTS" ]
+[ -s "$BASHBREW_META_SCRIPTS/oci.jq" ]
+BASHBREW_META_SCRIPTS="$(cd "$BASHBREW_META_SCRIPTS" && pwd -P)"
+
+# TODO come up with clean ways to harden this against path traversal attacks 🤔 (bad symlinks, "File:" values, etc)
+# - perhaps we run the script in a container? (so the impact of attacks declines to essentially zero)
+
+shell="$(jq -L"$BASHBREW_META_SCRIPTS" --slurp --raw-output '
+ include "validate";
+ validate_one
+ | @sh "buildObj=\(tojson)",
+ (
+ .source.entries[0] |
+ @sh "gitRepo=\(.GitRepo)",
+ @sh "gitFetch=\(.GitFetch)",
+ @sh "gitCommit=\(.GitCommit)",
+ @sh "gitArchive=\(.GitCommit + ":" + (.Directory | if . == "." then "" else . + "/" end))",
+ @sh "file=\(.File)",
+ empty # trailing comma
+ )
+')"
+eval "$shell"
+[ -n "$buildObj" ]
+[ -n "$gitRepo" ]
+[ -n "$gitFetch" ]
+[ -n "$gitCommit" ]
+[ -n "$gitArchive" ]
+[ -n "$file" ]
+export buildObj
+
+# "bashbrew fetch" but in Bash (because we have bashbrew, but not the library file -- we could synthesize a library file instead, but six of one half a dozen of another and this avoids the explicit hard bashbrew dependency)
+
+# initialize "~/.cache/bashbrew/git"
+#"gitCache=\"$(bashbrew cat --format '{{ gitCache }}' <(echo 'Maintainers: empty hack (@example)'))\"",
+# https://github.com/docker-library/bashbrew/blob/5152c0df682515cbe7ac62b68bcea4278856429f/cmd/bashbrew/git.go#L52-L80
+export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}"
+gitCache="$BASHBREW_CACHE/git"
+git init --quiet --bare "$gitCache"
+_git() { git -C "$gitCache" "$@"; }
+_git config gc.auto 0
+
+_commit() { _git rev-parse "$gitCommit^{commit}"; }
+if ! _commit &> /dev/null; then
+ _git fetch --quiet "$gitRepo" "$gitCommit:" \
+ || _git fetch --quiet "$gitRepo" "$gitFetch:"
+fi
+_commit > /dev/null
+
+mkdir "$target"
+
+# https://github.com/docker-library/bashbrew/blob/5152c0df682515cbe7ac62b68bcea4278856429f/cmd/bashbrew/git.go#L140-L147 (TODO "bashbrew context" ?)
+_git archive --format=tar "$gitArchive" > "$target/oci.tar"
+tar --extract --file "$target/oci.tar" --directory "$target"
+rm -f "$target/oci.tar"
+
+cd "$target"
+
+# TODO if we normalize everything to an OCI layout, we could have a "standard" script that validates *all* our outputs and not need quite so much here 🤔 (it would even be reasonable to let publishers provide a provenance attestation object like buildkit does, if they so desire, and then we validate that it's roughly something acceptable to us)
+
+# validate oci-layout
+jq -L"$BASHBREW_META_SCRIPTS" --slurp '
+ include "oci";
+ include "validate";
+
+ validate_one
+ | validate_oci_layout_file
+ | empty
+' oci-layout
+
+# validate "File:" (upgrading it to an index if it's not "index.json"), creating a new canonical "index.json" in the process
+jq -L"$BASHBREW_META_SCRIPTS" --slurp --tab '
+ include "oci";
+ include "validate";
+ include "meta";
+
+ validate_one
+
+ # https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L116
+ | if input_filename != "index.json" then
+ {
+ schemaVersion: 2,
+ mediaType: media_type_oci_index,
+ manifests: [ . ],
+ }
+ else . end
+
+ | .mediaType //= media_type_oci_index # TODO index normalize function? just force this to be set/valid instead?
+ | validate_oci_index
+ | validate_length(.manifests; 1) # TODO allow upstream attestation in the future?
+
+ # purge maintainer-provided URLs / annotations (https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L146-L147)
+ # (also purge maintainer-provided "data" fields here, since including that in the index is a bigger conversation/decision)
+ | del(.manifests[].urls, .manifests[].data)
+ | del(.manifests[0].annotations)
+ | if .manifests[1].annotations then # TODO have this mean something 😂 (see TODOs above about attestations)
+ # filter .manifest[1].annotations to *just* the attestation-related annotations
+ .manifests[1].annotations |= with_entries(
+ select(.key | IN(
+ "vnd.docker.reference.type",
+ "vnd.docker.reference.digest",
+ empty # trailing comma
+ ))
+ )
+ else . end
+
+ | (env.buildObj | fromjson) as $build
+
+ # make sure "platform" is correct
+ | .manifests[0].platform = (
+ $build
+ | .source.arches[.build.arch].platform
+ )
+ # TODO .manifests[1].platform ?
+
+ # inject our build annotations
+ | .manifests[0].annotations += (
+ $build
+ | build_annotations(.source.entries[0].GitRepo)
+ )
+ # TODO perhaps, instead, we stop injecting the index annotations via buildkit/buildx and we normalize these two in a separate "inject index annotations" step/script? 🤔
+
+ | normalize_manifest
+' "$file" | tee index.json.new
+mv -f index.json.new index.json
+
+# TODO "crane validate" is definitely interesting here -- it essentially validates all the descriptors recursively, including diff_ids, but it only supports "remote" or "tarball" (which refers to the *old* "docker save" tarball format), so isn't useful here, but we need to do basically that exact work
+
+# now that "index.json" represents the exact index we want to push, let's push it down into a blob and make a new appropriate "index.json" for "crane push"
+# TODO we probably want/need some "traverse/manipulate an OCI layout" helpers ðŸ˜
+mediaType="$(jq --raw-output '.mediaType' index.json)"
+digest="$(sha256sum index.json | cut -d' ' -f1)"
+digest="sha256:$digest"
+size="$(stat --dereference --format '%s' index.json)"
+mv -f index.json "blobs/${digest//://}"
+export mediaType digest size
+jq -L"$BASHBREW_META_SCRIPTS" --null-input --tab '
+ include "oci";
+ {
+ schemaVersion: 2,
+ mediaType: media_type_oci_index,
+ manifests: [ {
+ mediaType: env.mediaType,
+ digest: env.digest,
+ size: (env.size | tonumber),
+ } ],
+ }
+ | normalize_manifest
+' > index.json
diff --git a/helpers/oci-sbom.sh b/helpers/oci-sbom.sh
new file mode 100755
index 0000000..51eea2b
--- /dev/null
+++ b/helpers/oci-sbom.sh
@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# this will trick BuildKit into generating an SBOM for us, then inject it into our OCI layout
+
+# usage:
+# .../oci-sbom.sh input-oci output-oci
+
+input="$1"; shift # input OCI layout (single image)
+output="$1"; shift # output OCI layout
+# stdin: JSON of the full "builds.json" object
+
+[ -n "$BASHBREW_BUILDKIT_SBOM_GENERATOR" ]
+[ -d "$input" ]
+[ ! -e "$output" ]
+[ -d "$BASHBREW_META_SCRIPTS" ]
+[ -s "$BASHBREW_META_SCRIPTS/oci.jq" ]
+input="$(cd "$input" && pwd -P)"
+BASHBREW_META_SCRIPTS="$(cd "$BASHBREW_META_SCRIPTS" && pwd -P)"
+
+shell="$(jq -L"$BASHBREW_META_SCRIPTS" --slurp --raw-output '
+ include "validate";
+ validate_one
+ | @sh "buildObj=\(tojson)",
+ @sh "SOURCE_DATE_EPOCH=\(.source.entries[0].SOURCE_DATE_EPOCH)",
+ @sh "platform=\(.source.arches[.build.arch].platformString)",
+ empty # trailing comma
+')"
+eval "$shell"
+[ -n "$buildObj" ]
+[ -n "$SOURCE_DATE_EPOCH" ]
+[ -n "$platform" ]
+export buildObj
+
+mkdir "$output"
+cd "$output"
+
+imageIndex="$(jq -L"$BASHBREW_META_SCRIPTS" --raw-output '
+ include "oci";
+ include "validate";
+ validate_oci_index
+ | validate_length(.manifests; 1)
+ | validate_IN(.manifests[0].mediaType; media_types_index)
+ | .manifests[0].digest
+' "$input/index.json")"
+
+shell="$(jq -L"$BASHBREW_META_SCRIPTS" --raw-output '
+ include "oci";
+ include "validate";
+ validate_oci_index
+ | validate_length(.manifests; 1) # TODO technically it would be OK if we had provenance here 🤔 (it just is harder to "merge" 2x provenance than to append 1x)
+ | validate_IN(.manifests[0].mediaType; media_types_image)
+ # TODO should we pull "$platform" from .manifests[0].platform instead of the build object above? (making the build object input optional would make this script easier to test by hand; so maybe just if we did not get it from build?)
+ | @sh "export imageManifest=\(.manifests[0].digest)",
+ empty # trailing comma
+' "$input/blobs/${imageIndex/://}")"
+eval "$shell"
+
+copyBlobs=( "$imageManifest" )
+shell="$(jq -L"$BASHBREW_META_SCRIPTS" --raw-output '
+ include "oci";
+ validate_oci_image
+ | "copyBlobs+=( \(
+ [
+ .config.digest,
+ .layers[].digest
+ | @sh
+ ]
+ | join(" ")
+ ) )"
+' "$input/blobs/${imageManifest/://}")"
+eval "$shell"
+
+args=(
+ --progress=plain
+ --load=false --provenance=false # explicitly disable a few features we want to avoid
+ --build-arg BUILDKIT_DOCKERFILE_CHECK=skip=all # disable linting (https://github.com/moby/buildkit/pull/4962)
+ --sbom=generator="$BASHBREW_BUILDKIT_SBOM_GENERATOR"
+ --output "type=oci,tar=false,dest=."
+ # TODO also add appropriate "--tag" lines (which would give us a mostly correct "subject" block in the generated SBOM, but we'd then need to replace instances of $sbomImageManifest with $imageManifest for their values to be correct)
+ --platform "$platform"
+ --build-context "fake=oci-layout://$input@$imageManifest"
+ '-'
+)
+docker buildx build "${args[@]}" <<<'FROM fake'
+
+for blob in "${copyBlobs[@]}"; do
+ cp --force --dereference --link "$input/blobs/${blob/://}" "blobs/${blob/://}"
+done
+
+sbomIndex="$(jq -L"$BASHBREW_META_SCRIPTS" --raw-output '
+ include "oci";
+ include "validate";
+ validate_oci_index
+ | validate_length(.manifests; 1)
+ | validate_IN(.manifests[0].mediaType; media_types_index)
+ | .manifests[0].digest
+' index.json)"
+
+shell="$(jq -L"$BASHBREW_META_SCRIPTS" --raw-output '
+ include "oci";
+ include "validate";
+ validate_oci_index
+ | validate_length(.manifests; 2)
+ | validate_IN(.manifests[].mediaType; media_types_image)
+ | validate_IN(.manifests[1].annotations["vnd.docker.reference.type"]; "attestation-manifest")
+ | .manifests[0].digest as $fakeImageDigest
+ | validate_IN(.manifests[1].annotations["vnd.docker.reference.digest"]; $fakeImageDigest)
+ | @sh "sbomManifest=\(.manifests[1].digest)",
+ # TODO (see "--tag" TODO above) @sh "sbomImageManifest=\(.manifests[0].digest)",
+ @sh "export sbomManifestDesc=\(
+ .manifests[1]
+ | .annotations["vnd.docker.reference.digest"] = env.imageManifest
+ | tojson
+ )",
+ empty # trailing comma
+' "blobs/${sbomIndex/://}")"
+eval "$shell"
+
+jq -L"$BASHBREW_META_SCRIPTS" --tab '
+ include "oci";
+ # we already validate this exact object above, so we do not need to revalidate here
+ .manifests[1] = (env.sbomManifestDesc | fromjson) # TODO merge provenance, if applicable (see TODOs above)
+ | normalize_manifest
+' "$input/blobs/${imageIndex/://}" | tee index.json
+
+# (this is an exact copy of the end of "oci-import.sh" ðŸ˜)
+# now that "index.json" represents the exact index we want to push, let's push it down into a blob and make a new appropriate "index.json" for "crane push"
+# TODO we probably want/need some "traverse/manipulate an OCI layout" helpers ðŸ˜
+mediaType="$(jq --raw-output '.mediaType' index.json)"
+digest="$(sha256sum index.json | cut -d' ' -f1)"
+digest="sha256:$digest"
+size="$(stat --dereference --format '%s' index.json)"
+mv -f index.json "blobs/${digest//://}"
+export mediaType digest size
+jq -L"$BASHBREW_META_SCRIPTS" --null-input --tab '
+ include "oci";
+ {
+ schemaVersion: 2,
+ mediaType: media_type_oci_index,
+ manifests: [ {
+ mediaType: env.mediaType,
+ digest: env.digest,
+ size: (env.size | tonumber),
+ } ],
+ }
+ | normalize_manifest
+' > index.json
diff --git a/helpers/oci-validate.sh b/helpers/oci-validate.sh
new file mode 100755
index 0000000..16ed3dc
--- /dev/null
+++ b/helpers/oci-validate.sh
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+# given an OCI image layout (https://github.com/opencontainers/image-spec/blob/v1.1.1/image-layout.md), verifies all descriptors as much as possible (digest matches content, size, some media types, layer diff_ids, etc)
+
+layout="$1"; shift
+
+[ -d "$layout" ]
+[ -d "$BASHBREW_META_SCRIPTS" ]
+[ -s "$BASHBREW_META_SCRIPTS/oci.jq" ]
+BASHBREW_META_SCRIPTS="$(cd "$BASHBREW_META_SCRIPTS" && pwd -P)"
+
+cd "$layout"
+
+# validate oci-layout
+echo 'oci-layout'
+jq -L"$BASHBREW_META_SCRIPTS" --slurp '
+ include "oci";
+ include "validate";
+
+ validate_one
+ | validate_oci_layout_file
+ | empty
+' oci-layout
+
+# TODO this is all rubbish; it needs more thought (the jq functions it invokes are pretty solid now though)
+
+descriptor() {
+ local file="$1"; shift # "blobs/sha256/xxx"
+ echo "blob: $file"
+ local digest="$1"; shift # "sha256:xxx"
+ local size="$1"; shift # "123"
+ local algo="${digest%%:*}" # sha256
+ local hash="${digest#$algo:}" # xxx
+ local diskSize
+ [ "$algo" = 'sha256' ] # TODO error message
+ diskSize="$(stat --dereference --format '%s' "$file")"
+ [ "$size" = "$diskSize" ] # TODO error message
+ "${algo}sum" <<<"$hash *$file" --check --quiet --strict -
+}
+
+images() {
+ echo "image: $*"
+ local shell
+ shell="$(
+ jq -L"$BASHBREW_META_SCRIPTS" --arg expected "$#" --slurp --raw-output '
+ include "validate";
+ include "oci";
+ # TODO technically, this would pass if one file is empty and another file has two documents in it (since it is counting the total), so that is not great, but probably is not a real problem
+ validate_length(.; $expected | tonumber)
+ | map(validate_oci_image)
+ | (
+ (
+ .[].config, .[].layers[]
+ | @sh "descriptor \("blobs/\(.digest | sub(":"; "/"))") \(.digest) \(.size)"
+ # TODO data?
+ ),
+
+ empty # trailing comma
+ )
+ ' "$@"
+ )"
+ eval "$shell"
+}
+
+# TODO pass descriptor values down so we can validate that they match (.mediaType, .artifactType, .platform across *two* levels index->manifest->config), similar to .data
+# TODO disallow urls completely?
+
+indexes() {
+ echo "index: $*"
+ local shell
+ shell="$(
+ jq -L"$BASHBREW_META_SCRIPTS" --arg expected "$#" --slurp --raw-output '
+ include "validate";
+ include "oci";
+ # TODO technically, this would pass if one file is empty and another file has two documents in it (since it is counting the total), so that is not great, but probably is not a real problem
+ validate_length(.; $expected | tonumber)
+ | map(validate_oci_index)
+ | (
+ (
+ .[].manifests[]
+ | @sh "descriptor \("blobs/\(.digest | sub(":"; "/"))") \(.digest) \(.size)"
+ # TODO data?
+ ),
+
+ (
+ [ .[].manifests[] | select(IN(.mediaType; media_types_image)) | .digest ]
+ | if length > 0 then
+ "images \(map("blobs/\(sub(":"; "/"))" | @sh) | join(" "))"
+ else empty end
+ ),
+
+ (
+ [ .[].manifests[] | select(IN(.mediaType; media_types_index)) | .digest ]
+ | if length > 0 then
+ "indexes \(map("blobs/\(sub(":"; "/"))" | @sh) | join(" "))"
+ else empty end
+ ),
+
+ empty # trailing comma
+ )
+ ' "$@"
+ )"
+ eval "$shell"
+}
+
+indexes index.json
diff --git a/jenkins.jq b/jenkins.jq
index d027883..78e0780 100644
--- a/jenkins.jq
+++ b/jenkins.jq
@@ -59,7 +59,7 @@ def get_arch_queue($arch):
needs_build
and .build.arch == $arch
)
- | if .build.arch | IN("amd64", "i386", "windows-amd64") then
+ | if IN(.build.arch; "amd64", "i386", "windows-amd64") then
# "GHA" architectures (anything we add a "gha_payload" to will be run on GHA in the queue)
.gha_payload = (gha_payload | @json)
else . end
diff --git a/meta.jq b/meta.jq
index 69deb2a..bb1c2d9 100644
--- a/meta.jq
+++ b/meta.jq
@@ -148,7 +148,7 @@ def build_command:
"--output " + (
[
"type=oci",
- "dest=temp.tar", # TODO choose/find a good "safe" place to put this (temporarily)
+ "dest=temp.tar",
empty
]
| @csv
@@ -251,118 +251,14 @@ def build_command:
] | join("\n")
elif $builder == "oci-import" then
[
- # initialize "~/.cache/bashbrew/git"
- #"gitCache=\"$(bashbrew cat --format '{{ gitCache }}' <(echo 'Maintainers: empty hack (@example)'))\"",
- # https://github.com/docker-library/bashbrew/blob/5152c0df682515cbe7ac62b68bcea4278856429f/cmd/bashbrew/git.go#L52-L80
- "export BASHBREW_CACHE=\"${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}\"",
- "gitCache=\"$BASHBREW_CACHE/git\"",
- "git init --bare \"$gitCache\"",
- "_git() { git -C \"$gitCache\" \"$@\"; }",
- "_git config gc.auto 0",
- # "bashbrew fetch" but in Bash (because we have bashbrew, but not the library file -- we could synthesize a library file instead, but six of one half a dozen of another)
- @sh "_commit() { _git rev-parse \(.source.entries[0].GitCommit + "^{commit}"); }",
- @sh "if ! _commit &> /dev/null; then _git fetch \(.source.entries[0].GitRepo) \(.source.entries[0].GitCommit + ":") || _git fetch \(.source.entries[0].GitFetch + ":"); fi",
- "_commit",
-
- # TODO figure out a good, safe place to store our temporary build/push directory (maybe this is fine? we do it for buildx build too)
- "mkdir temp",
- # https://github.com/docker-library/bashbrew/blob/5152c0df682515cbe7ac62b68bcea4278856429f/cmd/bashbrew/git.go#L140-L147 (TODO "bashbrew context" ?)
- @sh "_git archive --format=tar \(.source.entries[0].GitCommit + ":" + (.source.entries[0].Directory | if . == "." then "" else . + "/" end)) | tar -xvC temp",
-
- # validate oci-layout file (https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L104-L112)
- @sh "jq -s \("
- if length != 1 then
- error(\"unexpected 'oci-layout' document count: \" + length)
- else .[0] end
- | if .imageLayoutVersion != \"1.0.0\" then
- error(\"unsupported imageLayoutVersion: \" + .imageLayoutVersion)
- else . end
- " | unindent_and_decomment_jq(3)) temp/oci-layout > /dev/null",
-
- # https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L116
- if .source.entries[0].File != "index.json" then
- @sh "jq -s \("{ schemaVersion: 2, manifests: . }") \("./" + .source.entries[0].File) > temp/index.json"
- else empty end,
-
- @sh "jq -s \("
- if length != 1 then
- error(\"unexpected 'index.json' document count: \" + length)
- else .[0] end
-
- # https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L117-L127
- | if .schemaVersion != 2 then
- error(\"unsupported schemaVersion: \" + .schemaVersion)
- else . end
- # TODO check .mediaType ? (technically optional, but does not have to be *and* shouldn't be); https://github.com/moby/buildkit/issues/4595
- | if .manifests | length != 1 then
- error(\"expected only one manifests entry, not \" + (.manifests | length))
- else . end
-
- | .manifests[0] |= (
- # https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L135-L144
- if .mediaType != \"application/vnd.oci.image.manifest.v1+json\" then
- error(\"unsupported descriptor mediaType: \" + .mediaType)
- else . end
- # TODO validate .digest somehow (`crane validate`? see below) - would also be good to validate all descriptors recursively (not sure if `crane push` does that)
- | if .size < 0 then
- error(\"invalid descriptor size: \" + .size)
- else . end
-
- # purge maintainer-provided URLs / annotations (https://github.com/docker-library/bashbrew/blob/4e0ea8d8aba49d54daf22bd8415fabba65dc83ee/cmd/bashbrew/oci-builder.go#L146-L147)
- | del(.annotations, .urls)
-
- # inject our annotations
- | .annotations = \(build_annotations(.source.entries[0].GitRepo) | @json)
- )
- " | unindent_and_decomment_jq(3)) temp/index.json > temp/index.json.new",
- "mv temp/index.json.new temp/index.json",
-
- # TODO consider / check what "crane validate" does and if it would be appropriate here
+ @sh "build=\(tojson)",
+ "\"$BASHBREW_META_SCRIPTS/helpers/oci-import.sh\" <<<\"$build\" temp",
if build_should_sbom then
- # we'll trick BuildKit into generating an SBOM for us, then inject it into our OCI layout
"# SBOM",
- "originalImageManifest=\"$(jq -r '.manifests[0].digest' temp/index.json)\"",
- (
- [
- @sh "SOURCE_DATE_EPOCH=\(.source.entries[0].SOURCE_DATE_EPOCH)",
- "docker buildx build --progress=plain",
- "--load=false", "--provenance=false", # explicitly disable a few features we want to avoid
- "--build-arg BUILDKIT_DOCKERFILE_CHECK=skip=all", # disable linting (https://github.com/moby/buildkit/pull/4962)
- "--sbom=generator=\"$BASHBREW_BUILDKIT_SBOM_GENERATOR\"",
- "--output 'type=oci,tar=false,dest=sbom'",
- # TODO also add appropriate "--tag" lines (which would give us a mostly correct "subject" block in the generated SBOM, but we'd then need to replace instances of ${sbomImageManifest#*:} with ${originalImageManifest#*:} for their values to be correct)
- @sh "--platform \(.source.arches[.build.arch].platformString)",
- "--build-context \"fake=oci-layout://$PWD/temp@$originalImageManifest\"",
- "- <<<'FROM fake'", # note: "<<<" is a bashism (so this output must be invoked via bash)
- empty
- ] | join(" \\\n\t")
- ),
- "sbomIndex=\"$(jq -r '.manifests[0].digest' sbom/index.json)\"",
- @sh "shell=\"$(jq -r --arg originalImageManifest \"$originalImageManifest\" \("
- # https://docs.docker.com/build/attestations/attestation-storage/
- first(
- .manifests[]
- | select(.annotations[\"vnd.docker.reference.type\"] == \"attestation-manifest\")
- ) as $attDesc
- | @sh \"sbomManifest=\\($attDesc.digest)\",
- @sh \"sbomManifestDesc=\\(
- $attDesc
- | .annotations[\"vnd.docker.reference.digest\"] = $originalImageManifest
- | tojson
- )\"
- " | unindent_and_decomment_jq(4)) \"sbom/blobs/${sbomIndex/://}\")\"",
- "eval \"$shell\"",
- @sh "shell=\"$(jq -r \("
- \"copyBlobs=( \\([ .config.digest, .layers[].digest | @sh ] | join(\" \")) )\"
- " | unindent_and_decomment_jq(4)) \"sbom/blobs/${sbomManifest/://}\")\"",
- "eval \"$shell\"",
- "copyBlobs+=( \"$sbomManifest\" )",
- "for blob in \"${copyBlobs[@]}\"; do",
- "\tcp \"sbom/blobs/${blob/://}\" \"temp/blobs/${blob/://}\"",
- "done",
- "jq -r --argjson sbomManifestDesc \"$sbomManifestDesc\" '.manifests += [ $sbomManifestDesc ]' temp/index.json > temp/index.json.new",
- "mv temp/index.json.new temp/index.json",
+ "mv temp temp.orig",
+ "\"$BASHBREW_META_SCRIPTS/helpers/oci-sbom.sh\" <<<\"$build\" temp.orig temp",
+ "rm -rf temp.orig",
empty
else empty end
] | join("\n")
@@ -376,19 +272,12 @@ def push_command:
normalized_builder as $builder
| if $builder == "classic" then
@sh "docker push \(.build.img)"
- elif $builder == "buildkit" then
+ elif IN($builder; "buildkit", "oci-import") then
[
- # "crane push" is easier to get correct than "ctr image import" + "ctr image push", especially with authentication
@sh "crane push temp \(.build.img)",
"rm -rf temp",
empty
] | join("\n")
- elif $builder == "oci-import" then
- [
- @sh "crane push --index temp \(.build.img)",
- "rm -rf temp",
- empty
- ] | join("\n")
else
error("unknown/unimplemented Builder: \($builder)")
end
diff --git a/oci.jq b/oci.jq
index a0cf576..efc3279 100644
--- a/oci.jq
+++ b/oci.jq
@@ -1,6 +1,15 @@
include "sort";
+include "validate";
-# https://github.com/opencontainers/image-spec/blob/v1.1.0/image-index.md#:~:text=generate%20an%20error.-,platform%20object,-This%20OPTIONAL%20property
+# TODO maybe this helper should be part of sort.jq? 👀
+def _sort_by_key(stuff):
+ to_entries
+ | sort_by(.key | stuff)
+ | from_entries
+;
+def _sort_by_key: _sort_by_key(.);
+
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/image-index.md#:~:text=generate%20an%20error.-,platform%20object,-This%20OPTIONAL%20property
# input: OCI "platform" object (see link above)
# output: normalized OCI "platform" object
@@ -10,7 +19,7 @@ def normalize_platform:
# https://github.com/golang/go/blob/e85968670e35fc24987944c56277d80d7884e9cc/src/cmd/dist/build.go#L145-L185
# https://github.com/golang/go/blob/e85968670e35fc24987944c56277d80d7884e9cc/src/internal/buildcfg/cfg.go#L58-L175
# https://github.com/containerd/platforms/blob/db76a43eaea9a004a5f240620f966b0081123884/database.go#L75-L109
- # https://github.com/opencontainers/image-spec/blob/v1.1.0/image-index.md#platform-variants
+ # https://github.com/opencontainers/image-spec/blob/v1.1.1/image-index.md#platform-variants
#"amd64/": "v1", # TODO https://github.com/opencontainers/image-spec/pull/1172
"arm/": "v7",
@@ -20,16 +29,14 @@ def normalize_platform:
}["\(.architecture // "")/\(.variant // "")"]
// .variant
)
- | to_entries
- | sort_by(.key | sort_split_pref([
+ | _sort_by_key(sort_split_pref([
"os",
"architecture",
"variant",
"os.version",
empty # trailing comma hack
]))
- | map(select(.value))
- | from_entries
+ | map_values(select(.))
;
# input: *normalized* OCI "platform" object (see link above)
@@ -45,14 +52,16 @@ def sort_split_platform:
]
;
-# https://github.com/opencontainers/image-spec/blob/v1.1.0/descriptor.md
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/descriptor.md
def normalize_descriptor:
if .platform then
.platform |= normalize_platform
else . end
- | to_entries
- | sort_by(.key | sort_split_pref([
+ | if has("annotations") then
+ .annotations |= _sort_by_key
+ else . end
+ | _sort_by_key(sort_split_pref([
"mediaType",
"artifactType",
"digest",
@@ -65,13 +74,13 @@ def normalize_descriptor:
"data",
empty # trailing comma hack
]))
- | from_entries
;
-# https://github.com/opencontainers/image-spec/blob/v1.1.0/image-index.md#:~:text=manifests%20array%20of%20objects
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/image-index.md#:~:text=manifests%20array%20of%20objects
# input: list of OCI "descriptor" objects (the "manifests" array of an image index; see link above)
# output: the same list, sorted such that attestation manifests are next to their subject
+# https://github.com/moby/buildkit/blob/c6145c2423de48f891862ac02f9b2653864d3c9e/docs/attestations/attestation-storage.md
def sort_attestations:
[ .[].digest ] as $digs
| sort_by(
@@ -88,52 +97,190 @@ def sort_manifests:
| sort_attestations
;
-# https://github.com/opencontainers/image-spec/blob/v1.1.0/image-index.md
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/image-index.md
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/manifest.md
+def normalize_manifest:
+ if has("manifests") then
+ .manifests[] |= normalize_descriptor
+ | .manifests |= sort_manifests
+ else . end
+ | if has("config") then
+ .config |= normalize_descriptor
+ else . end
+ | if has("layers") then
+ .layers[] |= normalize_descriptor
+ else . end
+ | if has("annotations") then
+ .annotations |= _sort_by_key
+ else . end
+ | _sort_by_key(sort_split_pref([
+ "schemaVersion",
+ "mediaType",
+ "artifactType",
+ "manifests", # image index
+ "config", "layers", # image manifest
+ empty # trailing comma hack
+ ]; [
+ "subject",
+ "annotations",
+ empty # trailing comma hack
+ ]))
+;
+
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/media-types.md
+def media_type_oci_index: "application/vnd.oci.image.index.v1+json";
+def media_type_oci_image: "application/vnd.oci.image.manifest.v1+json";
+def media_type_oci_config: "application/vnd.oci.image.config.v1+json";
+def media_type_oci_layer: "application/vnd.oci.image.layer.v1.tar";
+def media_type_oci_layer_gzip: media_type_oci_layer + "+gzip";
+
+# https://github.com/distribution/distribution/blob/v3.0.0/docs/content/spec/manifest-v2-2.md#media-types
+def media_type_dockerv2_list: "application/vnd.docker.distribution.manifest.list.v2+json";
+def media_type_dockerv2_image: "application/vnd.docker.distribution.manifest.v2+json";
+def media_type_dockerv2_config: "application/vnd.docker.container.image.v1+json";
+def media_type_dockerv2_layer: "application/vnd.docker.image.rootfs.diff.tar";
+def media_type_dockerv2_layer_gzip: media_type_dockerv2_layer + ".gzip";
+
+def media_types_index: media_type_oci_index, media_type_dockerv2_list;
+def media_types_image: media_type_oci_image, media_type_dockerv2_image;
+def media_types_config: media_type_oci_config, media_type_dockerv2_config;
+def media_types_layer: media_type_oci_layer, media_type_oci_layer_gzip, media_type_dockerv2_layer, media_type_dockerv2_layer_gzip;
-def validate_oci_index_media_type:
- if . != "application/vnd.oci.image.index.v1+json" then
- error("unsupported index mediaType: \(.)")
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/descriptor.md#digests
+def validate_oci_digest:
+ validate(type == "string"; "digest must be a string")
+ | (capture("(?x)
+ ^
+ (?
+ [a-z0-9]+
+ ( [+._-] [a-z0-9]+ )*
+ )
+ [:]
+ (?
+ [a-zA-Z0-9=_-]+
+ )
+ $
+ ") // null) as $dig
+ | validate(.; $dig; "invalid digest syntax")
+ | {
+ "sha256": [ 64 ],
+ "sha512": [ 128 ],
+ } as $lengths
+ | validate_IN($dig.algorithm; $lengths | keys[])
+ | validate_length($dig.encoded; $lengths[$dig.algorithm][])
+;
+
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/annotations.md#rules
+def validate_oci_annotations_haver:
+ if has("annotations") then
+ validate(.annotations; type == "object"; "if present, annotations must be an object")
+ | validate(.annotations[]; type == "string"; "annotation values must be strings")
else . end
;
-def validate_oci_index:
- if .schemaVersion != 2 then
- error("unsupported index schemaVersion: \(.schemaVersion)")
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/descriptor.md
+def validate_oci_descriptor:
+ validate_IN(type; "object")
+
+ | validate(.mediaType; type == "string"; "mediaType must be a string")
+
+ | validate(.digest; validate_oci_digest)
+
+ | validate(.size; type == "number"; "size must be numeric")
+ | validate(.size; . >= 0; "size must not be negative")
+ | validate(.size; . == floor; "size must be whole")
+ | validate(.size; . == ceil; "size must be whole")
+
+ # TODO urls?
+
+ | validate_oci_annotations_haver
+
+ | if has("data") then
+ validate(.data; type == "string"; "if present, data must be a string")
+ # https://datatracker.ietf.org/doc/html/rfc4648#section-4
+ | validate(.data; test("^[A-Za-z0-9+/]*=*$"); "data must be valid base64")
+ | .size as $size
+ | ($size / 3 | ceil * 4) as $dataSize
+ | validate(.data; length == $dataSize; "given size of \($size), data should be \($dataSize) characters long (with padding), not \(length)")
+ # someday, maybe we can validate that .data matches .digest here (needs more jq functionality, including and especially the ability to deal with non-UTF8 binary data from base64 and perform sha256 over it)
else . end
- | .mediaType |= if . then # TODO drop this conditional (BuildKit 0.14+): https://github.com/moby/buildkit/issues/4595
- validate_oci_index_media_type
+
+ # TODO artifactType?
+
+ # https://github.com/opencontainers/image-spec/blob/v1.1.1/image-index.md#image-index-property-descriptions
+ | if has("platform") then
+ validate(.platform;
+ validate_IN(type; "object")
+ | validate(.architecture; type == "string" and length > 0)
+ | validate(.os; type == "string" and length > 0)
+ | if has("os.version") then
+ validate(."os.version"; type == "string" and length > 0)
+ else . end
+ | if has("os.features") then
+ validate(."os.features"; type == "array")
+ | validate(."os.features"[]; type == "string")
+ else . end
+ | if has("variant") then
+ validate(.variant; type == "string" and length > 0)
+ else . end
+ | if has("features") then
+ validate(."features"; type == "array")
+ | validate(."features"[]; type == "string")
+ else . end
+ )
else . end
;
-# https://github.com/opencontainers/image-spec/blob/v1.1.0/image-layout.md#oci-layout-file
-def validate_oci_layout_file:
- if .imageLayoutVersion != "1.0.0" then
- error("unsupported imageLayoutVersion: \(.imageLayoutVersion)")
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/manifest.md
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/image-index.md
+def validate_oci_subject_haver:
+ if has("subject") then
+ validate(.subject; validate_oci_descriptor)
else . end
;
-# https://github.com/opencontainers/image-spec/blob/v1.1.0/image-layout.md#indexjson-file
-def validate_oci_layout_index:
- validate_oci_index
- | .manifests |= (
- if length != 1 then
- error("expected only one manifests entry, not \(length)")
- else . end
- | .[0] |= (
- if .size < 0 then
- error("invalid descriptor size: \(.size)")
- else . end
- # TODO validate .digest somehow (`crane validate`?) - would also be good to validate all descriptors recursively
- | .mediaType |= validate_oci_index_media_type
- )
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/image-index.md
+def validate_oci_index:
+ validate_IN(type; "object")
+ | validate_IN(.schemaVersion; 2)
+ | validate_IN(.mediaType; media_types_index) # TODO allow "null" here too? (https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh)
+ # TODO artifactType?
+ | validate(.manifests[];
+ validate_oci_descriptor
+ | validate_IN(.mediaType; media_types_index, media_types_image)
+ | validate(.size; . > 2; "manifest size must be at *least* big enough for {} plus *some* content")
+ # https://github.com/opencontainers/distribution-spec/pull/293#issuecomment-1452780554
+ | validate(.size; . <= 4 * 1024 * 1024; "manifest size must be 4MiB (\(4 * 1024 * 1024)) or less")
)
+ # TODO if any manifest has "vnd.docker.reference.digest", validate the subject exists in the list
+ | validate_oci_subject_haver
+ | validate_oci_annotations_haver
;
-# input: array of 'oci-layout' file contents followed by 'index.json' file contents (`jq -s 'validate_oci_layout' dir/oci-layout dir/index.json`)
-def validate_oci_layout:
- if length != 2 then
- error("unexpected input: expecting single-document 'oci-layout' and 'index.json'")
- else . end
- | .[0] |= validate_oci_layout_file
- | .[1] |= validate_oci_layout_index
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/manifest.md
+def validate_oci_image:
+ validate_IN(type; "object")
+ | validate_IN(.schemaVersion; 2)
+ | validate_IN(.mediaType; media_types_image) # TODO allow "null" here too? (https://github.com/opencontainers/image-spec/security/advisories/GHSA-77vh-xpmg-72qh)
+ # TODO artifactType (but only selectively / certain values)
+ | validate(.config;
+ validate_oci_descriptor
+ | validate(.size; . >= 2; "config must be at *least* big enough for {}")
+ | validate_IN(.mediaType; media_types_config)
+ )
+ | validate(.layers[];
+ validate_oci_descriptor
+ | validate_IN(.mediaType; media_types_layer) # TODO allow "application/vnd.in-toto+json" and friends, but selectively 🤔
+ )
+ | validate_oci_subject_haver
+ | validate_oci_annotations_haver
+;
+
+# https://github.com/opencontainers/image-spec/blob/v1.1.1/image-layout.md#oci-layout-file
+def validate_oci_layout_file:
+ validate_IN(.imageLayoutVersion; "1.0.0")
;
+
+# TODO validate digest, size of blobs (*somewhere*, probably not here - this is all "cheap" validations / version+ordering+format assumption validations)
+# TODO if .data, validate that somehow too (size, digest); https://github.com/jqlang/jq/issues/1116#issuecomment-2515814615
+# TODO also we should validate that the length of every/any manifest is <= 4MiB (https://github.com/opencontainers/distribution-spec/pull/293#issuecomment-1452780554)
diff --git a/registry/annotations.go b/registry/annotations.go
index 3666827..07c16fe 100644
--- a/registry/annotations.go
+++ b/registry/annotations.go
@@ -3,12 +3,12 @@ package registry
const (
AnnotationBashbrewArch = "com.docker.official-images.bashbrew.arch"
- // https://docs.docker.com/build/attestations/attestation-storage/
+ // https://github.com/moby/buildkit/blob/c6145c2423de48f891862ac02f9b2653864d3c9e/docs/attestations/attestation-storage.md
annotationBuildkitReferenceType = "vnd.docker.reference.type"
annotationBuildkitReferenceTypeAttestation = "attestation-manifest"
annotationBuildkitReferenceDigest = "vnd.docker.reference.digest"
- // https://github.com/distribution/distribution/blob/v3.0.0-alpha.1/docs/content/spec/manifest-v2-2.md
+ // https://github.com/distribution/distribution/blob/v3.0.0/docs/content/spec/manifest-v2-2.md
mediaTypeDockerManifestList = "application/vnd.docker.distribution.manifest.list.v2+json"
mediaTypeDockerImageManifest = "application/vnd.docker.distribution.manifest.v2+json"
mediaTypeDockerImageConfig = "application/vnd.docker.container.image.v1+json"
diff --git a/validate.jq b/validate.jq
new file mode 100644
index 0000000..fe8070b
--- /dev/null
+++ b/validate.jq
@@ -0,0 +1,63 @@
+# a set of ~generic validation helpers
+
+# usage: validate(.some.value; . >= 123; "must be 123 or bigger")
+# will also "nest" sanely: validate(.some; validate(.value; . >= 123; "123+"))
+def validate(selector; condition; err):
+ # if "selector" contains something like "$foo", "path($foo)" will break, but emit the first few things (so "path(.foo, $foo, .bar)" will emit ["foo"] before the exception is caught on the second round)
+ [ try path(selector) catch "BORKBORKBORK" ] as $paths
+ | IN($paths[]; "BORKBORKBORK") as $bork
+ | (if $bork then [ selector ] else $paths end) as $data
+ | reduce $data[] as $maybepath (.;
+ (if $bork then $maybepath else getpath($maybepath) end) as $val
+ | try (
+ if $val | condition then . else
+ error("")
+ end
+ ) catch (
+ # invalid .["foo"]["bar"]: ERROR MESSAGE HERE
+ # value: {"baz":"buzz"}
+ error(
+ "\ninvalid "
+ + if $bork then
+ "value"
+ else
+ ".\($maybepath | map("[\(tojson)]") | add // "")"
+ end
+ + ":\n\t\($val | tojson)"
+ + (
+ $val
+ | err
+ | if . and length > 0 then
+ "\n\(.)"
+ else "" end
+ )
+ + (
+ ltrimstr("\n")
+ | if . and length > 0 then "\n\(.)" else "" end
+ )
+ )
+ )
+ )
+;
+def validate(selector; condition):
+ validate(selector; condition; null)
+;
+def validate(condition):
+ validate(.; condition)
+;
+
+# usage: validate_IN(.some[].mediaType; "foo/bar", "baz/buzz")
+def validate_IN(selector; options):
+ validate(selector; IN(options); "valid:\n\t\([ options | tojson ] | join("\n\t"))")
+;
+
+# usage: validate_length(.manifests; 1, 2)
+def validate_length(selector; lengths):
+ validate(selector; IN(length; lengths); "length (\(length)) must be: \([ lengths | tojson ] | join(", "))")
+;
+
+# usage: (jq --slurp) validate_one | .some.thing
+def validate_one:
+ validate_length(.; 1)
+ | .[0]
+;