From 709bc96d3b2a577c948607477531b04557a4ba39 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 16:25:01 +0530 Subject: [PATCH 01/13] fix: select DSSE provenance for release drafts Recognize DSSE provenance entries and fail when no provenance can be found instead of substituting an image-signature entry. Pin the shared release task with corrected attestation instructions. Depends on tektoncd/plumbing#3807. Assisted-by: OmO Signed-off-by: Vibhav Bobade --- tekton/release-pipeline.yaml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index 9518f7034..b146f225d 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -365,7 +365,7 @@ spec: for uuid in ${CANDIDATES}; do KIND=$(curl -fsS "${REKOR_HOST}/api/v1/log/entries/${uuid}" \ | jq -r '.[].body' | base64 -d 2>/dev/null | jq -r '.kind' 2>/dev/null || true) - if [ "${KIND}" = "intoto" ]; then + if [ "${KIND}" = "intoto" ] || [ "${KIND}" = "dsse" ]; then REKOR_UUID="${uuid}" break fi @@ -373,19 +373,8 @@ spec: fi if [ -z "${REKOR_UUID}" ]; then - echo "WARNING: Could not find an in-toto attestation entry; falling back to the transparency annotation" - TRANSPARENCY=$(kubectl get taskrun "${TASKRUN_NAME}" -n "${NAMESPACE}" \ - -o jsonpath='{.metadata.annotations.chains\.tekton\.dev/transparency}' 2>/dev/null || echo "") - if [ -n "${TRANSPARENCY}" ]; then - REKOR_UUID=$(curl -fsS "${TRANSPARENCY}" | jq -r 'keys[0] // empty' || true) - if [ -z "${REKOR_UUID}" ]; then - REKOR_UUID=$(echo "${TRANSPARENCY}" | grep -oE 'logIndex=[0-9]+' | cut -d= -f2) - fi - fi - fi - - if [ -z "${REKOR_UUID}" ]; then - REKOR_UUID="unknown" + echo "ERROR: Could not find an intoto or dsse attestation entry for ${IMAGE_DIGEST}" + exit 1 fi echo "Rekor UUID: ${REKOR_UUID}" printf '%s' "${REKOR_UUID}" > "$(results.rekor-uuid.path)" @@ -491,7 +480,7 @@ spec: - name: url value: https://github.com/tektoncd/plumbing - name: revision - value: 894710e292fcb6b17276af01a2bc318107eef8a7 + value: df071e78c5b1a58867435efc7a9b295688d58969 - name: pathInRepo value: tekton/resources/release/base/github_release_oci.yaml params: From 438582186a1346d6c5a674fb2180c43d848a615c Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 17:51:16 +0530 Subject: [PATCH 02/13] feat: verify existing release artifacts against a trusted signing key Check metadata, bucket copies, image platforms and signed provenance before retaining immutable verification receipts. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- .github/workflows/release-verification.yaml | 23 ++ hack/verify-release-artifacts.sh | 179 +++++++++++++ hack/verify-release-artifacts_test.sh | 264 ++++++++++++++++++++ 3 files changed, 466 insertions(+) create mode 100644 .github/workflows/release-verification.yaml create mode 100644 hack/verify-release-artifacts.sh create mode 100644 hack/verify-release-artifacts_test.sh diff --git a/.github/workflows/release-verification.yaml b/.github/workflows/release-verification.yaml new file mode 100644 index 000000000..0d7b03d54 --- /dev/null +++ b/.github/workflows/release-verification.yaml @@ -0,0 +1,23 @@ +name: Release artifact verification + +on: + pull_request: + paths: + - hack/verify-release-artifacts.sh + - hack/verify-release-artifacts_test.sh + - .github/workflows/release-verification.yaml + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check verifier syntax and behavior + run: | + bash -n hack/verify-release-artifacts.sh hack/verify-release-artifacts_test.sh + bash hack/verify-release-artifacts_test.sh diff --git a/hack/verify-release-artifacts.sh b/hack/verify-release-artifacts.sh new file mode 100644 index 000000000..90848df82 --- /dev/null +++ b/hack/verify-release-artifacts.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fail() { + echo "release verification failed: $*" >&2 + exit 1 +} + +[[ $# == 5 ]] || fail "usage: $0 VERSION EXPECTED_COMMIT RELEASE_ID PUBLIC_KEY_FILE OUTPUT_DIR" +version=$1 +commit=$2 +release_id=$3 +key=$4 +output=$5 +repository=${GITHUB_REPOSITORY:-tektoncd/tekton-kueue} +image_repository=ghcr.io/tektoncd/tekton-kueue +source_url=https://github.com/tektoncd/tekton-kueue + +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "invalid version" +[[ $commit =~ ^[0-9a-f]{40}$ ]] || fail "invalid commit" +[[ $release_id =~ ^[1-9][0-9]*$ ]] || fail "invalid release ID" +[[ $repository =~ ^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$ && + ${repository##*/} != . && ${repository##*/} != .. ]] || fail "invalid GitHub repository" +[[ -n ${GH_TOKEN:-} ]] || fail "GH_TOKEN is required" +[[ -f $key && -s $key && -r $key ]] || fail "a nonempty configured public key file is required" +[[ -n $output && ! -L $output ]] || fail "invalid output directory" +if [[ -e $output ]]; then + [[ -d $output ]] || fail "output must be a directory" + [[ -z $(find "$output" -mindepth 1 -print -quit) ]] || fail "output directory must be empty" +fi +for command in gh jq curl crane cosign openssl awk cmp; do + command -v "$command" >/dev/null || fail "missing command: $command" +done +openssl pkey -pubin -in "$key" -noout >/dev/null 2>&1 || fail "invalid configured public key" +if command -v sha256sum >/dev/null; then + checksum=(sha256sum) +else + command -v shasum >/dev/null || fail "sha256sum or shasum is required" + checksum=(shasum -a 256) +fi + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +gh api --method GET "repos/$repository/releases/$release_id" > "$work/release.json" +jq -e --arg version "$version" --arg commit "$commit" --arg id "$release_id" ' + .id == ($id | tonumber) and .tag_name == $version and .target_commitish == $commit + and (.draft | type == "boolean") and (.assets | type == "array") + and ([.assets[] | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS")] + | length == 3 and (map(.name) | unique | length == 3) + and all(.[]; (.id | type == "number") and .id > 0 and .id == (.id | floor)) + and (map(.id) | unique | length == 3)) +' "$work/release.json" >/dev/null || fail "release metadata does not match candidate" + +for file in release.yaml release.notags.yaml SHA256SUMS; do + asset_id=$(jq -r --arg name "$file" '.assets[] | select(.name == $name) | .id' "$work/release.json") + gh api --method GET -H "Accept: application/octet-stream" \ + "repos/$repository/releases/assets/$asset_id" > "$work/$file" + [[ -s $work/$file ]] || fail "empty asset: $file" +done + +# Parse before hashing: never pass downloaded filenames to a checksum tool. +declare -A seen=() +while IFS= read -r line || [[ -n $line ]]; do + [[ $line =~ ^([0-9a-f]{64})[[:space:]][\ \*](release\.yaml|release\.notags\.yaml)$ ]] || + fail "unsafe or malformed SHA256SUMS" + hash=${BASH_REMATCH[1]} + file=${BASH_REMATCH[2]} + [[ -z ${seen[$file]:-} ]] || fail "duplicate checksum filename" + seen[$file]=1 + actual=$("${checksum[@]}" "$work/$file") + [[ ${actual%% *} == "$hash" ]] || fail "checksum mismatch: $file" +done < "$work/SHA256SUMS" +[[ ${#seen[@]} == 2 ]] || fail "SHA256SUMS must contain exactly the two manifests" +for file in release.yaml release.notags.yaml SHA256SUMS; do + curl --fail --silent --show-error --location \ + "https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$version/$file" \ + --output "$work/bucket-$file" + cmp -s "$work/$file" "$work/bucket-$file" || fail "bucket mismatch: $file" +done +echo "PASS release metadata, checksums and bucket copies" + +# Kustomize emits block YAML with top-level metadata and scalar image fields. +# Require the controller and webhook deployments, not just an image somewhere. +for file in release.yaml release.notags.yaml; do + awk ' + /^---/ { kind = ""; name = "" } + /^kind: / { kind = $2 } + /^metadata:/ { metadata = 1; next } + /^[^ ]/ { metadata = 0 } + metadata && /^ name: / { name = $2 } + $1 == "image:" { print kind "\t" name "\t" $2 } + ' "$work/$file" > "$work/images" + count=0 + controller=0 + webhook=0 + while IFS=$'\t' read -r kind name reference; do + [[ $kind == Deployment ]] || fail "unexpected image outside a Deployment" + case "$name" in + tekton-kueue-controller-manager) controller=$((controller + 1)) ;; + tekton-kueue-webhook) webhook=$((webhook + 1)) ;; + *) fail "unexpected image deployment" ;; + esac + # Both variants must have the same digest; only release.yaml may have the version tag. + prefix=$image_repository + if [[ $file == release.yaml ]]; then + prefix="$prefix:$version" + fi + [[ $reference == "$prefix"@sha256:* ]] || fail "unexpected image repository or tag" + candidate=${reference##*@} + [[ $candidate =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid image digest" + [[ ${digest:-$candidate} == "$candidate" ]] || fail "inconsistent manifest image digests" + digest=$candidate + count=$((count + 1)) + done < "$work/images" + [[ $count == 2 && $controller == 1 && $webhook == 1 ]] || + fail "both controller and webhook images are required" +done +image="$image_repository@$digest" + +crane manifest "$image" > "$work/index.json" +jq -e ' + [.manifests[].platform | .os + "/" + .architecture] | sort == + ["linux/amd64", "linux/arm64", "linux/ppc64le", "linux/s390x"] +' "$work/index.json" >/dev/null || fail "unexpected image platforms" +for platform in linux/amd64 linux/arm64 linux/ppc64le linux/s390x; do + crane config --platform "$platform" "$image" > "$work/config.json" + jq -e --arg source "$source_url" --arg platform "$platform" ' + (.os + "/" + .architecture) == $platform + and .config.Labels["org.opencontainers.image.source"] == $source + and .config.Labels["org.opencontainers.image.url"] == $source + ' "$work/config.json" >/dev/null || fail "incorrect OCI labels or platform: $platform" +done +echo "PASS manifest images, four platforms and OCI labels" + +cosign verify --key "$key" "$image" > "$work/signatures.json" +jq -e -s --arg digest "$digest" ' + [ .[] | if type == "array" then .[] else . end + | select(.critical.image["docker-manifest-digest"] == $digest) ] | length > 0 +' \ + "$work/signatures.json" >/dev/null || fail "no verified image signature" +cosign verify-attestation --key "$key" --type https://slsa.dev/provenance/v1 \ + "$image" > "$work/attestations.json" +# cosign emits verified DSSE envelopes as JSON objects (or an array). +jq -e -s --arg digest "${digest#sha256:}" --arg commit "$commit" ' + def repository: + # Chains uses git+https URIs with an optional ref suffix. + test("^(git\\+)?https://github\\.com/tektoncd/tekton-kueue(\\.git)?(@[^[:space:]]+)?$"); + def source: + (.uri // "" | repository) + and ((.digest.sha1 // .digest.gitCommit) == $commit); + [ .[] | if type == "array" then .[] else . end + | .payload as $payload | $payload | @base64d | fromjson + | select(._type == "https://in-toto.io/Statement/v1" + and .predicateType == "https://slsa.dev/provenance/v1") + | select(any(.subject[]; .digest.sha256 == $digest)) + | select( + any(.predicate.buildDefinition.resolvedDependencies[]?; source) + or (.predicate.buildDefinition.externalParameters.buildConfigSource + | (.repository // "" | repository) and .ref == ("sha1:" + $commit))) + | $payload + ] | if length > 0 then . else error("no matching verified provenance") end +' "$work/attestations.json" > "$work/verified-payloads.json" || + fail "no verified provenance matching source and digest" +echo "PASS configured-key signature and SLSA provenance" + +jq -n --argjson release_id "$release_id" --arg version "$version" --arg commit "$commit" \ + --arg image "$image" --arg digest "$digest" --slurpfile release "$work/release.json" \ + '{release_id: $release_id, version: $version, commit: $commit, image: $image, + digest: $digest, draft: $release[0].draft, + assets: [$release[0].assets[] + | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS") + | {name, id}] | sort_by(.name)}' > "$work/release-info.json" +mkdir -p "$output" +for file in release.yaml release.notags.yaml SHA256SUMS release-info.json verified-payloads.json; do + # Refuse to overwrite even if another process populated the output during verification. + (set -o noclobber; cat "$work/$file" > "$output/$file") +done +echo "PASS $version artifacts saved to $output" diff --git a/hack/verify-release-artifacts_test.sh b/hack/verify-release-artifacts_test.sh new file mode 100644 index 000000000..f3e038824 --- /dev/null +++ b/hack/verify-release-artifacts_test.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +export FIXTURES="$work/fixtures" +mkdir -p "$work/bin" "$FIXTURES" +export PATH="$work/bin:$PATH" +export GH_TOKEN=test-token +unset GITHUB_REPOSITORY +export VERSION=v0.4.3 +export COMMIT=0123456789abcdef0123456789abcdef01234567 +export DIGEST=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export IMAGE="ghcr.io/tektoncd/tekton-kueue@sha256:$DIGEST" + +# A fixed Ed25519 public key; openssl (not a fake) validates this boundary. +cat > "$work/key.pub" <<'EOF' +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo= +-----END PUBLIC KEY----- +EOF +write_artifacts() { +for variant in release.yaml release.notags.yaml; do + reference=$IMAGE + [[ $variant != release.yaml ]] || reference="ghcr.io/tektoncd/tekton-kueue:$VERSION@sha256:$DIGEST" + for deployment in controller-manager webhook; do + name=$deployment + candidate=$reference + case "${SCENARIO:-success}" in + wrong-image-repository) candidate=${candidate/tektoncd\/tekton-kueue/other\/tekton-kueue} ;; + inconsistent-image) [[ $deployment != webhook ]] || candidate=${candidate/$DIGEST/${DIGEST//a/b}} ;; + missing-webhook) [[ $deployment != webhook ]] || continue ;; + wrong-deployment) [[ $deployment != webhook ]] || name=other ;; + tag-in-notags) [[ $variant != release.notags.yaml ]] || candidate="ghcr.io/tektoncd/tekton-kueue:$VERSION@sha256:$DIGEST" ;; + esac + cat < "$FIXTURES/$variant" +done +( + cd "$FIXTURES" + if command -v sha256sum >/dev/null; then + sha256sum release.yaml release.notags.yaml + else + shasum -a 256 release.yaml release.notags.yaml + fi +) > "$FIXTURES/SHA256SUMS" +} + +# Only network/registry/signing executables are replaced. JSON parsing, hashing, +# key parsing, manifest extraction and byte comparisons use the real commands. +cat > "$work/bin/fake" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "${0##*/}" in + gh) + [[ $1 == api && $2 == --method && $3 == GET ]] || exit 90 + if [[ $4 == repos/tektoncd/tekton-kueue/releases/123 ]]; then + if [[ $SCENARIO == malformed-metadata ]]; then printf '{\n'; exit; fi + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {id:123,tag_name:$version,target_commitish:$commit,draft:true,assets:[ + {id:11,name:"release.yaml"},{id:12,name:"release.notags.yaml"},{id:13,name:"SHA256SUMS"}]} + | if $scenario == "published" then .draft = false + elif $scenario == "metadata-id" then .id = 124 + elif $scenario == "metadata-tag" then .tag_name = "v0.4.2" + elif $scenario == "metadata-commit" then .target_commitish = "main" + elif $scenario == "metadata-draft" then .draft = "true" + elif $scenario == "metadata-assets" then .assets = null + elif $scenario == "duplicate-asset" then .assets += [.assets[0]] + elif $scenario == "missing-asset" then .assets |= .[0:2] + elif $scenario == "unsafe-asset-id" then .assets[0].id = "../evil" + elif $scenario == "duplicate-asset-id" then .assets[1].id = 11 + else . end' + else + [[ $4 == -H && $5 == "Accept: application/octet-stream" ]] || exit 90 + case "$6" in + repos/tektoncd/tekton-kueue/releases/assets/11) file=release.yaml ;; + repos/tektoncd/tekton-kueue/releases/assets/12) file=release.notags.yaml ;; + repos/tektoncd/tekton-kueue/releases/assets/13) file=SHA256SUMS ;; + *) exit 90 ;; + esac + if [[ $file == SHA256SUMS ]]; then + case "$SCENARIO" in + checksum) printf '%064d release.yaml\n' 0; exit ;; + traversal) printf '%s ../release.yaml\n' "$DIGEST"; exit ;; + absolute) printf '%s /tmp/release.yaml\n' "$DIGEST"; exit ;; + extra-checksum) cat "$FIXTURES/$file"; printf '%s extra.yaml\n' "$DIGEST"; exit ;; + duplicate-checksum) cat "$FIXTURES/$file" "$FIXTURES/$file"; exit ;; + missing-checksum) awk 'NR == 1' "$FIXTURES/$file"; exit ;; + empty-checksum) exit ;; + esac + fi + cat "$FIXTURES/$file" + fi + ;; + curl) + [[ $# == 7 && $1 == --fail && $2 == --silent && $3 == --show-error && + $4 == --location && $6 == --output ]] || exit 90 + prefix="https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$VERSION/" + [[ $5 == "$prefix"* ]] || exit 90 + file=${5#"$prefix"} + case "$file" in release.yaml|release.notags.yaml|SHA256SUMS) ;; *) exit 90 ;; esac + cat "$FIXTURES/$file" > "$7" + [[ $SCENARIO != "bucket-$file" ]] || printf '\nchanged\n' >> "$7" + ;; + crane) + if [[ $1 == manifest && $2 == "$IMAGE" && $# == 2 ]]; then + jq -n --arg scenario "$SCENARIO" ' + ["amd64","arm64","ppc64le","s390x"] + | if $scenario == "missing-platform" then .[0:3] + elif $scenario == "duplicate-platform" then . + ["amd64"] + elif $scenario == "wrong-platform" then .[3] = "riscv64" + else . end + | {schemaVersion:2, manifests:map({digest:("sha256:" + ("b" * 64)), + platform:{os:"linux",architecture:.}})}' + elif [[ $1 == config && $2 == --platform && $4 == "$IMAGE" && $# == 4 ]]; then + jq -n --arg platform "$3" --arg scenario "$SCENARIO" ' + {os:"linux",architecture:($platform | split("/")[1]),config:{Labels:{ + "org.opencontainers.image.source":"https://github.com/tektoncd/tekton-kueue", + "org.opencontainers.image.url":"https://github.com/tektoncd/tekton-kueue"}}} + | if $platform == "linux/s390x" then + if $scenario == "bad-source-label" then .config.Labels["org.opencontainers.image.source"] = "https://example.com" + elif $scenario == "bad-url-label" then del(.config.Labels["org.opencontainers.image.url"]) + elif $scenario == "bad-config-platform" then .os = "windows" + else . end + else . end' + else exit 90; fi + ;; + cosign) + [[ $2 == --key && -s $3 ]] || exit 90 + case "$1" in + verify) + [[ $# == 4 && $4 == "$IMAGE" ]] || exit 90 + [[ $SCENARIO != signature-failure ]] || exit 1 + [[ $SCENARIO != empty-signature ]] || exit + printf '[{"critical":{"image":{"docker-manifest-digest":"sha256:%s"}}}]\n' "$DIGEST" + ;; + verify-attestation) + [[ $# == 6 && $4 == --type && $5 == https://slsa.dev/provenance/v1 && $6 == "$IMAGE" ]] || exit 90 + [[ $SCENARIO != attestation-failure ]] || exit 1 + [[ $SCENARIO != empty-attestation ]] || exit + if [[ $SCENARIO == malformed-attestation ]]; then + printf '{"payload":"invalid base64"}\n' + exit + fi + jq -n --arg digest "$DIGEST" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {_type:"https://in-toto.io/Statement/v1",predicateType:"https://slsa.dev/provenance/v1", + subject:[{name:"ghcr.io/tektoncd/tekton-kueue",digest:{sha256:$digest}}], + predicate:{buildDefinition:{resolvedDependencies:[ + {uri:"git+https://github.com/tektoncd/tekton-kueue.git@refs/tags/v0.4.3",digest:{sha1:$commit}}]}}} + | if $scenario == "bad-source" then .predicate.buildDefinition.resolvedDependencies[0].uri = "https://github.com/tektoncd/tekton-kueue-evil" + elif $scenario == "bad-commit" then .predicate.buildDefinition.resolvedDependencies[0].digest.sha1 = ("f" * 40) + elif $scenario == "bad-digest" then .subject[0].digest.sha256 = ("b" * 64) + elif $scenario == "bad-predicate" then .predicateType = "https://slsa.dev/provenance/v0.2" + elif $scenario == "build-config-source" then + .predicate.buildDefinition.externalParameters.buildConfigSource = + {repository:"git+https://github.com/tektoncd/tekton-kueue",ref:("sha1:" + $commit),path:"tekton/release-pipeline.yaml"} + | del(.predicate.buildDefinition.resolvedDependencies) + else . end + | if $scenario == "multiple-attestations" then + (.subject[0].digest.sha256 = ("b" * 64)), . + else . end + | {payloadType:"application/vnd.in-toto+json",payload:(tojson | @base64),signatures:[{sig:"verified-by-fake-cosign"}]} + | if $scenario == "array-attestations" then [.] else . end' + ;; + *) exit 90 ;; + esac + ;; + *) exit 90 ;; +esac +EOF +chmod +x "$work/bin/fake" +for executable in gh curl crane cosign; do + ln -s fake "$work/bin/$executable" +done + +run_verifier() { + bash "$root/hack/verify-release-artifacts.sh" \ + "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +} + +passed=0 +for SCENARIO in success published build-config-source multiple-attestations array-attestations \ + checksum traversal absolute extra-checksum duplicate-checksum missing-checksum empty-checksum \ + bucket-release.yaml bucket-release.notags.yaml bucket-SHA256SUMS \ + malformed-metadata metadata-id metadata-tag metadata-commit metadata-draft metadata-assets \ + duplicate-asset missing-asset unsafe-asset-id duplicate-asset-id \ + wrong-image-repository inconsistent-image missing-webhook wrong-deployment tag-in-notags \ + missing-platform duplicate-platform wrong-platform bad-source-label bad-url-label bad-config-platform \ + signature-failure empty-signature attestation-failure empty-attestation malformed-attestation \ + bad-source bad-commit bad-digest bad-predicate; do + export SCENARIO + rm -rf "$work/output" + write_artifacts + status=0 + run_verifier > "$work/log" 2>&1 || status=$? + case "$SCENARIO" in + success|published|build-config-source|multiple-attestations|array-attestations) + if [[ $status != 0 ]]; then cat "$work/log" >&2; exit 1; fi + for file in release.yaml release.notags.yaml SHA256SUMS; do + cmp "$FIXTURES/$file" "$work/output/$file" + done + jq -e --arg commit "$COMMIT" --arg image "$IMAGE" --arg scenario "$SCENARIO" \ + --arg version "$VERSION" --arg digest "sha256:$DIGEST" ' + . == {release_id:123,version:$version,commit:$commit,image:$image,digest:$digest, + draft:($scenario != "published"),assets:[ + {name:"SHA256SUMS",id:13},{name:"release.notags.yaml",id:12}, + {name:"release.yaml",id:11}]}' "$work/output/release-info.json" >/dev/null + jq -e --arg digest "$DIGEST" ' + [.[] | @base64d | fromjson + | select(any(.subject[]; .digest.sha256 == $digest))] + | length > 0' "$work/output/verified-payloads.json" >/dev/null + ;; + *) + [[ $status != 0 && ! -e $work/output/release-info.json ]] || { + echo "unexpected verification success: $SCENARIO" >&2; exit 1; + } + ;; + esac + passed=$((passed + 1)) +done + +export SCENARIO=success +expect_failure() { + if "$@" > "$work/log" 2>&1; then + echo "unexpected boundary acceptance" >&2 + exit 1 + fi + passed=$((passed + 1)) +} +verifier="$root/hack/verify-release-artifacts.sh" +rm -rf "$work/output" +expect_failure bash "$verifier" +expect_failure bash "$verifier" ../v0.4.3 "$COMMIT" 123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "${COMMIT^^}" 123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" ../123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/missing.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "" +expect_failure env GH_TOKEN= bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +expect_failure env GITHUB_REPOSITORY=../repo bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +touch "$work/empty.pub" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/empty.pub" "$work/output" +printf 'not a public key\n' > "$work/invalid.pub" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/invalid.pub" "$work/output" +mkdir "$work/output" +printf 'existing artifact\n' > "$work/output/release.yaml" +expect_failure run_verifier +[[ $(< "$work/output/release.yaml") == "existing artifact" ]] +echo "PASS $passed release artifact verification cases" From 117c55801c7d48b524f69acf3b0ed339ebb96f51 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:10:58 +0530 Subject: [PATCH 03/13] fix: keep release staging from promoting latest artifacts Remove latest inputs and publication paths; reject legacy promotion before side effects and verify the shipped guard during release configuration checks. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- .github/workflows/patch-release.yaml | 28 +------------------ .tekton/release-patch.yaml | 2 -- .tekton/release.yaml | 2 -- hack/trigger-patch-release.sh | 9 +++---- hack/verify-release-config.sh | 21 +++++++++++++++ tekton/publish.yaml | 23 +++++----------- tekton/release-cheat-sheet.md | 9 +++++-- tekton/release-pipeline.yaml | 40 +--------------------------- 8 files changed, 40 insertions(+), 94 deletions(-) diff --git a/.github/workflows/patch-release.yaml b/.github/workflows/patch-release.yaml index 2f7ae9923..561ed2de9 100644 --- a/.github/workflows/patch-release.yaml +++ b/.github/workflows/patch-release.yaml @@ -11,11 +11,6 @@ name: Patch Release description: "Version to release (for example, v0.5.1)" required: true type: string - release_as_latest: - description: "Publish as latest release" - required: false - type: boolean - default: true schedule: - cron: "0 10 * * 4" @@ -47,23 +42,6 @@ jobs: id: scan shell: bash run: | - latest_branch="" - latest_major=0 - latest_minor=0 - for ref in $(git branch -r --list 'origin/release-v*'); do - branch="${ref#origin/}" - if [[ "$branch" =~ ^release-v([0-9]+)\.([0-9]+)\.x$ ]]; then - major="${BASH_REMATCH[1]}" - minor="${BASH_REMATCH[2]}" - if [ "$major" -gt "$latest_major" ] || { [ "$major" -eq "$latest_major" ] && [ "$minor" -gt "$latest_minor" ]; }; then - latest_major=$major - latest_minor=$minor - latest_branch=$branch - fi - fi - done - echo "::notice::Latest release branch: ${latest_branch}" - min_major="${MIN_RELEASE_VERSION%%.*}" min_minor="${MIN_RELEASE_VERSION##*.}" releases=() @@ -93,9 +71,7 @@ jobs: fi next_version=$(echo "$last_tag" | awk -F. '{printf "%s.%s.%d", $1, $2, $3+1}') - is_latest=false - [ "$branch" = "$latest_branch" ] && is_latest=true - releases+=("{\"branch\":\"${branch}\",\"version\":\"${next_version}\",\"release_as_latest\":\"${is_latest}\"}") + releases+=("{\"branch\":\"${branch}\",\"version\":\"${next_version}\"}") done if [ ${#releases[@]} -eq 0 ]; then @@ -133,7 +109,6 @@ jobs: PAC_INCOMING_SECRET: ${{ secrets.PAC_INCOMING_SECRET }} RELEASE_BRANCH: ${{ matrix.release.branch }} RELEASE_VERSION: ${{ matrix.release.version }} - RELEASE_AS_LATEST: ${{ matrix.release.release_as_latest }} run: bash ./hack/trigger-patch-release.sh trigger-manual-release: @@ -184,5 +159,4 @@ jobs: PAC_INCOMING_SECRET: ${{ secrets.PAC_INCOMING_SECRET }} RELEASE_BRANCH: ${{ inputs.branch }} RELEASE_VERSION: ${{ inputs.version }} - RELEASE_AS_LATEST: ${{ inputs.release_as_latest }} run: bash ./hack/trigger-patch-release.sh diff --git a/.tekton/release-patch.yaml b/.tekton/release-patch.yaml index e67af9ea1..ccfde1333 100644 --- a/.tekton/release-patch.yaml +++ b/.tekton/release-patch.yaml @@ -43,8 +43,6 @@ spec: value: tekton-robot - name: releaseBucket value: tekton-releases - - name: releaseAsLatest - value: "{{ release_as_latest }}" - name: platforms value: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountImagesPath diff --git a/.tekton/release.yaml b/.tekton/release.yaml index f6337bc36..310c69208 100644 --- a/.tekton/release.yaml +++ b/.tekton/release.yaml @@ -46,8 +46,6 @@ spec: value: tekton-robot - name: releaseBucket value: tekton-releases - - name: releaseAsLatest - value: "true" - name: platforms value: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountImagesPath diff --git a/hack/trigger-patch-release.sh b/hack/trigger-patch-release.sh index 38a22fcfa..563423efd 100644 --- a/hack/trigger-patch-release.sh +++ b/hack/trigger-patch-release.sh @@ -7,7 +7,6 @@ set -euo pipefail : "${PAC_CONTROLLER_URL:?}" : "${PAC_INCOMING_SECRET:?}" : "${PAC_REPOSITORY_NAME:?}" -: "${RELEASE_AS_LATEST:?}" : "${RELEASE_BRANCH:?}" : "${RELEASE_VERSION:?}" @@ -23,8 +22,7 @@ jq -n '{ pipelinerun: "release-patch", secret: env.PAC_INCOMING_SECRET, params: { - version: env.RELEASE_VERSION, - release_as_latest: env.RELEASE_AS_LATEST + version: env.RELEASE_VERSION } }' | curl --fail-with-body --silent --show-error \ -H "Content-Type: application/json" \ @@ -97,7 +95,8 @@ draft_url=$(jq -r '.html_url' <<<"$draft") echo echo "- External PipelineRun: ${details_url}" echo "- Draft release: ${draft_url}" - echo "- Publication: manual review required" + echo "- Finalization: approval, artifact verification, and installation smoke test required" + echo "- Latest image and bucket promotion: disabled during staging" } >> "$GITHUB_STEP_SUMMARY" -echo "::warning title=Draft release awaiting publication::Review and publish ${draft_url}" +echo "::warning title=Draft release awaiting finalization::Review ${draft_url}; publication and latest promotion require separate approval" diff --git a/hack/verify-release-config.sh b/hack/verify-release-config.sh index 9513a9676..0aad70ffa 100644 --- a/hack/verify-release-config.sh +++ b/hack/verify-release-config.sh @@ -25,12 +25,33 @@ test -n "$go_image" grep -Fq "apk add --no-cache bash make" "$root/tekton/publish.yaml" grep -Fq 'org.opencontainers.image.source=https://$PACKAGE' "$root/tekton/publish.yaml" grep -Fq 'org.opencontainers.image.url=https://$PACKAGE' "$root/tekton/publish.yaml" +if grep -Fq '/latest/' "$root/tekton/release-pipeline.yaml" || + grep -Eq 'crane (cp|copy|tag).*latest' "$root/tekton/publish.yaml" || + grep -Fq '$(params.releaseAsLatest)' "$root/tekton/release-pipeline.yaml"; then + echo "release staging must not promote latest" >&2 + exit 1 +fi +for caller in "$root/.tekton/release.yaml" "$root/.tekton/release-patch.yaml"; do + if grep -Fq 'name: releaseAsLatest' "$caller"; then + echo "release entry points must not accept latest promotion" >&2 + exit 1 + fi +done "$container_tool" run --rm --platform "$platform" \ -v "$root:/workspace/source" \ "$publish_image" sh -ec ' apk add --no-cache bash make >/dev/null cd /workspace/source + # Exercise the shipped guard, not a copy of its condition. + awk '\'' + $0 == " - name: publish-image-and-manifests" { step = 1 } + step && $0 == " script: |" { script = 1; next } + script { sub(/^ /, ""); print } + '\'' tekton/publish.yaml > /tmp/publish.sh + status=0 + VERSION_TAG=v0.0.0 RELEASE_AS_LATEST=true sh /tmp/publish.sh || status=$? + test "$status" -eq 64 make release \ KUSTOMIZE=/go/bin/kustomize \ RELEASE_DIR=/tmp/release \ diff --git a/tekton/publish.yaml b/tekton/publish.yaml index bc4413fae..e339d0644 100644 --- a/tekton/publish.yaml +++ b/tekton/publish.yaml @@ -31,7 +31,8 @@ spec: - name: imageRegistryUser default: tekton-robot - name: releaseAsLatest - default: "true" + description: Must be false; latest promotion belongs to release finalization. + default: "false" - name: platforms default: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountPath @@ -90,17 +91,17 @@ spec: #!/usr/bin/env sh set -eu - apk add --no-cache bash make - printf '%s\n' "$VERSION_TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' || { echo "invalid release version: $VERSION_TAG" >&2 exit 1 } case "$RELEASE_AS_LATEST" in - true|false) ;; - *) echo "releaseAsLatest must be true or false" >&2; exit 1 ;; + false) ;; + *) echo "latest promotion requires the release finalization workflow" >&2; exit 64 ;; esac + apk add --no-cache bash make + OUTPUT_RELEASE_DIR="$OUTPUT_ROOT/$VERSION_TAG" git config --global --add safe.directory "$PROJECT_ROOT" mkdir -p "$HOME/.docker" "$OUTPUT_RELEASE_DIR/template" @@ -139,15 +140,3 @@ spec: *) echo "unexpected release image: $image" >&2; exit 1 ;; esac printf '%s,' "$image" > "$(results.IMAGES.path)" - - - name: tag-latest - image: cgr.dev/chainguard/crane:latest-dev@sha256:dd9a345764e47517b80391648020b78c0ae6d4db0e392c7013b71e3817ed544c - script: | - #!/bin/sh - set -eu - - [ "$RELEASE_AS_LATEST" = "true" ] || exit 0 - mkdir -p "$HOME/.docker" - cp /workspace/docker-config.json "$HOME/.docker/config.json" - image=$(cat /workspace/built-images) - crane cp "$image" "${image%@*}:latest" diff --git a/tekton/release-cheat-sheet.md b/tekton/release-cheat-sheet.md index b6f6e8322..da31d8e9d 100644 --- a/tekton/release-cheat-sheet.md +++ b/tekton/release-cheat-sheet.md @@ -9,7 +9,7 @@ The automated steps below become operational only after the release Pipeline tra 1. Confirm the target version, LTS designation, end-of-life date, and compatibility ranges with the maintainers. 2. Confirm all intended changes are merged and their release-note blocks are complete. 3. From a clean checkout of the candidate commit, run generation, lint, unit, and end-to-end checks. -4. Run a release candidate with `releaseAsLatest=false` through the Tekton-controlled release Pipeline so it cannot update the `latest` image tag or artifacts. +4. Stage the release candidate through the Tekton-controlled release Pipeline. It publishes only versioned images and artifacts; it cannot update `latest`. 5. Verify both generated manifests reference only `ghcr.io/tektoncd/tekton-kueue` images by immutable digest. 6. Install and upgrade the candidate on the supported Kubernetes, Tekton Pipelines, and Kueue versions. @@ -23,7 +23,7 @@ The automated steps below become operational only after the release Pipeline tra ## Publish a patch release 1. Cherry-pick an approved fix to the supported release branch. -2. After the cherry-pick passes CI on the release branch, run the **Patch Release** GitHub workflow with the branch, next `vX.Y.Z` version, and whether it should update `latest`. +2. After the cherry-pick passes CI on the release branch, run the **Patch Release** GitHub workflow with the branch and next `vX.Y.Z` version. This stages a draft and does not update `latest`. 3. Monitor and verify the release as for an initial release. The scheduled patch workflow scans `release-v0.3.x` and newer for unreleased commits. Older release lines require manual dispatch. A release operator must still verify every resulting release before announcing it. @@ -41,6 +41,11 @@ Verify and record: Do not announce the release until every promised artifact is downloadable and independently verifiable. +Staging does not authorize publication or promotion. Those actions require +maintainer approval and verification of the finished release. Automated +approval-gated finalization is a separate rollout; staging alone leaves the +GitHub release as a draft and preserves the existing `latest` artifacts. + ## After the release 1. Update [`releases.md`](../releases.md) with the release date, exact end-of-life date, compatibility ranges, and release links. diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index b146f225d..3e0ca03d1 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -27,9 +27,6 @@ spec: - name: releaseBucket description: bucket where the release is stored default: tekton-releases - - name: releaseAsLatest - description: Whether to tag and publish this release as latest - default: "false" - name: platforms description: Linux platforms to publish default: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le @@ -163,7 +160,7 @@ spec: - name: imageRegistryUser value: $(params.imageRegistryUser) - name: releaseAsLatest - value: $(params.releaseAsLatest) + value: "false" - name: serviceAccountPath value: $(params.serviceAccountImagesPath) - name: platforms @@ -208,41 +205,6 @@ spec: - name: recursive value: "true" - - name: publish-to-bucket-latest - runAfter: [publish-images] - when: - - input: "$(params.releaseAsLatest)" - operator: in - values: ["true"] - taskRef: - resolver: bundles - params: - - name: bundle - value: ghcr.io/tektoncd/catalog/upstream/tasks/oracle-cloud-storage-upload@sha256:f1dd28da10dfb72ebbb3a5258e9f1ba6bfe82ee587311d708a7be3207f69e5e6 - - name: name - value: oracle-cloud-storage-upload - - name: kind - value: task - workspaces: - - name: credentials - workspace: release-secret - - name: source - workspace: workarea - subPath: bucket - params: - - name: path - value: $(params.versionTag) - - name: bucketName - value: $(params.releaseBucket) - - name: objectPrefix - value: $(params.repoName)/latest/ - - name: replaceExistingFiles - value: "true" - - name: recursive - value: "true" - - name: deleteExtraFiles - value: "true" - - name: report-bucket runAfter: [publish-to-bucket] params: From 12946417b0d00bb42fcb981345e8fc25f5e30c62 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:15:20 +0530 Subject: [PATCH 04/13] feat: smoke test finished release artifacts in an isolated cluster Install the immutable manifest and image, require webhook mutation, admission and PipelineRun success, and retain diagnostics while cleaning only the owned cluster. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- .github/workflows/release-smoke.yaml | 56 ++++++++ hack/smoke-release.sh | 201 +++++++++++++++++++++++++++ hack/smoke-release_test.sh | 181 ++++++++++++++++++++++++ test/release-smoke/README.md | 23 +++ test/release-smoke/release.yaml | 40 ++++++ 5 files changed, 501 insertions(+) create mode 100644 .github/workflows/release-smoke.yaml create mode 100644 hack/smoke-release.sh create mode 100644 hack/smoke-release_test.sh create mode 100644 test/release-smoke/README.md create mode 100644 test/release-smoke/release.yaml diff --git a/.github/workflows/release-smoke.yaml b/.github/workflows/release-smoke.yaml new file mode 100644 index 000000000..1b98f10a1 --- /dev/null +++ b/.github/workflows/release-smoke.yaml @@ -0,0 +1,56 @@ +name: Released Artifact Smoke + +"on": + pull_request: + paths: + - ".github/workflows/release-smoke.yaml" + - ".github/workflows/finalize-release.yaml" + - "hack/*release*.sh" + - "tekton/publish.yaml" + - "tekton/release-pipeline.yaml" + - "test/release-smoke/**" + +permissions: + contents: read + +concurrency: + group: release-smoke-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Install pinned Kind + run: | + GOTOOLCHAIN=local go install sigs.k8s.io/kind@v0.32.0 + - name: Check smoke script behavior + run: | + bash -n hack/smoke-release.sh hack/smoke-release_test.sh + bash hack/smoke-release_test.sh + - name: Install and exercise the immutable release fixture + shell: bash + run: | + # A pinned public fixture tests installation, not the production trust bootstrap. + curl -fsSL https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/v0.4.3/release.yaml \ + -o "$RUNNER_TEMP/release.yaml" + printf '%s %s\n' \ + 340e21dbed798284de3e665e816e02f29f024af0cd32efa3cc98bbb7f9830270 \ + "$RUNNER_TEMP/release.yaml" | sha256sum -c - + bash hack/smoke-release.sh "$RUNNER_TEMP/release.yaml" \ + ghcr.io/tektoncd/tekton-kueue@sha256:6d89cbd8b47cebebc1c05c9e45292433e755372db68e0b66d5dd68d6ede9cb17 \ + "$RUNNER_TEMP/release-smoke" + - name: Save smoke diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-smoke-${{ github.run_id }} + path: ${{ runner.temp }}/release-smoke + if-no-files-found: ignore diff --git a/hack/smoke-release.sh b/hack/smoke-release.sh new file mode 100644 index 000000000..d7a91c2e5 --- /dev/null +++ b/hack/smoke-release.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release smoke failed: $*" >&2; exit 1; } +[[ $# == 3 ]] || fail "usage: $0 VERIFIED_MANIFEST EXPECTED_IMAGE ARTIFACT_DIR" +manifest=$1 +image=$2 +artifacts=$3 +repository=ghcr.io/tektoncd/tekton-kueue +[[ $image =~ ^ghcr\.io/tektoncd/tekton-kueue@sha256:[0-9a-f]{64}$ ]] || + fail "EXPECTED_IMAGE must be the canonical digest reference" +[[ -f $manifest && -r $manifest && -s $manifest ]] || fail "manifest must be a nonempty readable file" +[[ -n $artifacts && ! -L $artifacts ]] || fail "invalid artifact directory" +if [[ -e $artifacts ]]; then + [[ -d $artifacts && -z $(find "$artifacts" -mindepth 1 -print -quit) ]] || + fail "artifact directory must be empty" +fi +for command in kind kubectl jq openssl; do + command -v "$command" >/dev/null || fail "missing command: $command" +done +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d) +cluster="release-smoke-$(basename "$work" | tr '[:upper:]._' '[:lower:]--')" +context="kind-$cluster" +export KUBECONFIG="$work/kubeconfig" +owned=false + +k() { kubectl --context="$context" --request-timeout=30s "$@"; } +cleanup() { + status=$? + trap - EXIT + set +e + if [[ $owned == true ]]; then + for resource in pods deployments services endpoints events certificates \ + pipelineruns.tekton.dev taskruns.tekton.dev workloads.kueue.x-k8s.io; do + k get "$resource" -A -o yaml > "$artifacts/$resource.yaml" 2>&1 + done + k describe pods -A > "$artifacts/pod-descriptions.txt" 2>&1 + for namespace in cert-manager kueue-system tekton-pipelines tekton-kueue release-smoke; do + pods=$(k get pods -n "$namespace" -o name) + for pod in $pods; do + k logs -n "$namespace" "$pod" --all-containers=true --prefix=true \ + --tail=200 > "$artifacts/$namespace-${pod#*/}.log" 2>&1 + done + done + kind export logs --name "$cluster" "$artifacts/kind" > "$artifacts/kind-export.log" 2>&1 + kind delete cluster --name "$cluster" > "$artifacts/cleanup.log" 2>&1 + if [[ $? != 0 ]]; then + echo "failed to delete owned cluster: $cluster" >&2 + status=1 + fi + fi + rm -rf "$work" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# Parse the finished YAML locally, without discovery or the user's kubeconfig. +# This labels only the in-memory objects, never the installed release. +cp "$manifest" "$work/release.yaml" +kubectl label --local --overwrite -f "$work/release.yaml" -o json release-smoke-validation=true > "$work/manifest.json" +jq -e -s --arg repo "$repository" --arg digest "${image##*@}" ' + [ .[] | if .kind == "List" then .items[] else . end ] as $objects + | [$objects[] | select(.kind == "Deployment")] as $deployments + | ($deployments | length == 2) + and ($deployments | map(.metadata.name) | sort == + ["tekton-kueue-controller-manager", "tekton-kueue-webhook"]) + and all($deployments[]; + .apiVersion == "apps/v1" and .metadata.namespace == "tekton-kueue" + and (.spec.template.spec.containers | length == 1) + and ((.spec.template.spec.initContainers // []) | length == 0)) + and ([$objects[] | .. | objects | select(has("image")) | .image] | + length == 2 and all(.[]; + type == "string" and + (split("@") | length == 2 and .[1] == $digest + and (.[0] == $repo or + (.[0] | startswith($repo + ":") and + (ltrimstr($repo + ":") | test("^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$"))))))) +' "$work/manifest.json" >/dev/null || fail "manifest controller/webhook images do not match EXPECTED_IMAGE" + +mkdir -p "$artifacts" +artifacts=$(cd "$artifacts" && pwd) +cp "$work/release.yaml" "$artifacts/release.yaml" +printf '%s\n' "$image" > "$artifacts/expected-image.txt" +printf '%s\n' "$cluster" > "$artifacts/cluster-name.txt" +# Install only the snapshot checked above, even if the caller changes its file. +exec > >(tee "$artifacts/smoke.log") 2>&1 +owned=true # Creation can fail after provisioning nodes; still clean up our name. +kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 180s --retain \ + --image kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95 + +k apply --server-side -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml +k wait --for=condition=Available deployment --all -n cert-manager --timeout=300s +k apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml \ + -f "$root/hack/kueue-config.yaml" +k rollout restart deployment/kueue-controller-manager -n kueue-system +k rollout status deployment/kueue-controller-manager -n kueue-system --timeout=300s +k apply --server-side -f https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml +k wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=300s +k apply --server-side -f "$work/release.yaml" +k wait --for=condition=Ready certificate --all -n tekton-kueue --timeout=300s +k rollout status deployment/tekton-kueue-controller-manager -n tekton-kueue --timeout=300s +k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +k wait --for=jsonpath='{.webhooks[0].clientConfig.caBundle}' \ + mutatingwebhookconfiguration/tekton-kueue-mutating-webhook-configuration --timeout=300s + +# Configure one CPU resource mutation. Restart after configuration so the new +# webhook starts with the ConfigMap already present, rather than racing reload. +k apply -f - <<'YAML' +apiVersion: v1 +kind: ConfigMap +metadata: + name: tekton-kueue-config + namespace: tekton-kueue +data: + config.yaml: | + queueName: pipelines-queue + cel: + expressions: + - resource("cpu", 1) +--- +apiVersion: v1 +kind: Namespace +metadata: + name: release-smoke +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: ResourceFlavor +metadata: + name: release-smoke +spec: {} +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: ClusterQueue +metadata: + name: release-smoke +spec: + namespaceSelector: {} + resourceGroups: + - coveredResources: ["cpu", "tekton.dev/pipelineruns"] + flavors: + - name: release-smoke + resources: + - name: cpu + nominalQuota: 1 + - name: tekton.dev/pipelineruns + nominalQuota: 1 +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: LocalQueue +metadata: + name: pipelines-queue + namespace: release-smoke +spec: + clusterQueue: release-smoke +YAML +k rollout restart deployment/tekton-kueue-webhook -n tekton-kueue +k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +k wait --for=condition=Active clusterqueue/release-smoke --timeout=120s +k wait --for=condition=Active localqueue/pipelines-queue -n release-smoke --timeout=120s +k create -f - -o json > "$artifacts/created-pipelinerun.json" <<'YAML' +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: release-smoke + namespace: release-smoke +spec: + timeouts: + pipeline: 5m0s + pipelineSpec: + tasks: + - name: hello + taskSpec: + steps: + - name: hello + image: registry.access.redhat.com/ubi9/ubi@sha256:66233eebd72bb5baa25190d4f55e1dc3fff3a9b77186c1f91a0abdb274452072 + command: ["/bin/echo", "release smoke"] +YAML +# The create response is the admission result, before controller updates. +jq -e '.spec.status == "Pending" + and .metadata.labels["kueue.x-k8s.io/queue-name"] == "pipelines-queue" + and .metadata.annotations["kueue.konflux-ci.dev/requests-cpu"] == "1" + and (.metadata.uid | type == "string" and length > 0) +' "$artifacts/created-pipelinerun.json" >/dev/null || fail "webhook mutation missing" +uid=$(jq -r .metadata.uid "$artifacts/created-pipelinerun.json") +# Kueue v0.16.6 GetWorkloadNameForOwnerWithGVK hashes kind, group, name, UID. +hash=$(printf 'PipelineRun\ntekton.dev\nrelease-smoke\n%s' "$uid" | openssl dgst -sha1 -r) +workload="workload/pipelinerun-release-smoke-${hash:0:5}" +k wait --for=create "$workload" -n release-smoke --timeout=120s +k get "$workload" -n release-smoke -o json > "$artifacts/created-workload.json" +jq -e --arg uid "$uid" ' + any(.metadata.ownerReferences[]; .uid == $uid and .kind == "PipelineRun" and .controller == true) + and .spec.queueName == "pipelines-queue" + and .spec.podSets[0].template.spec.containers[0].resources.requests.cpu == "1" + and .spec.podSets[0].template.spec.containers[0].resources.requests["tekton.dev/pipelineruns"] == "1" +' "$artifacts/created-workload.json" >/dev/null || fail "Workload does not represent the mutated PipelineRun" +k wait --for=condition=Admitted "$workload" -n release-smoke --timeout=300s +k wait --for=condition=Succeeded pipelinerun/release-smoke -n release-smoke --timeout=360s +echo "PASS published release $image" diff --git a/hack/smoke-release_test.sh b/hack/smoke-release_test.sh new file mode 100644 index 000000000..856d0b36c --- /dev/null +++ b/hack/smoke-release_test.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The same executable supplies both boundary fakes. Local YAML parsing remains +# real kubectl, using the smoke script's isolated kubeconfig and --local. +if [[ ${SMOKE_FAKE:-} == true ]]; then + command=$(basename "$0") + [[ -n ${KUBECONFIG:-} && $KUBECONFIG != "$USER_KUBECONFIG" ]] || exit 90 + jq -cn --arg command "$command" --arg kubeconfig "$KUBECONFIG" \ + --args '$ARGS.positional as $args | {command:$command,kubeconfig:$kubeconfig,args:$args}' \ + -- "$@" >> "$CALLS" + if [[ $command == kind ]]; then + case "$*" in + "create cluster "*) + [[ $* == *"--kubeconfig $KUBECONFIG"* && $* == *"--wait 180s"* ]] || exit 91 + [[ $MODE != creation ]] || exit 42 + ;; + "delete cluster "*) + [[ $MODE != cleanup ]] || exit 42 + ;; + "export logs "*) ;; + *) exit 92 ;; + esac + exit 0 + fi + if [[ $1 == label ]]; then + [[ $* == *"--local"* ]] || exit 93 + exec "$REAL_KUBECTL" "$@" + fi + [[ $1 == --context=kind-release-smoke-* && $2 == --request-timeout=30s ]] || exit 94 + shift 2 + case "$1" in + apply) + if [[ $* == *"-f -"* ]]; then + # Parse actual fixtures offline; invalid YAML cannot silently pass. + "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ + > "$CASE_DIR/fixtures.json" + elif [[ $* == *"/release.yaml"* && $* != *https:* ]]; then + for ((i=1; i<=$#; i++)); do + if [[ ${!i} == -f ]]; then + j=$((i + 1)) + cmp "$VERIFIED_MANIFEST" "${!j}" + fi + done + fi + ;; + rollout) + if [[ $MODE == readiness && $* == *"status deployment/tekton-kueue-webhook"* ]]; then exit 42; fi + ;; + wait) + [[ $* == *--timeout=* ]] || exit 95 + if [[ $MODE == run && $* == *condition=Succeeded* ]]; then exit 42; fi + if [[ $MODE == admission && $* == *condition=Admitted* ]]; then exit 42; fi + if [[ $MODE == workload && $* == *--for=create* ]]; then exit 42; fi + ;; + create) + "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ + > "$CASE_DIR/pipelinerun.json" + jq -n --arg mode "$MODE" '{ + metadata:{uid:"smoke-uid", + labels:{"kueue.x-k8s.io/queue-name":"pipelines-queue"}, + annotations:{"kueue.konflux-ci.dev/requests-cpu":(if $mode == "mutation" then "2" else "1" end)}}, + spec:{status:"Pending"}}' + ;; + get) + if [[ $2 == workload/* ]]; then + jq -n '{metadata:{ownerReferences:[{uid:"smoke-uid",kind:"PipelineRun",controller:true}]}, + spec:{queueName:"pipelines-queue",podSets:[{template:{spec:{containers:[ + {resources:{requests:{cpu:"1","tekton.dev/pipelineruns":"1"}}} + ]}}}]}}' + elif [[ $* != *"-o name"* ]]; then + printf 'kind: List\nitems: []\n' + fi + ;; + describe|logs) ;; + *) exit 96 ;; + esac + exit 0 +fi + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'status=$?; if [[ $status == 0 ]]; then rm -rf "$tmp"; else echo "test artifacts: $tmp" >&2; fi' EXIT +export REAL_KUBECTL +REAL_KUBECTL=$(command -v kubectl) +mkdir "$tmp/bin" +cp "${BASH_SOURCE[0]}" "$tmp/bin/kubectl" +cp "${BASH_SOURCE[0]}" "$tmp/bin/kind" +chmod +x "$tmp/bin/kubectl" "$tmp/bin/kind" +export PATH="$tmp/bin:$PATH" SMOKE_FAKE=true +export USER_KUBECONFIG="$tmp/user-kubeconfig" +printf 'untouched user configuration\n' > "$USER_KUBECONFIG" +cp "$USER_KUBECONFIG" "$tmp/user-original" +image=ghcr.io/tektoncd/tekton-kueue@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export CASE_DIR CALLS MODE VERIFIED_MANIFEST +cases=0 + +check() { + jq -e -s "$1" "$CALLS" >/dev/null || { echo "failed assertion in $MODE: $1" >&2; exit 1; } +} +run_case() { + MODE=$1 + CASE_DIR="$tmp/$MODE" + mkdir "$CASE_DIR" + CALLS="$CASE_DIR/calls.jsonl" + : > "$CALLS" + VERIFIED_MANIFEST="$CASE_DIR/release.yaml" + local expected=$image + cp "$root/test/release-smoke/release.yaml" "$VERIFIED_MANIFEST" + case "$MODE" in + digest) expected=ghcr.io/tektoncd/tekton-kueue:latest ;; + mismatch) expected=${image/aaaa/bbbb} ;; + malformed) printf 'not: [valid yaml\n' > "$VERIFIED_MANIFEST" ;; + missing) rm "$VERIFIED_MANIFEST" ;; + incomplete) + # A valid document without the required controller/webhook pair. + printf 'apiVersion: v1\nkind: Namespace\nmetadata:\n name: test\n' > "$VERIFIED_MANIFEST" + ;; + artifacts) mkdir "$CASE_DIR/artifacts"; touch "$CASE_DIR/artifacts/keep" ;; + esac + local status=0 + KUBECONFIG="$USER_KUBECONFIG" bash "$root/hack/smoke-release.sh" \ + "$VERIFIED_MANIFEST" "$expected" "$CASE_DIR/artifacts" > "$CASE_DIR/output" 2>&1 || status=$? + case "$MODE" in + success|success-again) [[ $status == 0 ]] || { echo "success flow failed; see $CASE_DIR/output" >&2; "$REAL_KUBECTL" version --client; exit 1; } ;; + *) [[ $status != 0 ]] || { echo "$MODE unexpectedly passed" >&2; exit 1; } ;; + esac + cmp "$USER_KUBECONFIG" "$tmp/user-original" + case "$MODE" in + digest|mismatch|malformed|missing|incomplete|artifacts) + check 'all(.[]; .command != "kind")' + ;; + *) + check ' + [.[] | select(.command == "kind" and .args[0] == "create")][0] as $create + | ($create.args | index("--name")) as $nameIndex + | $create.args[$nameIndex + 1] as $name + | ($name | startswith("release-smoke-")) + and all(.[] | select(.command == "kind"); + (.args | index("--name")) as $i | .args[$i + 1] == $name) + and any(.[]; .command == "kind" and .args[0] == "delete") + and all(.[] | select(.command == "kubectl" and .args[0] != "label"); + .args[0] == ("--context=kind-" + $name) and .kubeconfig == $create.kubeconfig) + ' + [[ -s $CASE_DIR/artifacts/events.yaml && -f $CASE_DIR/artifacts/cleanup.log ]] || + { echo "missing failure diagnostics" >&2; exit 1; } + # The private kubeconfig directory must disappear even after failures. + local kubeconfig + kubeconfig=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .kubeconfig' "$CALLS") + [[ ! -e $(dirname "$kubeconfig") ]] + if [[ $MODE == success || $MODE == success-again ]]; then + check ' + any(.[]; .args | index("--for=condition=Ready")) + and any(.[]; .args | index("deployment/tekton-kueue-controller-manager")) + and any(.[]; .args | index("deployment/tekton-kueue-webhook")) + and any(.[]; .args | index("--for=create")) + and any(.[]; .args | index("--for=condition=Admitted")) + and any(.[]; .args | index("--for=condition=Succeeded")) + and any(.[]; .args | index("https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml")) + and any(.[]; .args | index("https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml")) + and any(.[]; .args | index("https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml")) + ' + cmp "$VERIFIED_MANIFEST" "$CASE_DIR/artifacts/release.yaml" + jq -e '.apiVersion == "tekton.dev/v1" and + (.spec.pipelineSpec.tasks[0].taskSpec.steps[0].image | contains("@sha256:"))' \ + "$CASE_DIR/pipelinerun.json" >/dev/null + fi + ;; + esac + cases=$((cases + 1)) + echo "PASS $MODE" +} + +for mode in digest mismatch malformed missing incomplete artifacts success success-again \ + creation readiness mutation workload admission run cleanup; do + run_case "$mode" +done +first=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success/calls.jsonl") +second=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success-again/calls.jsonl") +[[ $first != "$second" ]] +echo "PASS $cases deterministic release smoke cases" diff --git a/test/release-smoke/README.md b/test/release-smoke/README.md new file mode 100644 index 000000000..13d5a9b6b --- /dev/null +++ b/test/release-smoke/README.md @@ -0,0 +1,23 @@ +# Finished-release smoke test + +Run the published artifact verifier first, using an independently configured +signing public key. Pass its saved manifest and immutable image reference to: + +```sh +bash hack/smoke-release.sh VERIFIED_MANIFEST EXPECTED_IMAGE ARTIFACT_DIR +``` + +The script requires Bash, Kind, kubectl, jq, OpenSSL and a working container +provider. It creates its own cluster and kubeconfig, installs the manifest +without rebuilding its image, checks webhook mutation and Kueue admission, +and waits for a PipelineRun to succeed. It saves diagnostics and deletes only +the cluster it created, including on failure. The artifact directory must be +empty. + +The pull-request workflow exercises the existing public v0.4.3 manifest, +pinned by SHA-256 and image digest, on an isolated runner. This installation +check is not a signing-key trust bootstrap. The finalization workflow will +call the same script with the candidate that passed signature verification. + +`release.yaml` in this directory is an offline parsing fixture for +`bash hack/smoke-release_test.sh`; it is not an installable release. diff --git a/test/release-smoke/release.yaml b/test/release-smoke/release.yaml new file mode 100644 index 000000000..f45caa1db --- /dev/null +++ b/test/release-smoke/release.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tekton-kueue +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tekton-kueue-controller-manager + namespace: tekton-kueue +spec: + selector: + matchLabels: + app: controller + template: + metadata: + labels: + app: controller + spec: + containers: + - name: manager + image: ghcr.io/tektoncd/tekton-kueue:v0.4.3@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tekton-kueue-webhook + namespace: tekton-kueue +spec: + selector: + matchLabels: + app: webhook + template: + metadata: + labels: + app: webhook + spec: + containers: + - name: webhook + image: ghcr.io/tektoncd/tekton-kueue@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa From 972ca8b655d2ca59fb270e9b1327546b734a2b12 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:28:20 +0530 Subject: [PATCH 05/13] fix: assert the released Tekton pending status in smoke tests Use PipelineRunPending from the actual admission response and reject the incorrect Pending value in the deterministic boundary test. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- hack/smoke-release.sh | 2 +- hack/smoke-release_test.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/smoke-release.sh b/hack/smoke-release.sh index d7a91c2e5..f3cd989de 100644 --- a/hack/smoke-release.sh +++ b/hack/smoke-release.sh @@ -179,7 +179,7 @@ spec: command: ["/bin/echo", "release smoke"] YAML # The create response is the admission result, before controller updates. -jq -e '.spec.status == "Pending" +jq -e '.spec.status == "PipelineRunPending" and .metadata.labels["kueue.x-k8s.io/queue-name"] == "pipelines-queue" and .metadata.annotations["kueue.konflux-ci.dev/requests-cpu"] == "1" and (.metadata.uid | type == "string" and length > 0) diff --git a/hack/smoke-release_test.sh b/hack/smoke-release_test.sh index 856d0b36c..05fcbda92 100644 --- a/hack/smoke-release_test.sh +++ b/hack/smoke-release_test.sh @@ -60,7 +60,7 @@ if [[ ${SMOKE_FAKE:-} == true ]]; then metadata:{uid:"smoke-uid", labels:{"kueue.x-k8s.io/queue-name":"pipelines-queue"}, annotations:{"kueue.konflux-ci.dev/requests-cpu":(if $mode == "mutation" then "2" else "1" end)}}, - spec:{status:"Pending"}}' + spec:{status:(if $mode == "wrong-pending" then "Pending" else "PipelineRunPending" end)}}' ;; get) if [[ $2 == workload/* ]]; then @@ -172,7 +172,7 @@ run_case() { } for mode in digest mismatch malformed missing incomplete artifacts success success-again \ - creation readiness mutation workload admission run cleanup; do + creation readiness mutation wrong-pending workload admission run cleanup; do run_case "$mode" done first=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success/calls.jsonl") From 7bb191ab81fdf09096c7432b258238e6a3b2cada Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:31:44 +0530 Subject: [PATCH 06/13] feat: finalize verified releases through robot approval gates Verify the exact candidate and smoke its published artifacts before optional latest promotion. Bind repaired provenance notes to verified payloads and retain publication state during repair. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- .github/workflows/ci.yml | 20 +- .github/workflows/finalize-release.yaml | 118 ++++++++ docs/releases.md | 125 +++++++++ hack/finalize-release.sh | 188 +++++++++++++ hack/finalize-release_test.sh | 345 ++++++++++++++++++++++++ 5 files changed, 795 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/finalize-release.yaml create mode 100644 docs/releases.md create mode 100644 hack/finalize-release.sh create mode 100644 hack/finalize-release_test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0fd1474a..acfee2867 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,9 +90,25 @@ jobs: - name: Run actionlint run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 + release-automation: + name: Release automation tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check release verification, smoke and finalization + run: | + bash -n hack/verify-release-artifacts.sh hack/verify-release-artifacts_test.sh \ + hack/smoke-release.sh hack/smoke-release_test.sh \ + hack/finalize-release.sh hack/finalize-release_test.sh + bash hack/verify-release-artifacts_test.sh + bash hack/smoke-release_test.sh + bash hack/finalize-release_test.sh + ci-summary: name: CI summary - needs: [build, test, lint, e2e, dependency-review, workflow-lint] + needs: [build, test, lint, e2e, dependency-review, workflow-lint, release-automation] runs-on: ubuntu-latest if: always() steps: @@ -104,6 +120,7 @@ jobs: E2E_RESULT: ${{ needs.e2e.result }} DEPENDENCY_REVIEW_RESULT: ${{ needs.dependency-review.result }} WORKFLOW_LINT_RESULT: ${{ needs.workflow-lint.result }} + RELEASE_AUTOMATION_RESULT: ${{ needs.release-automation.result }} run: | results=( "build=${BUILD_RESULT}" @@ -112,6 +129,7 @@ jobs: "e2e=${E2E_RESULT}" "dependency-review=${DEPENDENCY_REVIEW_RESULT}" "workflow-lint=${WORKFLOW_LINT_RESULT}" + "release-automation=${RELEASE_AUTOMATION_RESULT}" ) failed=0 for result in "${results[@]}"; do diff --git a/.github/workflows/finalize-release.yaml b/.github/workflows/finalize-release.yaml new file mode 100644 index 000000000..f5a39e99c --- /dev/null +++ b/.github/workflows/finalize-release.yaml @@ -0,0 +1,118 @@ +name: Finalize Release + +"on": + workflow_dispatch: + inputs: + operation: + description: Publish a verified candidate or repair only its attestation notes + required: true + type: choice + options: [finalize, repair] + version: + description: Candidate version, for example v0.4.4 + required: true + type: string + source_commit: + description: Full source commit approved for this release + required: true + type: string + release_id: + description: Existing GitHub release ID + required: true + type: string + image_digest: + description: Exact approved sha256 image digest + required: true + type: string + promote_latest: + description: Also promote the verified image and manifests to latest + required: false + default: false + type: boolean + +permissions: + contents: read + +concurrency: + group: release-finalization + cancel-in-progress: false + +jobs: + finalize: + if: github.repository == 'tektoncd/tekton-kueue' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: + name: kueue-release-finalization + env: + GOTOOLCHAIN: local + GH_TOKEN: ${{ secrets.CHATOPS_TOKEN }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + OCI_API_KEY: ${{ secrets.OCI_API_KEY }} + OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }} + OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }} + OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }} + OCI_REGION: ${{ secrets.OCI_REGION }} + RELEASE_SIGNING_PUBLIC_KEY: ${{ vars.RELEASE_SIGNING_PUBLIC_KEY }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Require configured signing trust + shell: bash + run: | + test -n "$RELEASE_SIGNING_PUBLIC_KEY" + printf '%s\n' "$RELEASE_SIGNING_PUBLIC_KEY" > "$RUNNER_TEMP/release-signing.pub" + openssl pkey -pubin -in "$RUNNER_TEMP/release-signing.pub" -noout + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + + - name: Install pinned verification tools + shell: bash + run: | + export GOBIN="$RUNNER_TEMP/release-tools" + mkdir -p "$GOBIN" + go install github.com/sigstore/cosign/v3/cmd/cosign@v3.0.6 + go install github.com/google/go-containerregistry/cmd/crane@v0.20.6 + go install sigs.k8s.io/kind@v0.32.0 + printf '%s\n' "$GOBIN" >> "$GITHUB_PATH" + + - name: Install pinned OCI client + if: inputs.promote_latest + shell: bash + run: | + python3 -m venv "$RUNNER_TEMP/oci-client" + "$RUNNER_TEMP/oci-client/bin/pip" install oci-cli==3.74.2 + printf '%s\n' "$RUNNER_TEMP/oci-client/bin" >> "$GITHUB_PATH" + + - name: Verify, smoke-test and finalize + shell: bash + env: + OPERATION: ${{ inputs.operation }} + VERSION: ${{ inputs.version }} + COMMIT: ${{ inputs.source_commit }} + RELEASE_ID: ${{ inputs.release_id }} + DIGEST: ${{ inputs.image_digest }} + PROMOTE_LATEST: ${{ inputs.promote_latest }} + run: | + bash hack/finalize-release.sh "$OPERATION" "$VERSION" "$COMMIT" \ + "$RELEASE_ID" "$DIGEST" "$RUNNER_TEMP/release-signing.pub" \ + "$PROMOTE_LATEST" "$RUNNER_TEMP/release-finalization" + { + printf '### Release %s\n\n' "$VERSION" + printf -- '- Operation: %s\n' "$OPERATION" + printf -- '- Source: `%s`\n' "$COMMIT" + printf -- '- Image: `%s`\n' "$DIGEST" + printf -- '- Release: https://github.com/tektoncd/tekton-kueue/releases/tag/%s\n' "$VERSION" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Save isolated smoke diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-smoke-${{ github.run_id }} + path: ${{ runner.temp }}/release-finalization/smoke + if-no-files-found: ignore diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 000000000..41743c6bb --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,125 @@ +# Release staging and finalization + +Releases have two separate stages. The release pipeline builds and signs a +versioned candidate, uploads its manifests under `previous//`, and +creates a GitHub draft. It does not update the image's `latest` tag or the +bucket's `latest/` manifests. + +The **Finalize Release** workflow verifies that candidate, installs its exact +published image in a disposable cluster, and then publishes it. Updating +`latest` is an explicit choice in this workflow, after maintainer approval. +It never builds another image. + +## Required environment + +Provision `kueue-release-finalization` using the standalone module +in `tektoncd/plumbing` under `terraform/kueue-release-finalization/`. The environment +requires review by `tekton-kueue-maintainers`, disallows self-review and +administrator bypass, and allows only the protected workflow on `main`. +Merging the workflow alone does not configure these protections. The +finalizer also checks the environment's actual protection before mutations. + +Set the environment variable `RELEASE_SIGNING_PUBLIC_KEY` through the required +Terraform signing-key input. Obtain the key from independently managed +release infrastructure and confirm its fingerprint with the release +maintainers. Do not bootstrap trust from the candidate's signature or Rekor +entry. The historical `tektoncd/chains/tekton.pub` does not match the key used +by the OCI `v0.4.3` release; it is not a fallback. + +The workflow uses these existing credential names. Environment-scoped secrets +can override organization secrets where the existing credential is unsuitable: + +| Name | Required access | +| --- | --- | +| `CHATOPS_TOKEN` | `tekton-robot` GitHub credential with repository contents write access, including draft releases | +| `GHCR_TOKEN` | `tekton-robot` package credential that can promote this repository's image | +| `OCI_API_KEY`, `OCI_FINGERPRINT`, `OCI_TENANCY_OCID`, `OCI_USER_OCID`, `OCI_REGION` | Existing release-storage identity, authorized to read the namespace and write `tekton-kueue/latest/` | + +The finalizer verifies token ownership through GitHub, rather than assuming a +secret name proves its owner. Missing or incorrect credentials stop the run; +there is no personal-account fallback. Registry and storage credentials are +needed only when `promote_latest` is selected. Temporary OCI and Docker +credential files are removed on exit. + +For key rotation, independently authenticate the replacement key, update the +managed environment variable, and test an existing candidate before allowing +publication. Keep historical public keys available for verifying old releases; +do not silently trust whatever key a candidate supplies. + +## Approving a candidate + +Run **Finalize Release** from `main`. Supply: + +| Input | Meaning | +| --- | --- | +| `operation` | `finalize` to publish, or `repair` to correct only the attestation notes | +| `version` | The existing candidate's `vX.Y.Z` version | +| `source_commit` | Its full, approved source commit | +| `release_id` | The existing GitHub release's numeric ID | +| `image_digest` | The exact approved `sha256:` digest | +| `promote_latest` | Also update the image, bucket and GitHub latest release; default is false | + +Check the source commit and digest before approving the environment deployment. +The workflow runs only the trusted `main` scripts; it does not check out or +execute scripts supplied by the candidate branch. + +Before any promotion or publication, the finalizer: + +1. Downloads the three release assets by ID, checks both manifest hashes, and + compares the assets byte-for-byte with the versioned bucket objects. +2. Confirms that the controller and webhook reference the same expected image + digest, that all four supported Linux platforms exist, and that their OCI + repository labels are correct. +3. Verifies the image signature and SLSA v1 provenance against the configured + key, including the provenance's source commit and subject digest. +4. For `finalize`, installs the finished manifests with supported pinned + dependencies in a uniquely named Kind cluster. It checks controller and + webhook readiness, resource mutation, Kueue admission, and PipelineRun + completion. The cluster and its private kubeconfig are cleaned up on every + exit; diagnostics are retained as workflow artifacts. +5. Checks that the release's tag, source, draft state and asset IDs have not + changed since verification. +6. Corrects the attestation section using the Rekor entry whose payload hash + matches the verified statement. + +For `finalize`, optional latest promotion happens only after these gates. +The finalizer refuses to promote an older version over the current latest, +checks the resulting image digest and bucket contents, and publishes the +existing GitHub release with an explicit tag and source commit. + +## Repair and retries + +`repair` performs artifact and signature verification but does not install a +cluster, publish a draft, or promote image/bucket latest. It preserves the +release's publication state and GitHub latest selection while replacing only +the attestation section. `promote_latest` must be false for this operation. + +A failed verification never falls back to an image-signature entry, an +untrusted key, or another account. A promotion can fail after an approved +image or object has already moved; the GitHub release is not published until +all requested promotion checks pass. Rerun with the same approved candidate +to complete that promotion without rebuilding it. + +Do not delete versioned artifacts to work around a partial release. Preserve +them and their original source/signing evidence for recovery. + +## Developer checks + +```sh +bash hack/verify-release-artifacts_test.sh +bash hack/smoke-release_test.sh +bash hack/finalize-release_test.sh +``` + +These deterministic boundary tests do not mutate GitHub, the registry or +storage, and do not use the developer's current cluster. Real smoke QA uses +`hack/smoke-release.sh` with an already verified manifest and canonical image +digest; it creates and deletes only its own Kind cluster. The finalization +workflow pins its tool versions, including Kind, rather than relying on older +workstation tools. + +The Patch Release workflow no longer accepts `release_as_latest`, and the +release pipeline no longer accepts `releaseAsLatest`. Select +`promote_latest` during finalization instead. The image-publishing Task keeps +the legacy parameter only to accept `false` from older supported branches; +passing `true` is rejected. diff --git a/hack/finalize-release.sh b/hack/finalize-release.sh new file mode 100644 index 000000000..d3a7c5531 --- /dev/null +++ b/hack/finalize-release.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release finalization failed: $*" >&2; exit 1; } +robot() { + [[ $(gh api user --jq .login) == tekton-robot ]] || fail "GH_TOKEN must belong to tekton-robot" +} + +[[ $# == 8 ]] || + fail "usage: $0 finalize|repair VERSION COMMIT RELEASE_ID DIGEST PUBLIC_KEY_FILE true|false WORK_DIR" +operation=$1 +version=$2 +commit=$3 +release_id=$4 +digest=$5 +key=$6 +latest=$7 +work=$8 +repository=tektoncd/tekton-kueue +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +[[ $operation == finalize || $operation == repair ]] || fail "invalid operation" +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ && + $commit =~ ^[0-9a-f]{40}$ && $release_id =~ ^[1-9][0-9]*$ && + $digest =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid candidate identity" +[[ $latest == true || $latest == false ]] || fail "invalid latest selection" +[[ $operation != repair || $latest == false ]] || fail "repair cannot promote latest" +[[ ${GITHUB_REPOSITORY:-} == "$repository" && + ${GITHUB_REF:-} == refs/heads/main ]] || fail "run the trusted main-branch workflow" +[[ -n ${GH_TOKEN:-} && -f $key && -s $key ]] || fail "bot token and trusted public key are required" +[[ -n $work && ! -L $work ]] || fail "invalid work directory" +if [[ -e $work ]]; then + [[ -d $work && -z $(find "$work" -mindepth 1 -print -quit) ]] || + fail "work directory must be empty" +fi +mkdir -p "$work" +work=$(cd "$work" && pwd) +robot + +# An environment name in a workflow does not itself configure review protection. +gh api "repos/$repository/environments/kueue-release-finalization" > "$work/environment.json" +jq -e ' + .can_admins_bypass == false and + any(.protection_rules[]?; + .type == "required_reviewers" and .prevent_self_review == true and + any(.reviewers[]?; .type == "Team" and .reviewer.id == 18766447)) +' "$work/environment.json" >/dev/null || fail "release approval environment is not protected" + +current=$(gh api "repos/$repository/releases/latest" --jq .tag_name) +[[ $current =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "cannot determine current latest release" +publish_latest=$latest +[[ $current != "$version" ]] || publish_latest=true +if [[ $latest == true ]]; then + for name in GHCR_TOKEN OCI_API_KEY OCI_FINGERPRINT OCI_TENANCY_OCID OCI_USER_OCID OCI_REGION; do + [[ -n ${!name:-} ]] || fail "$name is required for latest promotion" + done + [[ $(GH_TOKEN="$GHCR_TOKEN" gh api user --jq .login) == tekton-robot ]] || + fail "GHCR_TOKEN must belong to tekton-robot" + IFS=. read -r major minor patch <<< "${version#v}" + IFS=. read -r old_major old_minor old_patch <<< "${current#v}" + (( 10#$major > 10#$old_major || + (10#$major == 10#$old_major && 10#$minor > 10#$old_minor) || + (10#$major == 10#$old_major && 10#$minor == 10#$old_minor && 10#$patch >= 10#$old_patch) )) || + fail "refusing to promote an older release as latest" +fi + +bash "$root/hack/verify-release-artifacts.sh" "$version" "$commit" "$release_id" "$key" "$work/verified" +info="$work/verified/release-info.json" +jq -e --arg digest "$digest" '.digest == $digest' "$info" >/dev/null || + fail "verified digest differs from the approved candidate" +image=$(jq -r .image "$info") +if [[ $operation == finalize ]]; then + bash "$root/hack/smoke-release.sh" "$work/verified/release.yaml" "$image" "$work/smoke" +fi + +# Bind the release-note UUID to bytes from an already verified statement. +jq -r '.[]' "$work/verified/verified-payloads.json" | + while IFS= read -r payload; do + hash=$(printf '%s' "$payload" | base64 -d | openssl dgst -sha256 -r) + printf '%s\n' "${hash%% *}" + done > "$work/payload-hashes" +curl -fsS -H 'Content-Type: application/json' \ + -d "{\"hash\":\"$digest\"}" https://rekor.sigstore.dev/api/v1/index/retrieve > "$work/rekor-index.json" +uuid="" +while IFS= read -r candidate; do + [[ $candidate =~ ^[0-9a-f]+$ ]] || fail "invalid Rekor entry ID" + curl -fsS "https://rekor.sigstore.dev/api/v1/log/entries/$candidate" | + jq -r '.[].body' | base64 -d > "$work/rekor-entry.json" + kind=$(jq -r .kind "$work/rekor-entry.json") + [[ $kind == dsse || $kind == intoto ]] || continue + hash=$(jq -r '.spec.payloadHash.value // .spec.content.payloadHash.value // empty' "$work/rekor-entry.json") + if [[ -n $hash ]] && grep -Fxq -- "$hash" "$work/payload-hashes"; then + uuid=$candidate + break + fi +done < <(jq -er '.[]' "$work/rekor-index.json") +[[ -n $uuid ]] || fail "no Rekor entry matches the verified provenance" + +gh api "repos/$repository/releases/$release_id" > "$work/current-release.json" +jq -e --arg version "$version" --arg commit "$commit" --slurpfile info "$info" ' + .id == $info[0].release_id and .tag_name == $version + and .target_commitish == $commit and .draft == $info[0].draft + and ([.assets[] + | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS") + | {name, id}] | sort_by(.name)) == $info[0].assets +' "$work/current-release.json" >/dev/null || fail "release changed after verification" + +cat > "$work/attestation.md" < "$work/patch.json" + +if [[ $latest == true ]]; then + # Keep credentials out of the caller's Docker configuration and process args. + export DOCKER_CONFIG="$work/docker" + mkdir -p "$DOCKER_CONFIG" "$work/oci" + trap 'rm -rf "$work/oci" "$work/docker"' EXIT + chmod 700 "$work/oci" + printf '%s\n' "$OCI_API_KEY" > "$work/oci/key.pem" + chmod 600 "$work/oci/key.pem" + for value in "$OCI_USER_OCID" "$OCI_TENANCY_OCID" "$OCI_FINGERPRINT" "$OCI_REGION"; do + [[ $value != *$'\n'* && $value != *$'\r'* ]] || fail "invalid OCI configuration" + done + cat > "$work/oci/config" </dev/null + curl -fsSL "https://infra.tekton.dev/tekton-releases/tekton-kueue/latest/$file" \ + --output "$work/latest-$file" + cmp -s "$work/verified/$file" "$work/latest-$file" || fail "latest bucket verification failed: $file" + done +fi + +robot +gh api --method PATCH "repos/$repository/releases/$release_id" --input "$work/patch.json" > "$work/result.json" +jq -e --arg version "$version" --arg commit "$commit" --arg operation "$operation" \ + --slurpfile previous "$work/current-release.json" --slurpfile patch "$work/patch.json" ' + .tag_name == $version and .target_commitish == $commit and + .body == $patch[0].body and + .draft == (if $operation == "repair" then $previous[0].draft else false end) +' "$work/result.json" >/dev/null || fail "GitHub did not retain the requested release state" +if [[ $publish_latest == true ]]; then + [[ $(gh api "repos/$repository/releases/latest" --jq .tag_name) == "$version" ]] || + fail "GitHub latest release does not match the finalized candidate" +fi +echo "PASS $operation https://github.com/$repository/releases/tag/$version" diff --git a/hack/finalize-release_test.sh b/hack/finalize-release_test.sh new file mode 100644 index 000000000..aef768ed2 --- /dev/null +++ b/hack/finalize-release_test.sh @@ -0,0 +1,345 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Copies of this file provide only the separately tested verifier/smoke and +# network executable boundaries. The finalizer itself is copied without edits. +if [[ ${FINALIZER_TEST_CHILD:-} == true ]]; then + command=${0##*/} + event() { + jq -cn --arg event "$1" --arg token "${GH_TOKEN:-}" --args \ + '{event:$event,token:$token,args:$ARGS.positional}' -- "${@:2}" >> "$CALLS" + } + case "$command" in + verify-release-artifacts.sh) + event verify "$@" + [[ $# == 5 && $1 == "$VERSION" && $2 == "$COMMIT" && $3 == 123 && + $4 == "$CASE_DIR/key.pub" && $5 == "$CASE_DIR/work/verified" ]] + openssl pkey -pubin -in "$4" -noout + [[ $MODE != verifier-failure ]] || exit 42 + mkdir -p "$5" + cp "$CASE_DIR"/{release.yaml,release.notags.yaml,SHA256SUMS,verified-payloads.json} "$5/" + jq --arg image "$IMAGE" --arg digest "$DIGEST" --arg version "$VERSION" \ + --arg commit "$COMMIT" --arg mode "$MODE" ' + {release_id:.id,version:$version,commit:$commit,image:$image, + digest:(if $mode == "bad-digest" then "sha256:" + ("b"*64) else $digest end), + draft:.draft,assets:(.assets | sort_by(.name))} + ' "$CASE_DIR/original.json" > "$5/release-info.json" + ;; + smoke-release.sh) + event smoke "$@" + [[ $# == 3 && $1 == "$CASE_DIR/work/verified/release.yaml" && + $2 == "$IMAGE" && $3 == "$CASE_DIR/work/smoke" ]] + cmp "$1" "$CASE_DIR/release.yaml" + [[ $MODE != smoke-failure ]] || exit 42 + mkdir "$3" + cp "$1" "$3/release.yaml" + printf '%s\n' "$2" > "$3/expected-image.txt" + ;; + gh) + [[ $1 == api ]] + shift + if [[ $1 == user ]]; then + [[ $# == 3 && $2 == --jq && $3 == .login ]] + event identity "$@" + if [[ $MODE == wrong-actor ]]; then echo waveywaves; else echo tekton-robot; fi + elif [[ $1 == --method ]]; then + [[ $# == 5 && $2 == PATCH && + $3 == repos/tektoncd/tekton-kueue/releases/123 && $4 == --input ]] + event patch "$@" + [[ $GH_TOKEN == test-bot-token ]] + cp "$5" "$CASE_DIR/sent-patch.json" + [[ $MODE != api-failure ]] || exit 42 + jq -s '.[0] * (.[1] | del(.make_latest))' \ + "$CASE_DIR/release.json" "$5" > "$CASE_DIR/updated.json" + if [[ $MODE == wrong-result ]]; then + jq '.draft = true' "$CASE_DIR/updated.json" > "$CASE_DIR/release.json" + else + cp "$CASE_DIR/updated.json" "$CASE_DIR/release.json" + fi + if [[ $(jq -r .make_latest "$5") == true && $MODE != wrong-latest ]]; then + printf '%s\n' "$VERSION" > "$CASE_DIR/latest" + fi + cat "$CASE_DIR/release.json" + else + case "$1" in + repos/tektoncd/tekton-kueue/environments/kueue-release-finalization) + [[ $# == 1 ]] + event environment + cat "$CASE_DIR/environment.json" + ;; + repos/tektoncd/tekton-kueue/releases/latest) + [[ $# == 3 && $2 == --jq && $3 == .tag_name ]] + event latest + cat "$CASE_DIR/latest" + ;; + repos/tektoncd/tekton-kueue/releases/123) + [[ $# == 1 ]] + event release + cat "$CASE_DIR/release.json" + ;; + *) exit 90 ;; + esac + fi + ;; + curl) + if [[ $# == 6 && $1 == -fsS && $2 == -H && + $3 == 'Content-Type: application/json' && $4 == -d && + $6 == https://rekor.sigstore.dev/api/v1/index/retrieve ]]; then + jq -e --arg digest "$DIGEST" '.hash == $digest' <<< "$5" >/dev/null + event rekor-index + printf '["abcdef"]\n' + elif [[ $# == 2 && $1 == -fsS && + $2 == https://rekor.sigstore.dev/api/v1/log/entries/abcdef ]]; then + event rekor-entry + hash=$(openssl dgst -sha256 -r "$CASE_DIR/payload.json") + hash=${hash%% *} + [[ $MODE != rekor-mismatch ]] || hash=${hash//?/0} + jq -cn --arg hash "$hash" \ + '{abcdef:{body:({kind:"dsse",spec:{payloadHash:{value:$hash}}}|tojson|@base64)}}' + elif [[ $# == 4 && $1 == -fsSL && $3 == --output ]]; then + prefix=https://infra.tekton.dev/tekton-releases/tekton-kueue/latest/ + [[ $2 == "$prefix"* ]] + file=${2#"$prefix"} + case "$file" in release.yaml|release.notags.yaml|SHA256SUMS) ;; *) exit 90 ;; esac + event bucket-read "$file" + cp "$CASE_DIR/$file" "$4" + [[ $MODE != bucket-failure ]] || printf 'changed\n' >> "$4" + else exit 90; fi + ;; + crane) + [[ $DOCKER_CONFIG == "$CASE_DIR/work/docker" && -d $DOCKER_CONFIG ]] + case "$1" in + auth) + [[ $# == 6 && $2 == login && $3 == ghcr.io && + $4 == --username && $5 == tekton-robot && $6 == --password-stdin ]] + token=$(cat) + [[ $token == test-registry-token ]] + event login "$@" + printf '{}\n' > "$DOCKER_CONFIG/config.json" + ;; + cp) + [[ $# == 3 && $2 == "$IMAGE" && $3 == ghcr.io/tektoncd/tekton-kueue:latest ]] + event image-copy "$@" + ;; + digest) + [[ $# == 2 && $2 == ghcr.io/tektoncd/tekton-kueue:latest ]] + event image-digest "$@" + if [[ $MODE == image-failure ]]; then printf 'sha256:%064d\n' 0; else echo "$DIGEST"; fi + ;; + *) exit 90 ;; + esac + ;; + oci) + [[ $1 == --config-file && $2 == "$CASE_DIR/work/oci/config" ]] + [[ -s $2 && -s $CASE_DIR/work/oci/key.pem ]] + if [[ $(uname -s) == Darwin ]]; then mode=(-f %Lp); else mode=(-c %a); fi + [[ $(stat "${mode[@]}" "$2") == 600 ]] + [[ $(stat "${mode[@]}" "$CASE_DIR/work/oci") == 700 ]] + shift 2 + if [[ $* == 'os ns get --query data --raw-output' ]]; then + event namespace "$@" + echo test-namespace + elif [[ $# == 12 && $1 == os && $2 == object && $3 == put && + $4 == --namespace-name && $5 == test-namespace && + $6 == --bucket-name && $7 == tekton-releases && $8 == --name && + ${10} == --file && ${12} == --force ]]; then + file=${9#tekton-kueue/latest/} + case "$file" in release.yaml|release.notags.yaml|SHA256SUMS) ;; *) exit 90 ;; esac + [[ $9 == "tekton-kueue/latest/$file" && ${11} == "$CASE_DIR/work/verified/$file" ]] + cmp "${11}" "$CASE_DIR/$file" + event bucket-put "$file" + else exit 90; fi + ;; + *) exit 90 ;; + esac + exit 0 +fi + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'status=$?; if [[ $status == 0 ]]; then rm -rf "$tmp"; else echo "test artifacts: $tmp" >&2; fi' EXIT +mkdir -p "$tmp/checkout/hack" "$tmp/bin" +mkdir "$tmp/caller-docker" +printf 'caller configuration\n' > "$tmp/caller-docker/config.json" +cp "$tmp/caller-docker/config.json" "$tmp/caller-config-original" +export DOCKER_CONFIG="$tmp/caller-docker" +cp "$root/hack/finalize-release.sh" "$tmp/checkout/hack/" +for name in verify-release-artifacts.sh smoke-release.sh; do + cp "${BASH_SOURCE[0]}" "$tmp/checkout/hack/$name" +done +for name in gh curl crane oci; do + cp "${BASH_SOURCE[0]}" "$tmp/bin/$name" + chmod +x "$tmp/bin/$name" +done +export PATH="$tmp/bin:$PATH" FINALIZER_TEST_CHILD=true +export GITHUB_REPOSITORY=tektoncd/tekton-kueue GITHUB_REF=refs/heads/main +export GH_TOKEN=test-bot-token GHCR_TOKEN=test-registry-token +export OCI_API_KEY=test-private-key OCI_FINGERPRINT=test-fingerprint +export OCI_TENANCY_OCID=test-tenancy OCI_USER_OCID=test-user OCI_REGION=test-region +export VERSION=v0.4.3 COMMIT=0123456789abcdef0123456789abcdef01234567 +export DIGEST=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export IMAGE="ghcr.io/tektoncd/tekton-kueue@$DIGEST" +export CASE_DIR CALLS MODE +cases=0 +check() { + jq -e -s "$1" "$CALLS" >/dev/null || { + echo "failed event assertion: $MODE: $1" >&2; exit 1; + } +} +run_case() { + MODE=$1 + CASE_DIR="$tmp/$MODE" + CALLS="$CASE_DIR/calls.jsonl" + mkdir "$CASE_DIR" + : > "$CALLS" + cat > "$CASE_DIR/key.pub" <<'EOF' +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo= +-----END PUBLIC KEY----- +EOF + local operation=finalize latest=false version=$VERSION ref=refs/heads/main + case "$MODE" in + success|older-latest|image-failure|bucket-failure|wrong-latest) latest=true ;; + repair-*) operation=repair ;; + invalid-input) version=../v0.4.3 ;; + bad-ref) ref=refs/heads/untrusted ;; + esac + printf 'v0.4.2\n' > "$CASE_DIR/latest" + [[ $MODE != older-latest ]] || printf 'v0.5.0\n' > "$CASE_DIR/latest" + [[ $MODE != repair-published ]] || printf '%s\n' "$VERSION" > "$CASE_DIR/latest" + jq -n --arg mode "$MODE" '{ + can_admins_bypass:($mode == "admin-bypass"), + protection_rules:(if $mode == "unprotected" then [] else + [{type:"required_reviewers",prevent_self_review:($mode != "self-review"), + reviewers:[{type:"Team",reviewer:{id:18766447}}]}] end)}' > "$CASE_DIR/environment.json" + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg mode "$MODE" '{ + id:123,tag_name:$version,target_commitish:$commit,draft:($mode != "repair-published"), + body:(if $mode == "generated-notes" then "Generated release notes\n" + else "Release notes\n\n## Attestation\nold\n\n\nKeep this\n" end), + assets:[{name:"release.yaml",id:11},{name:"release.notags.yaml",id:12},{name:"SHA256SUMS",id:13}]} + ' > "$CASE_DIR/original.json" + jq --arg mode "$MODE" ' + if $mode == "wrong-source" then .target_commitish = "main" + elif $mode == "changed-assets" then .assets[0].id = 99 + else . end' "$CASE_DIR/original.json" > "$CASE_DIR/release.json" + printf 'image: %s\n' "$IMAGE" > "$CASE_DIR/release.yaml" + cp "$CASE_DIR/release.yaml" "$CASE_DIR/release.notags.yaml" + (cd "$CASE_DIR"; openssl dgst -sha256 -r release.yaml release.notags.yaml) > "$CASE_DIR/SHA256SUMS" + # Hash exactly the decoded verified bytes, not the encoded envelope or JSON + # pretty-printing. Both finalizer and Rekor fixture use real SHA-256. + jq -cn --arg commit "$COMMIT" --arg digest "${DIGEST#sha256:}" '{ + _type:"https://in-toto.io/Statement/v1",predicateType:"https://slsa.dev/provenance/v1", + subject:[{name:"ghcr.io/tektoncd/tekton-kueue",digest:{sha256:$digest}}], + predicate:{buildDefinition:{resolvedDependencies:[{ + uri:"git+https://github.com/tektoncd/tekton-kueue.git",digest:{sha1:$commit}}]}}} + ' > "$CASE_DIR/payload.json" + jq -n --rawfile payload "$CASE_DIR/payload.json" '[$payload | @base64]' > "$CASE_DIR/verified-payloads.json" + local status=0 + GITHUB_REF=$ref bash "$tmp/checkout/hack/finalize-release.sh" \ + "$operation" "$version" "$COMMIT" 123 "$DIGEST" "$CASE_DIR/key.pub" "$latest" \ + "$CASE_DIR/work" > "$CASE_DIR/output" 2>&1 || status=$? + if [[ $status == 90 ]]; then + echo "unexpected executable boundary: $MODE; see $CASE_DIR/output" >&2; exit 1; + fi + case "$MODE" in + success|repair-draft|repair-published|generated-notes) + [[ $status == 0 ]] || { echo "failed $MODE; see $CASE_DIR/output" >&2; exit 1; } + local expected_latest=$latest + [[ $MODE != repair-published ]] || expected_latest=true + jq -e --arg version "$VERSION" --arg commit "$COMMIT" --arg latest "$expected_latest" \ + --arg operation "$operation" --slurpfile original "$CASE_DIR/original.json" ' + .tag_name == $version and .target_commitish == $commit and .make_latest == $latest + and .draft == (if $operation == "repair" then $original[0].draft else false end) + and (if $original[0].body | contains("## Attestation") then + (.body | startswith("Release notes\n\n") and endswith("\nKeep this\n")) + else .body | startswith($original[0].body) end) + and (.body | contains("abcdef"))' "$CASE_DIR/sent-patch.json" >/dev/null + jq -e -s '.[0] == (.[1] * (.[2] | del(.make_latest)))' \ + "$CASE_DIR/release.json" "$CASE_DIR/original.json" "$CASE_DIR/sent-patch.json" >/dev/null + check 'map(.event) as $e | ($e|index("verify")) < ($e|index("rekor-entry")) + and ($e|index("rekor-entry")) < ($e|index("release")) + and ($e|index("release")) < ($e|index("patch")) + and all(.[] | select(.event == "patch"); .token == "test-bot-token") + and ([.[] | select(.event == "identity")][-1].token == "test-bot-token")' + if [[ $operation == repair ]]; then + check 'all(.[]; .event != "smoke" and .event != "image-copy" and .event != "bucket-put" and .event != "login")' + elif [[ $latest == true ]]; then + check 'map(.event) as $e | ($e|index("verify")) < ($e|index("smoke")) + and ($e|index("smoke")) < ($e|index("image-copy")) + and ($e|index("image-copy")) < ($e|index("image-digest")) + and ($e|index("image-digest")) < ($e|index("bucket-put")) + and ($e|rindex("bucket-read")) < ($e|index("patch")) + and ([.[] | select(.event == "bucket-put") | .args[0]] | sort) == + ["SHA256SUMS","release.notags.yaml","release.yaml"]' + else + check 'map(.event) as $e | ($e|index("verify")) < ($e|index("smoke")) + and ($e|index("smoke")) < ($e|index("patch")) + and all(.[]; .event != "image-copy" and .event != "bucket-put" and .event != "login")' + fi + ;; + *) + [[ $status != 0 ]] || { echo "unexpected success: $MODE" >&2; exit 1; } + case "$MODE" in + verifier-failure|smoke-failure|api-failure) [[ $status == 42 ]] ;; + *) [[ $status == 1 ]] ;; + esac + case "$MODE" in + api-failure|wrong-result|wrong-latest) + check 'any(.[]; .event == "patch")' + [[ $MODE != api-failure ]] || cmp "$CASE_DIR/original.json" "$CASE_DIR/release.json" + ;; + *) + check 'all(.[]; .event != "patch")' + cmp "$CASE_DIR/release.json" "$CASE_DIR/work/current-release.json" 2>/dev/null || + [[ ! -e $CASE_DIR/work/current-release.json ]] + ;; + esac + case "$MODE" in + image-failure|bucket-failure) + check 'any(.[]; .event == "image-copy")' + cmp "$CASE_DIR/original.json" "$CASE_DIR/release.json" + ;; + api-failure|wrong-result|wrong-latest) ;; + *) + check 'all(.[]; .event != "image-copy" and .event != "bucket-put" and .event != "login")' + ;; + esac + case "$MODE" in + wrong-actor|unprotected|self-review|admin-bypass|older-latest|invalid-input|bad-ref) + check 'all(.[]; .event != "verify" and .event != "smoke")' ;; + verifier-failure|bad-digest) + check 'any(.[]; .event == "verify") and all(.[]; .event != "smoke")' ;; + smoke-failure) check 'any(.[]; .event == "smoke")' ;; + rekor-mismatch) check 'any(.[]; .event == "rekor-entry")' ;; + wrong-source|changed-assets) check 'any(.[]; .event == "release")' ;; + image-failure) + check 'any(.[]; .event == "image-digest") and all(.[]; .event != "bucket-put")' ;; + bucket-failure) check 'any(.[]; .event == "bucket-read")' ;; + esac + ;; + esac + [[ ! -e $CASE_DIR/work/docker && ! -e $CASE_DIR/work/oci ]] + cmp "$tmp/caller-docker/config.json" "$tmp/caller-config-original" + cases=$((cases + 1)) + echo "PASS $MODE" +} + +# Fifteen focused groups; related protection, repair and API outcomes share a +# group rather than multiplying every failure across finalize/repair/latest. +run_case success +run_case generated-notes +for mode in repair-draft repair-published; do run_case "$mode"; done +run_case wrong-actor +for mode in unprotected self-review admin-bypass; do run_case "$mode"; done +run_case bad-digest +run_case verifier-failure +run_case smoke-failure +run_case wrong-source +run_case changed-assets +run_case rekor-mismatch +run_case older-latest +run_case image-failure +run_case bucket-failure +for mode in api-failure wrong-result wrong-latest; do run_case "$mode"; done +for mode in invalid-input bad-ref; do run_case "$mode"; done +echo "PASS $cases deterministic release finalization cases (16 focused groups)" From 4be61fb0e932e153761758eadfeb88280d184575 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:39:43 +0530 Subject: [PATCH 07/13] fix: await webhook replacement endpoints before release admission Watch old Pod deletion and the Service endpoint switch after rollout instead of relying only on Deployment availability. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- hack/smoke-release.sh | 13 +++++++++++++ hack/smoke-release_test.sh | 12 ++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/hack/smoke-release.sh b/hack/smoke-release.sh index f3cd989de..a70b59e4e 100644 --- a/hack/smoke-release.sh +++ b/hack/smoke-release.sh @@ -156,8 +156,21 @@ metadata: spec: clusterQueue: release-smoke YAML +k get pods -n tekton-kueue -l app.kubernetes.io/name=tekton-kueue-webhook -o json \ + > "$work/old-webhook-pods.json" k rollout restart deployment/tekton-kueue-webhook -n tekton-kueue k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +# Deployment availability alone can precede removal of the old Service endpoint. +# Watch the old Pods disappear and the endpoint switch before admission. +while IFS= read -r pod; do + k wait --for=delete "pod/$pod" -n tekton-kueue --timeout=300s +done < <(jq -r '.items[].metadata.name' "$work/old-webhook-pods.json") +k get pods -n tekton-kueue -l app.kubernetes.io/name=tekton-kueue-webhook -o json \ + > "$work/webhook-pods.json" +pod=$(jq -er '[.items[] | select(.metadata.deletionTimestamp == null)] | + if length == 1 then .[0].metadata.name else error("expected one webhook Pod") end' "$work/webhook-pods.json") +k wait --for=jsonpath='{.subsets[0].addresses[0].targetRef.name}'="$pod" \ + endpoints/tekton-kueue-webhook-service -n tekton-kueue --timeout=120s k wait --for=condition=Active clusterqueue/release-smoke --timeout=120s k wait --for=condition=Active localqueue/pipelines-queue -n release-smoke --timeout=120s k create -f - -o json > "$artifacts/created-pipelinerun.json" <<'YAML' diff --git a/hack/smoke-release_test.sh b/hack/smoke-release_test.sh index 05fcbda92..546d84953 100644 --- a/hack/smoke-release_test.sh +++ b/hack/smoke-release_test.sh @@ -46,12 +46,14 @@ if [[ ${SMOKE_FAKE:-} == true ]]; then ;; rollout) if [[ $MODE == readiness && $* == *"status deployment/tekton-kueue-webhook"* ]]; then exit 42; fi + if [[ $* == *"restart deployment/tekton-kueue-webhook"* ]]; then touch "$CASE_DIR/restarted"; fi ;; wait) [[ $* == *--timeout=* ]] || exit 95 if [[ $MODE == run && $* == *condition=Succeeded* ]]; then exit 42; fi if [[ $MODE == admission && $* == *condition=Admitted* ]]; then exit 42; fi if [[ $MODE == workload && $* == *--for=create* ]]; then exit 42; fi + if [[ $MODE == endpoint && $* == *endpoints/tekton-kueue-webhook-service* ]]; then exit 42; fi ;; create) "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ @@ -63,7 +65,11 @@ if [[ ${SMOKE_FAKE:-} == true ]]; then spec:{status:(if $mode == "wrong-pending" then "Pending" else "PipelineRunPending" end)}}' ;; get) - if [[ $2 == workload/* ]]; then + if [[ $2 == pods && $* == *"-o json"* ]]; then + name=old-webhook + [[ ! -e $CASE_DIR/restarted ]] || name=new-webhook + jq -n --arg name "$name" '{items:[{metadata:{name:$name}}]}' + elif [[ $2 == workload/* ]]; then jq -n '{metadata:{ownerReferences:[{uid:"smoke-uid",kind:"PipelineRun",controller:true}]}, spec:{queueName:"pipelines-queue",podSets:[{template:{spec:{containers:[ {resources:{requests:{cpu:"1","tekton.dev/pipelineruns":"1"}}} @@ -156,6 +162,8 @@ run_case() { and any(.[]; .args | index("--for=create")) and any(.[]; .args | index("--for=condition=Admitted")) and any(.[]; .args | index("--for=condition=Succeeded")) + and any(.[]; .args | index("pod/old-webhook")) + and any(.[]; .args | index("endpoints/tekton-kueue-webhook-service")) and any(.[]; .args | index("https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml")) and any(.[]; .args | index("https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml")) and any(.[]; .args | index("https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml")) @@ -172,7 +180,7 @@ run_case() { } for mode in digest mismatch malformed missing incomplete artifacts success success-again \ - creation readiness mutation wrong-pending workload admission run cleanup; do + creation readiness mutation wrong-pending endpoint workload admission run cleanup; do run_case "$mode" done first=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success/calls.jsonl") From 81295f99f23572fdfdd20f75d24e2854b4a79206 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:47:13 +0530 Subject: [PATCH 08/13] fix: avoid shell interpolation warnings in release summaries Print candidate identifiers without Markdown command delimiters so the required workflow shell lint accepts the summary. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- .github/workflows/finalize-release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/finalize-release.yaml b/.github/workflows/finalize-release.yaml index f5a39e99c..2edb71784 100644 --- a/.github/workflows/finalize-release.yaml +++ b/.github/workflows/finalize-release.yaml @@ -104,8 +104,8 @@ jobs: { printf '### Release %s\n\n' "$VERSION" printf -- '- Operation: %s\n' "$OPERATION" - printf -- '- Source: `%s`\n' "$COMMIT" - printf -- '- Image: `%s`\n' "$DIGEST" + printf -- '- Source: %s\n' "$COMMIT" + printf -- '- Image: %s\n' "$DIGEST" printf -- '- Release: https://github.com/tektoncd/tekton-kueue/releases/tag/%s\n' "$VERSION" } >> "$GITHUB_STEP_SUMMARY" From 2d7bb6e805eac0b7f0bf540fd854e47bf2639b66 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:55:40 +0530 Subject: [PATCH 09/13] feat: resume partial release drafts from verified existing artifacts Verify the signed versioned bundle before restoring only missing draft assets; reject conflicting source and bytes. Wire authenticated preflight and retain all existing build evidence. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- .github/workflows/ci.yml | 4 +- .github/workflows/recover-release.yaml | 65 ++++++++ docs/releases.md | 26 ++++ hack/recover-release.sh | 128 ++++++++++++++++ hack/recover-release_test.sh | 200 +++++++++++++++++++++++++ hack/verify-release-artifacts.sh | 62 +++++--- hack/verify-release-artifacts_test.sh | 37 ++++- tekton/release-pipeline.yaml | 4 +- 8 files changed, 499 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/recover-release.yaml create mode 100644 hack/recover-release.sh create mode 100644 hack/recover-release_test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acfee2867..bb8b15fbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,10 +101,12 @@ jobs: run: | bash -n hack/verify-release-artifacts.sh hack/verify-release-artifacts_test.sh \ hack/smoke-release.sh hack/smoke-release_test.sh \ - hack/finalize-release.sh hack/finalize-release_test.sh + hack/finalize-release.sh hack/finalize-release_test.sh \ + hack/recover-release.sh hack/recover-release_test.sh bash hack/verify-release-artifacts_test.sh bash hack/smoke-release_test.sh bash hack/finalize-release_test.sh + bash hack/recover-release_test.sh ci-summary: name: CI summary diff --git a/.github/workflows/recover-release.yaml b/.github/workflows/recover-release.yaml new file mode 100644 index 000000000..4f3a1aef9 --- /dev/null +++ b/.github/workflows/recover-release.yaml @@ -0,0 +1,65 @@ +name: Recover Release Draft + +"on": + workflow_dispatch: + inputs: + version: + description: Existing versioned candidate, for example v0.4.4 + required: true + type: string + source_commit: + description: Full approved source commit + required: true + type: string + image_digest: + description: Exact approved sha256 image digest + required: true + type: string + +permissions: + contents: read + +concurrency: + group: release-finalization + cancel-in-progress: false + +jobs: + recover: + if: github.repository == 'tektoncd/tekton-kueue' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: kueue-release-finalization + env: + GH_TOKEN: ${{ secrets.CHATOPS_TOKEN }} + RELEASE_SIGNING_PUBLIC_KEY: ${{ vars.RELEASE_SIGNING_PUBLIC_KEY }} + GOTOOLCHAIN: local + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Require configured signing trust + run: | + test -n "$RELEASE_SIGNING_PUBLIC_KEY" + printf '%s\n' "$RELEASE_SIGNING_PUBLIC_KEY" > "$RUNNER_TEMP/release-signing.pub" + openssl pkey -pubin -in "$RUNNER_TEMP/release-signing.pub" -noout + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Install pinned verification tools + run: | + export GOBIN="$RUNNER_TEMP/release-tools" + mkdir -p "$GOBIN" + go install github.com/sigstore/cosign/v3/cmd/cosign@v3.0.6 + go install github.com/google/go-containerregistry/cmd/crane@v0.20.6 + printf '%s\n' "$GOBIN" >> "$GITHUB_PATH" + - name: Verify and resume the existing draft + env: + VERSION: ${{ inputs.version }} + COMMIT: ${{ inputs.source_commit }} + DIGEST: ${{ inputs.image_digest }} + run: | + bash hack/recover-release.sh "$VERSION" "$COMMIT" "$DIGEST" \ + "$RUNNER_TEMP/release-signing.pub" "$RUNNER_TEMP/release-recovery" + printf 'Recovered verified draft for %s. Run Finalize Release separately after approval.\n' \ + "$VERSION" >> "$GITHUB_STEP_SUMMARY" diff --git a/docs/releases.md b/docs/releases.md index 41743c6bb..cdf570641 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -103,12 +103,38 @@ to complete that promotion without rebuilding it. Do not delete versioned artifacts to work around a partial release. Preserve them and their original source/signing evidence for recovery. +## Recovering a partial draft + +Run **Recover Release Draft** on `main` with the approved version, full source +commit and image digest. It uses the same protected environment, independent +signing key and robot credential as finalization. It can create a missing draft +or attach missing files to an existing draft without rebuilding. + +Recovery requires a complete, signed versioned bucket bundle: `release.yaml`, +`release.notags.yaml` and `SHA256SUMS`. It verifies the bundle and provenance +before any draft mutation. Existing tags must resolve to the approved commit, +and existing assets must match the verified bytes. Conflicting or published +releases are rejected; matching assets and existing notes are preserved. +After attaching only missing files, it verifies the recovered draft against +the bucket again. Run **Finalize Release** separately to smoke-test and publish. + +An incomplete bucket bundle or missing signing evidence cannot be reconstructed +safely by this workflow. It stops without rebuilding or deleting anything; +preserve the state for an operator to recover the original artifacts. + +The release pipeline's authenticated preflight rejects any existing tag, +draft, image or versioned object and directs operators to recovery. Its +`github-token/GITHUB_TOKEN` namespace secret must belong to `tekton-robot` and +have repository push permission, so draft lookup cannot silently miss drafts. +Authentication, rate-limit and ambiguous registry/storage failures block staging. + ## Developer checks ```sh bash hack/verify-release-artifacts_test.sh bash hack/smoke-release_test.sh bash hack/finalize-release_test.sh +bash hack/recover-release_test.sh ``` These deterministic boundary tests do not mutate GitHub, the registry or diff --git a/hack/recover-release.sh b/hack/recover-release.sh new file mode 100644 index 000000000..a86f6840c --- /dev/null +++ b/hack/recover-release.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release recovery failed: $*" >&2; exit 1; } +robot() { + [[ $(gh api user --jq .login) == tekton-robot ]] || fail "GH_TOKEN must belong to tekton-robot" +} +[[ $# == 5 ]] || fail "usage: $0 VERSION COMMIT DIGEST PUBLIC_KEY_FILE WORK_DIR" +version=$1 +commit=$2 +digest=$3 +key=$4 +work=$5 +repository=tektoncd/tekton-kueue +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ && $commit =~ ^[0-9a-f]{40}$ && + $digest =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid candidate identity" +[[ ${GITHUB_REPOSITORY:-} == "$repository" && ${GITHUB_REF:-} == refs/heads/main ]] || + fail "run the trusted main-branch workflow" +[[ -n ${GH_TOKEN:-} && -s $key && -r $key ]] || fail "robot token and trusted key are required" +[[ -n $work && ! -L $work ]] || fail "invalid work directory" +if [[ -e $work ]]; then + [[ -d $work && -z $(find "$work" -mindepth 1 -print -quit) ]] || fail "work directory must be empty" +fi +mkdir -p "$work" +work=$(cd "$work" && pwd) +robot +[[ $(gh api "repos/$repository" --jq .permissions.push) == true ]] || + fail "repository push permission is required to inspect drafts" +gh api "repos/$repository/environments/kueue-release-finalization" > "$work/environment.json" +jq -e ' + .can_admins_bypass == false and + any(.protection_rules[]?; + .type == "required_reviewers" and .prevent_self_review == true and + any(.reviewers[]?; .type == "Team" and .reviewer.id == 18766447)) +' "$work/environment.json" >/dev/null || fail "release approval environment is not protected" + +# No GitHub mutation is allowed until a complete, signed existing bundle passes. +bash "$root/hack/verify-release-artifacts.sh" "$version" "$commit" bucket "$key" "$work/verified" +jq -e --arg digest "$digest" '.digest == $digest' "$work/verified/release-info.json" >/dev/null || + fail "verified digest differs from approved candidate" + +check_tag() { + local object sha + gh api --paginate --slurp "repos/$repository/git/matching-refs/tags/$version" > "$work/refs.json" + jq -e --arg ref "refs/tags/$version" ' + [.[][] | select(.ref == $ref)] | length <= 1 + ' "$work/refs.json" >/dev/null || fail "ambiguous tag" + object=$(jq -c --arg ref "refs/tags/$version" ' + [.[][] | select(.ref == $ref)][0].object // null' "$work/refs.json") + while [[ $object != null ]]; do + sha=$(jq -r .sha <<< "$object") + [[ $sha =~ ^[0-9a-f]{40}$ ]] || fail "invalid tag object" + case $(jq -r .type <<< "$object") in + commit) [[ $sha == "$commit" ]] || fail "tag points to another source"; return ;; + tag) + gh api "repos/$repository/git/tags/$sha" > "$work/tag.json" + object=$(jq -ce '.object' "$work/tag.json") + ;; + *) fail "tag does not resolve to a commit" ;; + esac + done +} + +find_release() { + gh api --paginate --slurp "repos/$repository/releases?per_page=100" > "$work/releases.json" + jq --arg version "$version" '[.[][] | select(.tag_name == $version)]' \ + "$work/releases.json" > "$work/candidates.json" + jq -e 'length <= 1' "$work/candidates.json" >/dev/null || fail "multiple matching releases" + release_id=$(jq -r '.[0].id // empty' "$work/candidates.json") + [[ -z $release_id || $release_id =~ ^[1-9][0-9]*$ ]] || fail "invalid release ID" +} + +refresh() { + local file asset_id + check_tag + gh api "repos/$repository/releases/$release_id" > "$work/current.json" + jq -e --arg id "$release_id" --arg version "$version" --arg commit "$commit" ' + .id == ($id | tonumber) and .tag_name == $version + and .target_commitish == $commit and .draft == true + and (.assets | type == "array") + and ([.assets[] | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS")] + | (map(.name) | unique | length) == length + and all(.[]; (.id | type == "number") and .id > 0 and .id == (.id | floor)) + and (map(.id) | unique | length) == length) + ' "$work/current.json" >/dev/null || fail "release source, publication state or assets conflict" + for file in release.yaml release.notags.yaml SHA256SUMS; do + asset_id=$(jq -r --arg name "$file" '.assets[] | select(.name == $name) | .id' "$work/current.json") + if [[ -n $asset_id ]]; then + gh api --method GET -H 'Accept: application/octet-stream' \ + "repos/$repository/releases/assets/$asset_id" > "$work/existing-$file" + cmp -s "$work/existing-$file" "$work/verified/$file" || fail "conflicting existing asset: $file" + fi + done +} + +check_tag +find_release +if [[ -z $release_id ]]; then + gh api --method POST "repos/$repository/releases/generate-notes" \ + -f tag_name="$version" -f target_commitish="$commit" > "$work/notes.json" + jq -e '.body | type == "string" and length > 0' "$work/notes.json" >/dev/null || + fail "cannot generate release notes" + # Recheck absence immediately before creating; never replace another draft. + check_tag + find_release + if [[ -z $release_id ]]; then + jq --arg version "$version" --arg commit "$commit" ' + {tag_name:$version,target_commitish:$commit,draft:true, + name:$version,body:.body,make_latest:"false"}' "$work/notes.json" > "$work/create.json" + robot + gh api --method POST "repos/$repository/releases" --input "$work/create.json" > "$work/created.json" + release_id=$(jq -er '.id | select(type == "number" and . > 0 and . == floor)' "$work/created.json") + fi +fi +for file in release.yaml release.notags.yaml SHA256SUMS; do + refresh + if ! jq -e --arg name "$file" 'any(.assets[]; .name == $name)' "$work/current.json" >/dev/null; then + robot + # No --clobber: a concurrent attachment must fail, never replace bytes. + gh release upload "$version" "$work/verified/$file" --repo "$repository" + fi +done +refresh +bash "$root/hack/verify-release-artifacts.sh" "$version" "$commit" "$release_id" "$key" "$work/recovered" +jq -e --arg digest "$digest" '.digest == $digest and .draft == true' \ + "$work/recovered/release-info.json" >/dev/null || fail "recovered draft verification failed" +printf 'Verified draft ID: %s\nFinalize separately after approval; no rebuild or latest promotion occurred.\n' "$release_id" diff --git a/hack/recover-release_test.sh b/hack/recover-release_test.sh new file mode 100644 index 000000000..28f784cf9 --- /dev/null +++ b/hack/recover-release_test.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${RECOVERY_TEST_CHILD:-} == true ]]; then + event() { + jq -cn --arg event "$1" --args '{event:$event,args:$ARGS.positional}' \ + -- "${@:2}" >> "$CALLS" + } + case ${0##*/} in + verify-release-artifacts.sh) + [[ $# == 5 && $1 == "$VERSION" && $2 == "$COMMIT" && $4 == "$CASE_DIR/key.pub" ]] + openssl pkey -pubin -in "$4" -noout + event verify "$3" + [[ $MODE != verification-failure && $MODE != incomplete-bundle ]] || exit 42 + [[ $3 == bucket || $3 == 123 ]] || exit 90 + [[ $MODE != final-verification-failure || $3 == bucket ]] || exit 42 + mkdir -p "$5" + cp "$CASE_DIR"/{release.yaml,release.notags.yaml,SHA256SUMS} "$5/" + jq -n --arg digest "$DIGEST" --arg mode "$MODE" --arg id "$3" ' + {digest:(if $mode == "wrong-digest" then "sha256:" + ("b"*64) else $digest end), + draft:($id != "bucket"),release_id:(if $id == "bucket" then null else 123 end)} + ' > "$5/release-info.json" + ;; + gh) + if [[ $1 == release ]]; then + [[ $# == 6 && $2 == upload && $3 == "$VERSION" && + $5 == --repo && $6 == tektoncd/tekton-kueue ]] + file=${4##*/} + cmp "$4" "$CASE_DIR/$file" + # Mimic the real non-clobber upload contract. + if jq -e --arg name "$file" 'any(.assets[]; .name == $name)' "$CASE_DIR/state.json" >/dev/null; then + exit 90 + fi + [[ $MODE != upload-failure ]] || exit 42 + event upload "$file" + case $file in release.yaml) id=11 ;; release.notags.yaml) id=12 ;; SHA256SUMS) id=13 ;; *) exit 90 ;; esac + jq --arg name "$file" --argjson id "$id" '.assets += [{name:$name,id:$id}]' \ + "$CASE_DIR/state.json" > "$CASE_DIR/next.json" + mv "$CASE_DIR/next.json" "$CASE_DIR/state.json" + exit + fi + [[ $1 == api ]] || exit 90 + shift + case $1 in + user) + [[ $# == 3 && $2 == --jq && $3 == .login ]] + if [[ $MODE == wrong-actor ]]; then echo waveywaves; else echo tekton-robot; fi + ;; + repos/tektoncd/tekton-kueue) + [[ $# == 3 && $2 == --jq && $3 == .permissions.push ]] + if [[ $MODE == no-permission ]]; then echo false; else echo true; fi + ;; + repos/tektoncd/tekton-kueue/environments/kueue-release-finalization) + jq -n --arg mode "$MODE" '{can_admins_bypass:($mode == "admin-bypass"), + protection_rules:(if $mode == "unprotected" then [] else [{ + type:"required_reviewers",prevent_self_review:($mode != "self-review"), + reviewers:[{type:"Team",reviewer:{id:18766447}}]}] end)}' + ;; + --paginate) + [[ $# == 3 && $2 == --slurp ]] || exit 90 + case $3 in + repos/tektoncd/tekton-kueue/git/matching-refs/tags/*) + [[ $MODE != tag-api-failure ]] || exit 42 + if [[ $MODE == no-tag ]]; then echo '[[]]'; else + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg mode "$MODE" ' + [[{ref:("refs/tags/" + $version),object:{ + type:(if $mode == "annotated-tag" then "tag" else "commit" end), + sha:(if $mode == "wrong-tag" then "f"*40 + elif $mode == "annotated-tag" then "e"*40 else $commit end)}}]]' + fi + ;; + 'repos/tektoncd/tekton-kueue/releases?per_page=100') + [[ $MODE != lookup-failure ]] || exit 42 + jq --arg mode "$MODE" ' + if . == null then [[]] elif $mode == "duplicate-release" then [[.,.]] + else [[],[.]] end' "$CASE_DIR/state.json" + ;; + *) exit 90 ;; + esac + ;; + repos/tektoncd/tekton-kueue/git/tags/*) + [[ $MODE == annotated-tag ]] + jq -n --arg commit "$COMMIT" '{object:{type:"commit",sha:$commit}}' + ;; + repos/tektoncd/tekton-kueue/releases/123) + cat "$CASE_DIR/state.json" + ;; + --method) + if [[ $2 == POST && $3 == repos/tektoncd/tekton-kueue/releases/generate-notes ]]; then + [[ $# == 7 && $4 == -f && $5 == "tag_name=$VERSION" && + $6 == -f && $7 == "target_commitish=$COMMIT" ]] + [[ $MODE != notes-failure ]] || exit 42 + event notes + jq -n '{body:"Generated notes from existing commits"}' + elif [[ $2 == POST && $3 == repos/tektoncd/tekton-kueue/releases ]]; then + [[ $# == 5 && $4 == --input && $(< "$CASE_DIR/state.json") == null ]] + jq -e --arg version "$VERSION" --arg commit "$COMMIT" ' + .tag_name == $version and .target_commitish == $commit + and .draft == true and .make_latest == "false" and (.body | length > 0) + ' "$5" >/dev/null + [[ $MODE != create-failure ]] || exit 42 + event create + jq '. + {id:123,assets:[]}' "$5" > "$CASE_DIR/state.json" + cat "$CASE_DIR/state.json" + elif [[ $2 == GET && $3 == -H && $4 == 'Accept: application/octet-stream' ]]; then + [[ $# == 5 ]] + case ${5##*/} in 11) file=release.yaml ;; 12) file=release.notags.yaml ;; 13) file=SHA256SUMS ;; *) exit 90 ;; esac + cat "$CASE_DIR/$file" + [[ $MODE != conflicting-asset ]] || printf 'conflict\n' + else exit 90; fi + ;; + *) exit 90 ;; + esac + ;; + *) exit 90 ;; + esac + exit +fi + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'status=$?; if [[ $status == 0 ]]; then rm -rf "$tmp"; else echo "test artifacts: $tmp" >&2; fi' EXIT +mkdir -p "$tmp/checkout/hack" "$tmp/bin" +cp "$root/hack/recover-release.sh" "$tmp/checkout/hack/" +cp "${BASH_SOURCE[0]}" "$tmp/checkout/hack/verify-release-artifacts.sh" +cp "${BASH_SOURCE[0]}" "$tmp/bin/gh" +chmod +x "$tmp/bin/gh" +export PATH="$tmp/bin:$PATH" RECOVERY_TEST_CHILD=true GH_TOKEN=test-robot-token +export GITHUB_REPOSITORY=tektoncd/tekton-kueue GITHUB_REF=refs/heads/main +export VERSION=v0.4.3 COMMIT=0123456789abcdef0123456789abcdef01234567 +export DIGEST=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export MODE CASE_DIR CALLS +cases=0 +for MODE in missing-draft no-tag partial-assets complete annotated-tag \ + wrong-actor no-permission unprotected self-review admin-bypass verification-failure \ + incomplete-bundle wrong-digest wrong-tag tag-api-failure lookup-failure duplicate-release \ + wrong-source published duplicate-asset duplicate-asset-id conflicting-asset notes-failure create-failure \ + upload-failure final-verification-failure invalid-input bad-ref missing-key invalid-key; do + CASE_DIR="$tmp/$MODE" + mkdir "$CASE_DIR" + CALLS="$CASE_DIR/calls.jsonl" + : > "$CALLS" + cat > "$CASE_DIR/key.pub" <<'EOF' +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo= +-----END PUBLIC KEY----- +EOF + [[ $MODE != invalid-key ]] || printf 'not a public key\n' > "$CASE_DIR/key.pub" + printf 'verified manifest\n' > "$CASE_DIR/release.yaml" + cp "$CASE_DIR/release.yaml" "$CASE_DIR/release.notags.yaml" + (cd "$CASE_DIR"; openssl dgst -sha256 -r release.yaml release.notags.yaml) > "$CASE_DIR/SHA256SUMS" + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg mode "$MODE" ' + if (["missing-draft","no-tag","notes-failure","create-failure"] | index($mode)) != null + then null else + {id:123,tag_name:$version, + target_commitish:(if $mode == "wrong-source" then "main" else $commit end), + draft:($mode != "published"),body:"Keep existing notes",assets: + (if $mode == "complete" then + [{name:"release.yaml",id:11},{name:"release.notags.yaml",id:12},{name:"SHA256SUMS",id:13}] + elif $mode == "duplicate-asset" then [{name:"release.yaml",id:11},{name:"release.yaml",id:14}] + elif $mode == "duplicate-asset-id" then [{name:"release.yaml",id:11},{name:"release.notags.yaml",id:11}] + else [{name:"release.yaml",id:11}] end)} + end' > "$CASE_DIR/state.json" + cp "$CASE_DIR/state.json" "$CASE_DIR/original.json" + version=$VERSION + ref=refs/heads/main + key="$CASE_DIR/key.pub" + [[ $MODE != invalid-input ]] || version=../v0.4.3 + [[ $MODE != bad-ref ]] || ref=refs/heads/untrusted + [[ $MODE != missing-key ]] || key="$CASE_DIR/missing.pub" + status=0 + GITHUB_REF=$ref bash "$tmp/checkout/hack/recover-release.sh" \ + "$version" "$COMMIT" "$DIGEST" "$key" "$CASE_DIR/work" > "$CASE_DIR/output" 2>&1 || status=$? + case $MODE in + missing-draft|no-tag|partial-assets|complete|annotated-tag) + [[ $status == 0 ]] || { echo "failed $MODE; see $CASE_DIR/output" >&2; exit 1; } + jq -e '.draft == true and ([.assets[].name] | sort == + ["SHA256SUMS","release.notags.yaml","release.yaml"])' "$CASE_DIR/state.json" >/dev/null + if [[ $MODE != missing-draft && $MODE != no-tag ]]; then + jq -e -s '.[0].body == .[1].body' "$CASE_DIR/state.json" "$CASE_DIR/original.json" >/dev/null + fi + jq -e -s '.[0] == {event:"verify",args:["bucket"]} + and .[-1] == {event:"verify",args:["123"]}' "$CALLS" >/dev/null + writes=$(jq -s '[.[] | select(.event == "create" or .event == "upload")] | length' "$CALLS") + bash "$tmp/checkout/hack/recover-release.sh" "$VERSION" "$COMMIT" "$DIGEST" \ + "$key" "$CASE_DIR/again" >> "$CASE_DIR/output" 2>&1 + [[ $(jq -s '[.[] | select(.event == "create" or .event == "upload")] | length' "$CALLS") == "$writes" ]] + ;; + *) + [[ $status != 0 && $status != 90 ]] || { echo "unexpected $MODE status $status; see $CASE_DIR/output" >&2; exit 1; } + case $MODE in + upload-failure|final-verification-failure) ;; + *) jq -e -s 'all(.[]; .event != "create" and .event != "upload")' "$CALLS" >/dev/null ;; + esac + ;; + esac + cases=$((cases + 1)) + echo "PASS $MODE" +done +echo "PASS $cases deterministic release recovery cases" diff --git a/hack/verify-release-artifacts.sh b/hack/verify-release-artifacts.sh index 90848df82..8d82792dd 100644 --- a/hack/verify-release-artifacts.sh +++ b/hack/verify-release-artifacts.sh @@ -7,7 +7,7 @@ fail() { exit 1 } -[[ $# == 5 ]] || fail "usage: $0 VERSION EXPECTED_COMMIT RELEASE_ID PUBLIC_KEY_FILE OUTPUT_DIR" +[[ $# == 5 ]] || fail "usage: $0 VERSION EXPECTED_COMMIT RELEASE_ID|bucket PUBLIC_KEY_FILE OUTPUT_DIR" version=$1 commit=$2 release_id=$3 @@ -19,7 +19,7 @@ source_url=https://github.com/tektoncd/tekton-kueue [[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "invalid version" [[ $commit =~ ^[0-9a-f]{40}$ ]] || fail "invalid commit" -[[ $release_id =~ ^[1-9][0-9]*$ ]] || fail "invalid release ID" +[[ $release_id == bucket || $release_id =~ ^[1-9][0-9]*$ ]] || fail "invalid release ID" [[ $repository =~ ^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$ && ${repository##*/} != . && ${repository##*/} != .. ]] || fail "invalid GitHub repository" [[ -n ${GH_TOKEN:-} ]] || fail "GH_TOKEN is required" @@ -42,22 +42,34 @@ fi work=$(mktemp -d) trap 'rm -rf "$work"' EXIT -gh api --method GET "repos/$repository/releases/$release_id" > "$work/release.json" -jq -e --arg version "$version" --arg commit "$commit" --arg id "$release_id" ' - .id == ($id | tonumber) and .tag_name == $version and .target_commitish == $commit - and (.draft | type == "boolean") and (.assets | type == "array") - and ([.assets[] | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS")] - | length == 3 and (map(.name) | unique | length == 3) - and all(.[]; (.id | type == "number") and .id > 0 and .id == (.id | floor)) - and (map(.id) | unique | length == 3)) -' "$work/release.json" >/dev/null || fail "release metadata does not match candidate" +receipt_id=$release_id +if [[ $release_id == bucket ]]; then + receipt_id=null + printf '{"draft":null,"assets":[]}\n' > "$work/release.json" + for file in release.yaml release.notags.yaml SHA256SUMS; do + curl --fail --silent --show-error --location \ + "https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$version/$file" \ + --output "$work/$file" + [[ -s $work/$file ]] || fail "empty bucket artifact: $file" + done +else + gh api --method GET "repos/$repository/releases/$release_id" > "$work/release.json" + jq -e --arg version "$version" --arg commit "$commit" --arg id "$release_id" ' + .id == ($id | tonumber) and .tag_name == $version and .target_commitish == $commit + and (.draft | type == "boolean") and (.assets | type == "array") + and ([.assets[] | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS")] + | length == 3 and (map(.name) | unique | length == 3) + and all(.[]; (.id | type == "number") and .id > 0 and .id == (.id | floor)) + and (map(.id) | unique | length == 3)) + ' "$work/release.json" >/dev/null || fail "release metadata does not match candidate" -for file in release.yaml release.notags.yaml SHA256SUMS; do - asset_id=$(jq -r --arg name "$file" '.assets[] | select(.name == $name) | .id' "$work/release.json") - gh api --method GET -H "Accept: application/octet-stream" \ - "repos/$repository/releases/assets/$asset_id" > "$work/$file" - [[ -s $work/$file ]] || fail "empty asset: $file" -done + for file in release.yaml release.notags.yaml SHA256SUMS; do + asset_id=$(jq -r --arg name "$file" '.assets[] | select(.name == $name) | .id' "$work/release.json") + gh api --method GET -H "Accept: application/octet-stream" \ + "repos/$repository/releases/assets/$asset_id" > "$work/$file" + [[ -s $work/$file ]] || fail "empty asset: $file" + done +fi # Parse before hashing: never pass downloaded filenames to a checksum tool. declare -A seen=() @@ -72,12 +84,14 @@ while IFS= read -r line || [[ -n $line ]]; do [[ ${actual%% *} == "$hash" ]] || fail "checksum mismatch: $file" done < "$work/SHA256SUMS" [[ ${#seen[@]} == 2 ]] || fail "SHA256SUMS must contain exactly the two manifests" -for file in release.yaml release.notags.yaml SHA256SUMS; do - curl --fail --silent --show-error --location \ - "https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$version/$file" \ - --output "$work/bucket-$file" - cmp -s "$work/$file" "$work/bucket-$file" || fail "bucket mismatch: $file" -done +if [[ $release_id != bucket ]]; then + for file in release.yaml release.notags.yaml SHA256SUMS; do + curl --fail --silent --show-error --location \ + "https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$version/$file" \ + --output "$work/bucket-$file" + cmp -s "$work/$file" "$work/bucket-$file" || fail "bucket mismatch: $file" + done +fi echo "PASS release metadata, checksums and bucket copies" # Kustomize emits block YAML with top-level metadata and scalar image fields. @@ -164,7 +178,7 @@ jq -e -s --arg digest "${digest#sha256:}" --arg commit "$commit" ' fail "no verified provenance matching source and digest" echo "PASS configured-key signature and SLSA provenance" -jq -n --argjson release_id "$release_id" --arg version "$version" --arg commit "$commit" \ +jq -n --argjson release_id "$receipt_id" --arg version "$version" --arg commit "$commit" \ --arg image "$image" --arg digest "$digest" --slurpfile release "$work/release.json" \ '{release_id: $release_id, version: $version, commit: $commit, image: $image, digest: $digest, draft: $release[0].draft, diff --git a/hack/verify-release-artifacts_test.sh b/hack/verify-release-artifacts_test.sh index f3e038824..249547bf7 100644 --- a/hack/verify-release-artifacts_test.sh +++ b/hack/verify-release-artifacts_test.sh @@ -69,6 +69,7 @@ cat > "$work/bin/fake" <<'EOF' set -euo pipefail case "${0##*/}" in gh) + [[ ${VERIFY_MODE:-123} != bucket ]] || exit 90 [[ $1 == api && $2 == --method && $3 == GET ]] || exit 90 if [[ $4 == repos/tektoncd/tekton-kueue/releases/123 ]]; then if [[ $SCENARIO == malformed-metadata ]]; then printf '{\n'; exit; fi @@ -115,7 +116,9 @@ case "${0##*/}" in [[ $5 == "$prefix"* ]] || exit 90 file=${5#"$prefix"} case "$file" in release.yaml|release.notags.yaml|SHA256SUMS) ;; *) exit 90 ;; esac + [[ $SCENARIO != "missing-bucket-$file" ]] || exit 22 cat "$FIXTURES/$file" > "$7" + [[ $SCENARIO != "empty-bucket-$file" ]] || : > "$7" [[ $SCENARIO != "bucket-$file" ]] || printf '\nchanged\n' >> "$7" ;; crane) @@ -191,7 +194,7 @@ done run_verifier() { bash "$root/hack/verify-release-artifacts.sh" \ - "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" + "$VERSION" "$COMMIT" "${VERIFY_MODE:-123}" "$work/key.pub" "$work/output" } passed=0 @@ -235,6 +238,38 @@ for SCENARIO in success published build-config-source multiple-attestations arra passed=$((passed + 1)) done +export VERIFY_MODE=bucket +for SCENARIO in success build-config-source signature-failure attestation-failure \ + bad-source bad-commit bad-digest missing-platform bad-source-label \ + wrong-image-repository inconsistent-image missing-webhook \ + bucket-release.yaml bucket-release.notags.yaml bucket-SHA256SUMS \ + missing-bucket-release.yaml missing-bucket-release.notags.yaml missing-bucket-SHA256SUMS \ + empty-bucket-release.yaml empty-bucket-release.notags.yaml empty-bucket-SHA256SUMS; do + export SCENARIO + rm -rf "$work/output" + write_artifacts + status=0 + run_verifier > "$work/log" 2>&1 || status=$? + case "$SCENARIO" in + success|build-config-source) + [[ $status == 0 ]] || { cat "$work/log" >&2; exit 1; } + jq -e --arg commit "$COMMIT" --arg image "$IMAGE" --arg version "$VERSION" \ + --arg digest "sha256:$DIGEST" ' + . == {release_id:null,version:$version,commit:$commit,image:$image, + digest:$digest,draft:null,assets:[]}' "$work/output/release-info.json" >/dev/null + for file in release.yaml release.notags.yaml SHA256SUMS; do + cmp "$FIXTURES/$file" "$work/output/$file" + done + ;; + *) + [[ $status != 0 && ! -e $work/output/release-info.json ]] || { + echo "unexpected bucket verification success: $SCENARIO" >&2; exit 1; + } + ;; + esac + passed=$((passed + 1)) +done +unset VERIFY_MODE export SCENARIO=success expect_failure() { if "$@" > "$work/log" 2>&1; then diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index 3e0ca03d1..c54507356 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -94,7 +94,7 @@ spec: - name: url value: https://github.com/tektoncd/plumbing - name: revision - value: 0dc3174db86dcfa70724eced275ba4c1b2e25e76 + value: 39fbe8fe22b3336190d3f37fdb11c76eb2660419 - name: pathInRepo value: tekton/resources/release/base/prerelease_checks_oci.yaml params: @@ -104,6 +104,8 @@ spec: value: $(params.versionTag) - name: releaseBucket value: $(params.releaseBucket)/$(params.repoName) + - name: imageRepository + value: $(params.imageRegistry)/$(params.imageRegistryPath) workspaces: - name: source-to-release workspace: workarea From 74ec090640a386aa1d8e76aa22299f9510c7607c Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 22:05:15 +0530 Subject: [PATCH 10/13] fix: verify release tags against the approved source Peel existing tags and fail on source conflicts or lookup errors. Permit missing tags only for candidates that have not been published. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- hack/verify-release-artifacts.sh | 3 +++ hack/verify-release-artifacts_test.sh | 18 ++++++++++++-- hack/verify-release-tag.sh | 34 +++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 hack/verify-release-tag.sh diff --git a/hack/verify-release-artifacts.sh b/hack/verify-release-artifacts.sh index 8d82792dd..5fff2bd38 100644 --- a/hack/verify-release-artifacts.sh +++ b/hack/verify-release-artifacts.sh @@ -178,6 +178,9 @@ jq -e -s --arg digest "${digest#sha256:}" --arg commit "$commit" ' fail "no verified provenance matching source and digest" echo "PASS configured-key signature and SLSA provenance" +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +bash "$root/hack/verify-release-tag.sh" "$version" "$commit" "$(jq -r '.draft != false' "$work/release.json")" + jq -n --argjson release_id "$receipt_id" --arg version "$version" --arg commit "$commit" \ --arg image "$image" --arg digest "$digest" --slurpfile release "$work/release.json" \ '{release_id: $release_id, version: $version, commit: $commit, image: $image, diff --git a/hack/verify-release-artifacts_test.sh b/hack/verify-release-artifacts_test.sh index 249547bf7..c1f38f3c9 100644 --- a/hack/verify-release-artifacts_test.sh +++ b/hack/verify-release-artifacts_test.sh @@ -69,6 +69,19 @@ cat > "$work/bin/fake" <<'EOF' set -euo pipefail case "${0##*/}" in gh) + if [[ $* == "api --paginate --slurp repos/tektoncd/tekton-kueue/git/matching-refs/tags/$VERSION" ]]; then + [[ $SCENARIO != tag-api-failure ]] || exit 42 + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + if $scenario == "missing-tag" or $scenario == "published-missing-tag" then [[]] + else [[{ref:("refs/tags/" + $version),object:{ + type:(if $scenario == "annotated-tag" or $scenario == "wrong-annotated-tag" then "tag" else "commit" end), + sha:(if $scenario == "wrong-tag" then "f"*40 else $commit end)}}]] end' + exit + elif [[ $* == "api repos/tektoncd/tekton-kueue/git/tags/$COMMIT" ]]; then + jq -n --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {object:{type:"commit",sha:(if $scenario == "wrong-annotated-tag" then "f"*40 else $commit end)}}' + exit + fi [[ ${VERIFY_MODE:-123} != bucket ]] || exit 90 [[ $1 == api && $2 == --method && $3 == GET ]] || exit 90 if [[ $4 == repos/tektoncd/tekton-kueue/releases/123 ]]; then @@ -76,7 +89,7 @@ case "${0##*/}" in jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' {id:123,tag_name:$version,target_commitish:$commit,draft:true,assets:[ {id:11,name:"release.yaml"},{id:12,name:"release.notags.yaml"},{id:13,name:"SHA256SUMS"}]} - | if $scenario == "published" then .draft = false + | if $scenario == "published" or $scenario == "published-missing-tag" then .draft = false elif $scenario == "metadata-id" then .id = 124 elif $scenario == "metadata-tag" then .tag_name = "v0.4.2" elif $scenario == "metadata-commit" then .target_commitish = "main" @@ -199,6 +212,7 @@ run_verifier() { passed=0 for SCENARIO in success published build-config-source multiple-attestations array-attestations \ + annotated-tag missing-tag wrong-tag wrong-annotated-tag published-missing-tag tag-api-failure \ checksum traversal absolute extra-checksum duplicate-checksum missing-checksum empty-checksum \ bucket-release.yaml bucket-release.notags.yaml bucket-SHA256SUMS \ malformed-metadata metadata-id metadata-tag metadata-commit metadata-draft metadata-assets \ @@ -213,7 +227,7 @@ for SCENARIO in success published build-config-source multiple-attestations arra status=0 run_verifier > "$work/log" 2>&1 || status=$? case "$SCENARIO" in - success|published|build-config-source|multiple-attestations|array-attestations) + success|published|build-config-source|multiple-attestations|array-attestations|annotated-tag|missing-tag) if [[ $status != 0 ]]; then cat "$work/log" >&2; exit 1; fi for file in release.yaml release.notags.yaml SHA256SUMS; do cmp "$FIXTURES/$file" "$work/output/$file" diff --git a/hack/verify-release-tag.sh b/hack/verify-release-tag.sh new file mode 100644 index 000000000..1aa05b686 --- /dev/null +++ b/hack/verify-release-tag.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release tag verification failed: $*" >&2; exit 1; } +[[ $# == 3 ]] || fail "usage: $0 VERSION COMMIT ALLOW_MISSING" +version=$1 +commit=$2 +allow_missing=$3 +repository=${GITHUB_REPOSITORY:-tektoncd/tekton-kueue} +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ && $commit =~ ^[0-9a-f]{40}$ && + $repository =~ ^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$ && + ( $allow_missing == true || $allow_missing == false ) ]] || fail "invalid candidate" +object=$(gh api --paginate --slurp "repos/$repository/git/matching-refs/tags/$version" | + jq -c --arg ref "refs/tags/$version" ' + [.[][] | select(.ref == $ref)] + | if length <= 1 then .[0].object // null else error("ambiguous tag") end') +if [[ $object == null ]]; then + [[ $allow_missing == true ]] || fail "published release tag is missing" + exit 0 +fi +while :; do + sha=$(jq -r .sha <<< "$object") + [[ $sha =~ ^[0-9a-f]{40}$ ]] || fail "invalid tag object" + case $(jq -r .type <<< "$object") in + commit) + [[ $sha == "$commit" ]] || fail "tag points to another source" + exit 0 + ;; + tag) + object=$(gh api "repos/$repository/git/tags/$sha" | jq -ce '.object') + ;; + *) fail "tag does not resolve to a commit" ;; + esac +done From 046f5504f61a5ba8ec952afe5391804b9ddc802f Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 22:05:23 +0530 Subject: [PATCH 11/13] fix: keep generated release notes repairable Append the attestation boundary so a finalized recovery draft can be repaired repeatedly without losing its generated notes. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- hack/finalize-release.sh | 2 +- hack/finalize-release_test.sh | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hack/finalize-release.sh b/hack/finalize-release.sh index d3a7c5531..f525c4956 100644 --- a/hack/finalize-release.sh +++ b/hack/finalize-release.sh @@ -131,7 +131,7 @@ jq --rawfile section "$work/attestation.md" --arg version "$version" --arg commi | ($body | index("\n" elif $end == null or $end <= $start then error("unsupported release-note structure") else $body[0:$start] + $section + $body[$end:] end)} diff --git a/hack/finalize-release_test.sh b/hack/finalize-release_test.sh index aef768ed2..dc0ce7cd6 100644 --- a/hack/finalize-release_test.sh +++ b/hack/finalize-release_test.sh @@ -276,6 +276,15 @@ EOF and ($e|index("smoke")) < ($e|index("patch")) and all(.[]; .event != "image-copy" and .event != "bucket-put" and .event != "login")' fi + if [[ $MODE == generated-notes ]]; then + # Recovery-generated notes must remain repairable after publication. + cp "$CASE_DIR/release.json" "$CASE_DIR/original.json" + mv "$CASE_DIR/work" "$CASE_DIR/first-work" + bash "$tmp/checkout/hack/finalize-release.sh" repair "$VERSION" "$COMMIT" 123 \ + "$DIGEST" "$CASE_DIR/key.pub" false "$CASE_DIR/work" >> "$CASE_DIR/output" 2>&1 + jq -e -s '.[0].body == .[1].body and .[0].draft == false' \ + "$CASE_DIR/release.json" "$CASE_DIR/original.json" >/dev/null + fi ;; *) [[ $status != 0 ]] || { echo "unexpected success: $MODE" >&2; exit 1; } From 5d37cc297da3a1c27a9aeb4cf750cc06ab94e6ff Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 22:07:13 +0530 Subject: [PATCH 12/13] fix: recheck release tags at finalization Reuse the verifier tag check before promotion and after publishing, and reject a tag changed during candidate checks. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- hack/finalize-release.sh | 3 +++ hack/finalize-release_test.sh | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/hack/finalize-release.sh b/hack/finalize-release.sh index f525c4956..af8cf6a58 100644 --- a/hack/finalize-release.sh +++ b/hack/finalize-release.sh @@ -138,6 +138,8 @@ jq --rawfile section "$work/attestation.md" --arg version "$version" --arg commi | .make_latest = $latest ' "$work/current-release.json" > "$work/patch.json" +bash "$root/hack/verify-release-tag.sh" "$version" "$commit" "$(jq -r .draft "$work/current-release.json")" + if [[ $latest == true ]]; then # Keep credentials out of the caller's Docker configuration and process args. export DOCKER_CONFIG="$work/docker" @@ -181,6 +183,7 @@ jq -e --arg version "$version" --arg commit "$commit" --arg operation "$operatio .body == $patch[0].body and .draft == (if $operation == "repair" then $previous[0].draft else false end) ' "$work/result.json" >/dev/null || fail "GitHub did not retain the requested release state" +bash "$root/hack/verify-release-tag.sh" "$version" "$commit" "$(jq -r .draft "$work/result.json")" if [[ $publish_latest == true ]]; then [[ $(gh api "repos/$repository/releases/latest" --jq .tag_name) == "$version" ]] || fail "GitHub latest release does not match the finalized candidate" diff --git a/hack/finalize-release_test.sh b/hack/finalize-release_test.sh index dc0ce7cd6..4d2c17a97 100644 --- a/hack/finalize-release_test.sh +++ b/hack/finalize-release_test.sh @@ -42,6 +42,13 @@ if [[ ${FINALIZER_TEST_CHILD:-} == true ]]; then [[ $# == 3 && $2 == --jq && $3 == .login ]] event identity "$@" if [[ $MODE == wrong-actor ]]; then echo waveywaves; else echo tekton-robot; fi + elif [[ $1 == --paginate ]]; then + [[ $# == 3 && $2 == --slurp && + $3 == "repos/tektoncd/tekton-kueue/git/matching-refs/tags/$VERSION" ]] + event tag + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg mode "$MODE" ' + [[{ref:("refs/tags/" + $version),object:{type:"commit", + sha:(if $mode == "changed-tag" then "f"*40 else $commit end)}}]]' elif [[ $1 == --method ]]; then [[ $# == 5 && $2 == PATCH && $3 == repos/tektoncd/tekton-kueue/releases/123 && $4 == --input ]] @@ -164,6 +171,7 @@ printf 'caller configuration\n' > "$tmp/caller-docker/config.json" cp "$tmp/caller-docker/config.json" "$tmp/caller-config-original" export DOCKER_CONFIG="$tmp/caller-docker" cp "$root/hack/finalize-release.sh" "$tmp/checkout/hack/" +cp "$root/hack/verify-release-tag.sh" "$tmp/checkout/hack/" for name in verify-release-artifacts.sh smoke-release.sh; do cp "${BASH_SOURCE[0]}" "$tmp/checkout/hack/$name" done @@ -345,6 +353,7 @@ run_case verifier-failure run_case smoke-failure run_case wrong-source run_case changed-assets +run_case changed-tag run_case rekor-mismatch run_case older-latest run_case image-failure From 12231e2e6081206f6580c5620071f8d72119c18b Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 22:10:03 +0530 Subject: [PATCH 13/13] refactor: share release tag resolution with recovery Use the same checked tag lookup for recovery, artifact verification and finalization instead of retaining a separate implementation. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- hack/recover-release.sh | 20 +------------------- hack/recover-release_test.sh | 1 + 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/hack/recover-release.sh b/hack/recover-release.sh index a86f6840c..951966c35 100644 --- a/hack/recover-release.sh +++ b/hack/recover-release.sh @@ -41,25 +41,7 @@ jq -e --arg digest "$digest" '.digest == $digest' "$work/verified/release-info.j fail "verified digest differs from approved candidate" check_tag() { - local object sha - gh api --paginate --slurp "repos/$repository/git/matching-refs/tags/$version" > "$work/refs.json" - jq -e --arg ref "refs/tags/$version" ' - [.[][] | select(.ref == $ref)] | length <= 1 - ' "$work/refs.json" >/dev/null || fail "ambiguous tag" - object=$(jq -c --arg ref "refs/tags/$version" ' - [.[][] | select(.ref == $ref)][0].object // null' "$work/refs.json") - while [[ $object != null ]]; do - sha=$(jq -r .sha <<< "$object") - [[ $sha =~ ^[0-9a-f]{40}$ ]] || fail "invalid tag object" - case $(jq -r .type <<< "$object") in - commit) [[ $sha == "$commit" ]] || fail "tag points to another source"; return ;; - tag) - gh api "repos/$repository/git/tags/$sha" > "$work/tag.json" - object=$(jq -ce '.object' "$work/tag.json") - ;; - *) fail "tag does not resolve to a commit" ;; - esac - done + bash "$root/hack/verify-release-tag.sh" "$version" "$commit" true } find_release() { diff --git a/hack/recover-release_test.sh b/hack/recover-release_test.sh index 28f784cf9..26cbe4f37 100644 --- a/hack/recover-release_test.sh +++ b/hack/recover-release_test.sh @@ -122,6 +122,7 @@ tmp=$(mktemp -d) trap 'status=$?; if [[ $status == 0 ]]; then rm -rf "$tmp"; else echo "test artifacts: $tmp" >&2; fi' EXIT mkdir -p "$tmp/checkout/hack" "$tmp/bin" cp "$root/hack/recover-release.sh" "$tmp/checkout/hack/" +cp "$root/hack/verify-release-tag.sh" "$tmp/checkout/hack/" cp "${BASH_SOURCE[0]}" "$tmp/checkout/hack/verify-release-artifacts.sh" cp "${BASH_SOURCE[0]}" "$tmp/bin/gh" chmod +x "$tmp/bin/gh"