From 709bc96d3b2a577c948607477531b04557a4ba39 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 16:25:01 +0530 Subject: [PATCH 1/7] fix: select DSSE provenance for release drafts Recognize DSSE provenance entries and fail when no provenance can be found instead of substituting an image-signature entry. Pin the shared release task with corrected attestation instructions. Depends on tektoncd/plumbing#3807. Assisted-by: OmO Signed-off-by: Vibhav Bobade --- tekton/release-pipeline.yaml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index 9518f7034..b146f225d 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -365,7 +365,7 @@ spec: for uuid in ${CANDIDATES}; do KIND=$(curl -fsS "${REKOR_HOST}/api/v1/log/entries/${uuid}" \ | jq -r '.[].body' | base64 -d 2>/dev/null | jq -r '.kind' 2>/dev/null || true) - if [ "${KIND}" = "intoto" ]; then + if [ "${KIND}" = "intoto" ] || [ "${KIND}" = "dsse" ]; then REKOR_UUID="${uuid}" break fi @@ -373,19 +373,8 @@ spec: fi if [ -z "${REKOR_UUID}" ]; then - echo "WARNING: Could not find an in-toto attestation entry; falling back to the transparency annotation" - TRANSPARENCY=$(kubectl get taskrun "${TASKRUN_NAME}" -n "${NAMESPACE}" \ - -o jsonpath='{.metadata.annotations.chains\.tekton\.dev/transparency}' 2>/dev/null || echo "") - if [ -n "${TRANSPARENCY}" ]; then - REKOR_UUID=$(curl -fsS "${TRANSPARENCY}" | jq -r 'keys[0] // empty' || true) - if [ -z "${REKOR_UUID}" ]; then - REKOR_UUID=$(echo "${TRANSPARENCY}" | grep -oE 'logIndex=[0-9]+' | cut -d= -f2) - fi - fi - fi - - if [ -z "${REKOR_UUID}" ]; then - REKOR_UUID="unknown" + echo "ERROR: Could not find an intoto or dsse attestation entry for ${IMAGE_DIGEST}" + exit 1 fi echo "Rekor UUID: ${REKOR_UUID}" printf '%s' "${REKOR_UUID}" > "$(results.rekor-uuid.path)" @@ -491,7 +480,7 @@ spec: - name: url value: https://github.com/tektoncd/plumbing - name: revision - value: 894710e292fcb6b17276af01a2bc318107eef8a7 + value: df071e78c5b1a58867435efc7a9b295688d58969 - name: pathInRepo value: tekton/resources/release/base/github_release_oci.yaml params: From 438582186a1346d6c5a674fb2180c43d848a615c Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 17:51:16 +0530 Subject: [PATCH 2/7] feat: verify existing release artifacts against a trusted signing key Check metadata, bucket copies, image platforms and signed provenance before retaining immutable verification receipts. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- .github/workflows/release-verification.yaml | 23 ++ hack/verify-release-artifacts.sh | 179 +++++++++++++ hack/verify-release-artifacts_test.sh | 264 ++++++++++++++++++++ 3 files changed, 466 insertions(+) create mode 100644 .github/workflows/release-verification.yaml create mode 100644 hack/verify-release-artifacts.sh create mode 100644 hack/verify-release-artifacts_test.sh diff --git a/.github/workflows/release-verification.yaml b/.github/workflows/release-verification.yaml new file mode 100644 index 000000000..0d7b03d54 --- /dev/null +++ b/.github/workflows/release-verification.yaml @@ -0,0 +1,23 @@ +name: Release artifact verification + +on: + pull_request: + paths: + - hack/verify-release-artifacts.sh + - hack/verify-release-artifacts_test.sh + - .github/workflows/release-verification.yaml + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check verifier syntax and behavior + run: | + bash -n hack/verify-release-artifacts.sh hack/verify-release-artifacts_test.sh + bash hack/verify-release-artifacts_test.sh diff --git a/hack/verify-release-artifacts.sh b/hack/verify-release-artifacts.sh new file mode 100644 index 000000000..90848df82 --- /dev/null +++ b/hack/verify-release-artifacts.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fail() { + echo "release verification failed: $*" >&2 + exit 1 +} + +[[ $# == 5 ]] || fail "usage: $0 VERSION EXPECTED_COMMIT RELEASE_ID PUBLIC_KEY_FILE OUTPUT_DIR" +version=$1 +commit=$2 +release_id=$3 +key=$4 +output=$5 +repository=${GITHUB_REPOSITORY:-tektoncd/tekton-kueue} +image_repository=ghcr.io/tektoncd/tekton-kueue +source_url=https://github.com/tektoncd/tekton-kueue + +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "invalid version" +[[ $commit =~ ^[0-9a-f]{40}$ ]] || fail "invalid commit" +[[ $release_id =~ ^[1-9][0-9]*$ ]] || fail "invalid release ID" +[[ $repository =~ ^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$ && + ${repository##*/} != . && ${repository##*/} != .. ]] || fail "invalid GitHub repository" +[[ -n ${GH_TOKEN:-} ]] || fail "GH_TOKEN is required" +[[ -f $key && -s $key && -r $key ]] || fail "a nonempty configured public key file is required" +[[ -n $output && ! -L $output ]] || fail "invalid output directory" +if [[ -e $output ]]; then + [[ -d $output ]] || fail "output must be a directory" + [[ -z $(find "$output" -mindepth 1 -print -quit) ]] || fail "output directory must be empty" +fi +for command in gh jq curl crane cosign openssl awk cmp; do + command -v "$command" >/dev/null || fail "missing command: $command" +done +openssl pkey -pubin -in "$key" -noout >/dev/null 2>&1 || fail "invalid configured public key" +if command -v sha256sum >/dev/null; then + checksum=(sha256sum) +else + command -v shasum >/dev/null || fail "sha256sum or shasum is required" + checksum=(shasum -a 256) +fi + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +gh api --method GET "repos/$repository/releases/$release_id" > "$work/release.json" +jq -e --arg version "$version" --arg commit "$commit" --arg id "$release_id" ' + .id == ($id | tonumber) and .tag_name == $version and .target_commitish == $commit + and (.draft | type == "boolean") and (.assets | type == "array") + and ([.assets[] | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS")] + | length == 3 and (map(.name) | unique | length == 3) + and all(.[]; (.id | type == "number") and .id > 0 and .id == (.id | floor)) + and (map(.id) | unique | length == 3)) +' "$work/release.json" >/dev/null || fail "release metadata does not match candidate" + +for file in release.yaml release.notags.yaml SHA256SUMS; do + asset_id=$(jq -r --arg name "$file" '.assets[] | select(.name == $name) | .id' "$work/release.json") + gh api --method GET -H "Accept: application/octet-stream" \ + "repos/$repository/releases/assets/$asset_id" > "$work/$file" + [[ -s $work/$file ]] || fail "empty asset: $file" +done + +# Parse before hashing: never pass downloaded filenames to a checksum tool. +declare -A seen=() +while IFS= read -r line || [[ -n $line ]]; do + [[ $line =~ ^([0-9a-f]{64})[[:space:]][\ \*](release\.yaml|release\.notags\.yaml)$ ]] || + fail "unsafe or malformed SHA256SUMS" + hash=${BASH_REMATCH[1]} + file=${BASH_REMATCH[2]} + [[ -z ${seen[$file]:-} ]] || fail "duplicate checksum filename" + seen[$file]=1 + actual=$("${checksum[@]}" "$work/$file") + [[ ${actual%% *} == "$hash" ]] || fail "checksum mismatch: $file" +done < "$work/SHA256SUMS" +[[ ${#seen[@]} == 2 ]] || fail "SHA256SUMS must contain exactly the two manifests" +for file in release.yaml release.notags.yaml SHA256SUMS; do + curl --fail --silent --show-error --location \ + "https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$version/$file" \ + --output "$work/bucket-$file" + cmp -s "$work/$file" "$work/bucket-$file" || fail "bucket mismatch: $file" +done +echo "PASS release metadata, checksums and bucket copies" + +# Kustomize emits block YAML with top-level metadata and scalar image fields. +# Require the controller and webhook deployments, not just an image somewhere. +for file in release.yaml release.notags.yaml; do + awk ' + /^---/ { kind = ""; name = "" } + /^kind: / { kind = $2 } + /^metadata:/ { metadata = 1; next } + /^[^ ]/ { metadata = 0 } + metadata && /^ name: / { name = $2 } + $1 == "image:" { print kind "\t" name "\t" $2 } + ' "$work/$file" > "$work/images" + count=0 + controller=0 + webhook=0 + while IFS=$'\t' read -r kind name reference; do + [[ $kind == Deployment ]] || fail "unexpected image outside a Deployment" + case "$name" in + tekton-kueue-controller-manager) controller=$((controller + 1)) ;; + tekton-kueue-webhook) webhook=$((webhook + 1)) ;; + *) fail "unexpected image deployment" ;; + esac + # Both variants must have the same digest; only release.yaml may have the version tag. + prefix=$image_repository + if [[ $file == release.yaml ]]; then + prefix="$prefix:$version" + fi + [[ $reference == "$prefix"@sha256:* ]] || fail "unexpected image repository or tag" + candidate=${reference##*@} + [[ $candidate =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid image digest" + [[ ${digest:-$candidate} == "$candidate" ]] || fail "inconsistent manifest image digests" + digest=$candidate + count=$((count + 1)) + done < "$work/images" + [[ $count == 2 && $controller == 1 && $webhook == 1 ]] || + fail "both controller and webhook images are required" +done +image="$image_repository@$digest" + +crane manifest "$image" > "$work/index.json" +jq -e ' + [.manifests[].platform | .os + "/" + .architecture] | sort == + ["linux/amd64", "linux/arm64", "linux/ppc64le", "linux/s390x"] +' "$work/index.json" >/dev/null || fail "unexpected image platforms" +for platform in linux/amd64 linux/arm64 linux/ppc64le linux/s390x; do + crane config --platform "$platform" "$image" > "$work/config.json" + jq -e --arg source "$source_url" --arg platform "$platform" ' + (.os + "/" + .architecture) == $platform + and .config.Labels["org.opencontainers.image.source"] == $source + and .config.Labels["org.opencontainers.image.url"] == $source + ' "$work/config.json" >/dev/null || fail "incorrect OCI labels or platform: $platform" +done +echo "PASS manifest images, four platforms and OCI labels" + +cosign verify --key "$key" "$image" > "$work/signatures.json" +jq -e -s --arg digest "$digest" ' + [ .[] | if type == "array" then .[] else . end + | select(.critical.image["docker-manifest-digest"] == $digest) ] | length > 0 +' \ + "$work/signatures.json" >/dev/null || fail "no verified image signature" +cosign verify-attestation --key "$key" --type https://slsa.dev/provenance/v1 \ + "$image" > "$work/attestations.json" +# cosign emits verified DSSE envelopes as JSON objects (or an array). +jq -e -s --arg digest "${digest#sha256:}" --arg commit "$commit" ' + def repository: + # Chains uses git+https URIs with an optional ref suffix. + test("^(git\\+)?https://github\\.com/tektoncd/tekton-kueue(\\.git)?(@[^[:space:]]+)?$"); + def source: + (.uri // "" | repository) + and ((.digest.sha1 // .digest.gitCommit) == $commit); + [ .[] | if type == "array" then .[] else . end + | .payload as $payload | $payload | @base64d | fromjson + | select(._type == "https://in-toto.io/Statement/v1" + and .predicateType == "https://slsa.dev/provenance/v1") + | select(any(.subject[]; .digest.sha256 == $digest)) + | select( + any(.predicate.buildDefinition.resolvedDependencies[]?; source) + or (.predicate.buildDefinition.externalParameters.buildConfigSource + | (.repository // "" | repository) and .ref == ("sha1:" + $commit))) + | $payload + ] | if length > 0 then . else error("no matching verified provenance") end +' "$work/attestations.json" > "$work/verified-payloads.json" || + fail "no verified provenance matching source and digest" +echo "PASS configured-key signature and SLSA provenance" + +jq -n --argjson release_id "$release_id" --arg version "$version" --arg commit "$commit" \ + --arg image "$image" --arg digest "$digest" --slurpfile release "$work/release.json" \ + '{release_id: $release_id, version: $version, commit: $commit, image: $image, + digest: $digest, draft: $release[0].draft, + assets: [$release[0].assets[] + | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS") + | {name, id}] | sort_by(.name)}' > "$work/release-info.json" +mkdir -p "$output" +for file in release.yaml release.notags.yaml SHA256SUMS release-info.json verified-payloads.json; do + # Refuse to overwrite even if another process populated the output during verification. + (set -o noclobber; cat "$work/$file" > "$output/$file") +done +echo "PASS $version artifacts saved to $output" diff --git a/hack/verify-release-artifacts_test.sh b/hack/verify-release-artifacts_test.sh new file mode 100644 index 000000000..f3e038824 --- /dev/null +++ b/hack/verify-release-artifacts_test.sh @@ -0,0 +1,264 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +export FIXTURES="$work/fixtures" +mkdir -p "$work/bin" "$FIXTURES" +export PATH="$work/bin:$PATH" +export GH_TOKEN=test-token +unset GITHUB_REPOSITORY +export VERSION=v0.4.3 +export COMMIT=0123456789abcdef0123456789abcdef01234567 +export DIGEST=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export IMAGE="ghcr.io/tektoncd/tekton-kueue@sha256:$DIGEST" + +# A fixed Ed25519 public key; openssl (not a fake) validates this boundary. +cat > "$work/key.pub" <<'EOF' +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo= +-----END PUBLIC KEY----- +EOF +write_artifacts() { +for variant in release.yaml release.notags.yaml; do + reference=$IMAGE + [[ $variant != release.yaml ]] || reference="ghcr.io/tektoncd/tekton-kueue:$VERSION@sha256:$DIGEST" + for deployment in controller-manager webhook; do + name=$deployment + candidate=$reference + case "${SCENARIO:-success}" in + wrong-image-repository) candidate=${candidate/tektoncd\/tekton-kueue/other\/tekton-kueue} ;; + inconsistent-image) [[ $deployment != webhook ]] || candidate=${candidate/$DIGEST/${DIGEST//a/b}} ;; + missing-webhook) [[ $deployment != webhook ]] || continue ;; + wrong-deployment) [[ $deployment != webhook ]] || name=other ;; + tag-in-notags) [[ $variant != release.notags.yaml ]] || candidate="ghcr.io/tektoncd/tekton-kueue:$VERSION@sha256:$DIGEST" ;; + esac + cat < "$FIXTURES/$variant" +done +( + cd "$FIXTURES" + if command -v sha256sum >/dev/null; then + sha256sum release.yaml release.notags.yaml + else + shasum -a 256 release.yaml release.notags.yaml + fi +) > "$FIXTURES/SHA256SUMS" +} + +# Only network/registry/signing executables are replaced. JSON parsing, hashing, +# key parsing, manifest extraction and byte comparisons use the real commands. +cat > "$work/bin/fake" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "${0##*/}" in + gh) + [[ $1 == api && $2 == --method && $3 == GET ]] || exit 90 + if [[ $4 == repos/tektoncd/tekton-kueue/releases/123 ]]; then + if [[ $SCENARIO == malformed-metadata ]]; then printf '{\n'; exit; fi + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {id:123,tag_name:$version,target_commitish:$commit,draft:true,assets:[ + {id:11,name:"release.yaml"},{id:12,name:"release.notags.yaml"},{id:13,name:"SHA256SUMS"}]} + | if $scenario == "published" then .draft = false + elif $scenario == "metadata-id" then .id = 124 + elif $scenario == "metadata-tag" then .tag_name = "v0.4.2" + elif $scenario == "metadata-commit" then .target_commitish = "main" + elif $scenario == "metadata-draft" then .draft = "true" + elif $scenario == "metadata-assets" then .assets = null + elif $scenario == "duplicate-asset" then .assets += [.assets[0]] + elif $scenario == "missing-asset" then .assets |= .[0:2] + elif $scenario == "unsafe-asset-id" then .assets[0].id = "../evil" + elif $scenario == "duplicate-asset-id" then .assets[1].id = 11 + else . end' + else + [[ $4 == -H && $5 == "Accept: application/octet-stream" ]] || exit 90 + case "$6" in + repos/tektoncd/tekton-kueue/releases/assets/11) file=release.yaml ;; + repos/tektoncd/tekton-kueue/releases/assets/12) file=release.notags.yaml ;; + repos/tektoncd/tekton-kueue/releases/assets/13) file=SHA256SUMS ;; + *) exit 90 ;; + esac + if [[ $file == SHA256SUMS ]]; then + case "$SCENARIO" in + checksum) printf '%064d release.yaml\n' 0; exit ;; + traversal) printf '%s ../release.yaml\n' "$DIGEST"; exit ;; + absolute) printf '%s /tmp/release.yaml\n' "$DIGEST"; exit ;; + extra-checksum) cat "$FIXTURES/$file"; printf '%s extra.yaml\n' "$DIGEST"; exit ;; + duplicate-checksum) cat "$FIXTURES/$file" "$FIXTURES/$file"; exit ;; + missing-checksum) awk 'NR == 1' "$FIXTURES/$file"; exit ;; + empty-checksum) exit ;; + esac + fi + cat "$FIXTURES/$file" + fi + ;; + curl) + [[ $# == 7 && $1 == --fail && $2 == --silent && $3 == --show-error && + $4 == --location && $6 == --output ]] || exit 90 + prefix="https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$VERSION/" + [[ $5 == "$prefix"* ]] || exit 90 + file=${5#"$prefix"} + case "$file" in release.yaml|release.notags.yaml|SHA256SUMS) ;; *) exit 90 ;; esac + cat "$FIXTURES/$file" > "$7" + [[ $SCENARIO != "bucket-$file" ]] || printf '\nchanged\n' >> "$7" + ;; + crane) + if [[ $1 == manifest && $2 == "$IMAGE" && $# == 2 ]]; then + jq -n --arg scenario "$SCENARIO" ' + ["amd64","arm64","ppc64le","s390x"] + | if $scenario == "missing-platform" then .[0:3] + elif $scenario == "duplicate-platform" then . + ["amd64"] + elif $scenario == "wrong-platform" then .[3] = "riscv64" + else . end + | {schemaVersion:2, manifests:map({digest:("sha256:" + ("b" * 64)), + platform:{os:"linux",architecture:.}})}' + elif [[ $1 == config && $2 == --platform && $4 == "$IMAGE" && $# == 4 ]]; then + jq -n --arg platform "$3" --arg scenario "$SCENARIO" ' + {os:"linux",architecture:($platform | split("/")[1]),config:{Labels:{ + "org.opencontainers.image.source":"https://github.com/tektoncd/tekton-kueue", + "org.opencontainers.image.url":"https://github.com/tektoncd/tekton-kueue"}}} + | if $platform == "linux/s390x" then + if $scenario == "bad-source-label" then .config.Labels["org.opencontainers.image.source"] = "https://example.com" + elif $scenario == "bad-url-label" then del(.config.Labels["org.opencontainers.image.url"]) + elif $scenario == "bad-config-platform" then .os = "windows" + else . end + else . end' + else exit 90; fi + ;; + cosign) + [[ $2 == --key && -s $3 ]] || exit 90 + case "$1" in + verify) + [[ $# == 4 && $4 == "$IMAGE" ]] || exit 90 + [[ $SCENARIO != signature-failure ]] || exit 1 + [[ $SCENARIO != empty-signature ]] || exit + printf '[{"critical":{"image":{"docker-manifest-digest":"sha256:%s"}}}]\n' "$DIGEST" + ;; + verify-attestation) + [[ $# == 6 && $4 == --type && $5 == https://slsa.dev/provenance/v1 && $6 == "$IMAGE" ]] || exit 90 + [[ $SCENARIO != attestation-failure ]] || exit 1 + [[ $SCENARIO != empty-attestation ]] || exit + if [[ $SCENARIO == malformed-attestation ]]; then + printf '{"payload":"invalid base64"}\n' + exit + fi + jq -n --arg digest "$DIGEST" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {_type:"https://in-toto.io/Statement/v1",predicateType:"https://slsa.dev/provenance/v1", + subject:[{name:"ghcr.io/tektoncd/tekton-kueue",digest:{sha256:$digest}}], + predicate:{buildDefinition:{resolvedDependencies:[ + {uri:"git+https://github.com/tektoncd/tekton-kueue.git@refs/tags/v0.4.3",digest:{sha1:$commit}}]}}} + | if $scenario == "bad-source" then .predicate.buildDefinition.resolvedDependencies[0].uri = "https://github.com/tektoncd/tekton-kueue-evil" + elif $scenario == "bad-commit" then .predicate.buildDefinition.resolvedDependencies[0].digest.sha1 = ("f" * 40) + elif $scenario == "bad-digest" then .subject[0].digest.sha256 = ("b" * 64) + elif $scenario == "bad-predicate" then .predicateType = "https://slsa.dev/provenance/v0.2" + elif $scenario == "build-config-source" then + .predicate.buildDefinition.externalParameters.buildConfigSource = + {repository:"git+https://github.com/tektoncd/tekton-kueue",ref:("sha1:" + $commit),path:"tekton/release-pipeline.yaml"} + | del(.predicate.buildDefinition.resolvedDependencies) + else . end + | if $scenario == "multiple-attestations" then + (.subject[0].digest.sha256 = ("b" * 64)), . + else . end + | {payloadType:"application/vnd.in-toto+json",payload:(tojson | @base64),signatures:[{sig:"verified-by-fake-cosign"}]} + | if $scenario == "array-attestations" then [.] else . end' + ;; + *) exit 90 ;; + esac + ;; + *) exit 90 ;; +esac +EOF +chmod +x "$work/bin/fake" +for executable in gh curl crane cosign; do + ln -s fake "$work/bin/$executable" +done + +run_verifier() { + bash "$root/hack/verify-release-artifacts.sh" \ + "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +} + +passed=0 +for SCENARIO in success published build-config-source multiple-attestations array-attestations \ + checksum traversal absolute extra-checksum duplicate-checksum missing-checksum empty-checksum \ + bucket-release.yaml bucket-release.notags.yaml bucket-SHA256SUMS \ + malformed-metadata metadata-id metadata-tag metadata-commit metadata-draft metadata-assets \ + duplicate-asset missing-asset unsafe-asset-id duplicate-asset-id \ + wrong-image-repository inconsistent-image missing-webhook wrong-deployment tag-in-notags \ + missing-platform duplicate-platform wrong-platform bad-source-label bad-url-label bad-config-platform \ + signature-failure empty-signature attestation-failure empty-attestation malformed-attestation \ + bad-source bad-commit bad-digest bad-predicate; do + export SCENARIO + rm -rf "$work/output" + write_artifacts + status=0 + run_verifier > "$work/log" 2>&1 || status=$? + case "$SCENARIO" in + success|published|build-config-source|multiple-attestations|array-attestations) + if [[ $status != 0 ]]; then cat "$work/log" >&2; exit 1; fi + for file in release.yaml release.notags.yaml SHA256SUMS; do + cmp "$FIXTURES/$file" "$work/output/$file" + done + jq -e --arg commit "$COMMIT" --arg image "$IMAGE" --arg scenario "$SCENARIO" \ + --arg version "$VERSION" --arg digest "sha256:$DIGEST" ' + . == {release_id:123,version:$version,commit:$commit,image:$image,digest:$digest, + draft:($scenario != "published"),assets:[ + {name:"SHA256SUMS",id:13},{name:"release.notags.yaml",id:12}, + {name:"release.yaml",id:11}]}' "$work/output/release-info.json" >/dev/null + jq -e --arg digest "$DIGEST" ' + [.[] | @base64d | fromjson + | select(any(.subject[]; .digest.sha256 == $digest))] + | length > 0' "$work/output/verified-payloads.json" >/dev/null + ;; + *) + [[ $status != 0 && ! -e $work/output/release-info.json ]] || { + echo "unexpected verification success: $SCENARIO" >&2; exit 1; + } + ;; + esac + passed=$((passed + 1)) +done + +export SCENARIO=success +expect_failure() { + if "$@" > "$work/log" 2>&1; then + echo "unexpected boundary acceptance" >&2 + exit 1 + fi + passed=$((passed + 1)) +} +verifier="$root/hack/verify-release-artifacts.sh" +rm -rf "$work/output" +expect_failure bash "$verifier" +expect_failure bash "$verifier" ../v0.4.3 "$COMMIT" 123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "${COMMIT^^}" 123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" ../123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/missing.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "" +expect_failure env GH_TOKEN= bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +expect_failure env GITHUB_REPOSITORY=../repo bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +touch "$work/empty.pub" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/empty.pub" "$work/output" +printf 'not a public key\n' > "$work/invalid.pub" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/invalid.pub" "$work/output" +mkdir "$work/output" +printf 'existing artifact\n' > "$work/output/release.yaml" +expect_failure run_verifier +[[ $(< "$work/output/release.yaml") == "existing artifact" ]] +echo "PASS $passed release artifact verification cases" From 117c55801c7d48b524f69acf3b0ed339ebb96f51 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:10:58 +0530 Subject: [PATCH 3/7] fix: keep release staging from promoting latest artifacts Remove latest inputs and publication paths; reject legacy promotion before side effects and verify the shipped guard during release configuration checks. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- .github/workflows/patch-release.yaml | 28 +------------------ .tekton/release-patch.yaml | 2 -- .tekton/release.yaml | 2 -- hack/trigger-patch-release.sh | 9 +++---- hack/verify-release-config.sh | 21 +++++++++++++++ tekton/publish.yaml | 23 +++++----------- tekton/release-cheat-sheet.md | 9 +++++-- tekton/release-pipeline.yaml | 40 +--------------------------- 8 files changed, 40 insertions(+), 94 deletions(-) diff --git a/.github/workflows/patch-release.yaml b/.github/workflows/patch-release.yaml index 2f7ae9923..561ed2de9 100644 --- a/.github/workflows/patch-release.yaml +++ b/.github/workflows/patch-release.yaml @@ -11,11 +11,6 @@ name: Patch Release description: "Version to release (for example, v0.5.1)" required: true type: string - release_as_latest: - description: "Publish as latest release" - required: false - type: boolean - default: true schedule: - cron: "0 10 * * 4" @@ -47,23 +42,6 @@ jobs: id: scan shell: bash run: | - latest_branch="" - latest_major=0 - latest_minor=0 - for ref in $(git branch -r --list 'origin/release-v*'); do - branch="${ref#origin/}" - if [[ "$branch" =~ ^release-v([0-9]+)\.([0-9]+)\.x$ ]]; then - major="${BASH_REMATCH[1]}" - minor="${BASH_REMATCH[2]}" - if [ "$major" -gt "$latest_major" ] || { [ "$major" -eq "$latest_major" ] && [ "$minor" -gt "$latest_minor" ]; }; then - latest_major=$major - latest_minor=$minor - latest_branch=$branch - fi - fi - done - echo "::notice::Latest release branch: ${latest_branch}" - min_major="${MIN_RELEASE_VERSION%%.*}" min_minor="${MIN_RELEASE_VERSION##*.}" releases=() @@ -93,9 +71,7 @@ jobs: fi next_version=$(echo "$last_tag" | awk -F. '{printf "%s.%s.%d", $1, $2, $3+1}') - is_latest=false - [ "$branch" = "$latest_branch" ] && is_latest=true - releases+=("{\"branch\":\"${branch}\",\"version\":\"${next_version}\",\"release_as_latest\":\"${is_latest}\"}") + releases+=("{\"branch\":\"${branch}\",\"version\":\"${next_version}\"}") done if [ ${#releases[@]} -eq 0 ]; then @@ -133,7 +109,6 @@ jobs: PAC_INCOMING_SECRET: ${{ secrets.PAC_INCOMING_SECRET }} RELEASE_BRANCH: ${{ matrix.release.branch }} RELEASE_VERSION: ${{ matrix.release.version }} - RELEASE_AS_LATEST: ${{ matrix.release.release_as_latest }} run: bash ./hack/trigger-patch-release.sh trigger-manual-release: @@ -184,5 +159,4 @@ jobs: PAC_INCOMING_SECRET: ${{ secrets.PAC_INCOMING_SECRET }} RELEASE_BRANCH: ${{ inputs.branch }} RELEASE_VERSION: ${{ inputs.version }} - RELEASE_AS_LATEST: ${{ inputs.release_as_latest }} run: bash ./hack/trigger-patch-release.sh diff --git a/.tekton/release-patch.yaml b/.tekton/release-patch.yaml index e67af9ea1..ccfde1333 100644 --- a/.tekton/release-patch.yaml +++ b/.tekton/release-patch.yaml @@ -43,8 +43,6 @@ spec: value: tekton-robot - name: releaseBucket value: tekton-releases - - name: releaseAsLatest - value: "{{ release_as_latest }}" - name: platforms value: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountImagesPath diff --git a/.tekton/release.yaml b/.tekton/release.yaml index f6337bc36..310c69208 100644 --- a/.tekton/release.yaml +++ b/.tekton/release.yaml @@ -46,8 +46,6 @@ spec: value: tekton-robot - name: releaseBucket value: tekton-releases - - name: releaseAsLatest - value: "true" - name: platforms value: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountImagesPath diff --git a/hack/trigger-patch-release.sh b/hack/trigger-patch-release.sh index 38a22fcfa..563423efd 100644 --- a/hack/trigger-patch-release.sh +++ b/hack/trigger-patch-release.sh @@ -7,7 +7,6 @@ set -euo pipefail : "${PAC_CONTROLLER_URL:?}" : "${PAC_INCOMING_SECRET:?}" : "${PAC_REPOSITORY_NAME:?}" -: "${RELEASE_AS_LATEST:?}" : "${RELEASE_BRANCH:?}" : "${RELEASE_VERSION:?}" @@ -23,8 +22,7 @@ jq -n '{ pipelinerun: "release-patch", secret: env.PAC_INCOMING_SECRET, params: { - version: env.RELEASE_VERSION, - release_as_latest: env.RELEASE_AS_LATEST + version: env.RELEASE_VERSION } }' | curl --fail-with-body --silent --show-error \ -H "Content-Type: application/json" \ @@ -97,7 +95,8 @@ draft_url=$(jq -r '.html_url' <<<"$draft") echo echo "- External PipelineRun: ${details_url}" echo "- Draft release: ${draft_url}" - echo "- Publication: manual review required" + echo "- Finalization: approval, artifact verification, and installation smoke test required" + echo "- Latest image and bucket promotion: disabled during staging" } >> "$GITHUB_STEP_SUMMARY" -echo "::warning title=Draft release awaiting publication::Review and publish ${draft_url}" +echo "::warning title=Draft release awaiting finalization::Review ${draft_url}; publication and latest promotion require separate approval" diff --git a/hack/verify-release-config.sh b/hack/verify-release-config.sh index 9513a9676..0aad70ffa 100644 --- a/hack/verify-release-config.sh +++ b/hack/verify-release-config.sh @@ -25,12 +25,33 @@ test -n "$go_image" grep -Fq "apk add --no-cache bash make" "$root/tekton/publish.yaml" grep -Fq 'org.opencontainers.image.source=https://$PACKAGE' "$root/tekton/publish.yaml" grep -Fq 'org.opencontainers.image.url=https://$PACKAGE' "$root/tekton/publish.yaml" +if grep -Fq '/latest/' "$root/tekton/release-pipeline.yaml" || + grep -Eq 'crane (cp|copy|tag).*latest' "$root/tekton/publish.yaml" || + grep -Fq '$(params.releaseAsLatest)' "$root/tekton/release-pipeline.yaml"; then + echo "release staging must not promote latest" >&2 + exit 1 +fi +for caller in "$root/.tekton/release.yaml" "$root/.tekton/release-patch.yaml"; do + if grep -Fq 'name: releaseAsLatest' "$caller"; then + echo "release entry points must not accept latest promotion" >&2 + exit 1 + fi +done "$container_tool" run --rm --platform "$platform" \ -v "$root:/workspace/source" \ "$publish_image" sh -ec ' apk add --no-cache bash make >/dev/null cd /workspace/source + # Exercise the shipped guard, not a copy of its condition. + awk '\'' + $0 == " - name: publish-image-and-manifests" { step = 1 } + step && $0 == " script: |" { script = 1; next } + script { sub(/^ /, ""); print } + '\'' tekton/publish.yaml > /tmp/publish.sh + status=0 + VERSION_TAG=v0.0.0 RELEASE_AS_LATEST=true sh /tmp/publish.sh || status=$? + test "$status" -eq 64 make release \ KUSTOMIZE=/go/bin/kustomize \ RELEASE_DIR=/tmp/release \ diff --git a/tekton/publish.yaml b/tekton/publish.yaml index bc4413fae..e339d0644 100644 --- a/tekton/publish.yaml +++ b/tekton/publish.yaml @@ -31,7 +31,8 @@ spec: - name: imageRegistryUser default: tekton-robot - name: releaseAsLatest - default: "true" + description: Must be false; latest promotion belongs to release finalization. + default: "false" - name: platforms default: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountPath @@ -90,17 +91,17 @@ spec: #!/usr/bin/env sh set -eu - apk add --no-cache bash make - printf '%s\n' "$VERSION_TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' || { echo "invalid release version: $VERSION_TAG" >&2 exit 1 } case "$RELEASE_AS_LATEST" in - true|false) ;; - *) echo "releaseAsLatest must be true or false" >&2; exit 1 ;; + false) ;; + *) echo "latest promotion requires the release finalization workflow" >&2; exit 64 ;; esac + apk add --no-cache bash make + OUTPUT_RELEASE_DIR="$OUTPUT_ROOT/$VERSION_TAG" git config --global --add safe.directory "$PROJECT_ROOT" mkdir -p "$HOME/.docker" "$OUTPUT_RELEASE_DIR/template" @@ -139,15 +140,3 @@ spec: *) echo "unexpected release image: $image" >&2; exit 1 ;; esac printf '%s,' "$image" > "$(results.IMAGES.path)" - - - name: tag-latest - image: cgr.dev/chainguard/crane:latest-dev@sha256:dd9a345764e47517b80391648020b78c0ae6d4db0e392c7013b71e3817ed544c - script: | - #!/bin/sh - set -eu - - [ "$RELEASE_AS_LATEST" = "true" ] || exit 0 - mkdir -p "$HOME/.docker" - cp /workspace/docker-config.json "$HOME/.docker/config.json" - image=$(cat /workspace/built-images) - crane cp "$image" "${image%@*}:latest" diff --git a/tekton/release-cheat-sheet.md b/tekton/release-cheat-sheet.md index b6f6e8322..da31d8e9d 100644 --- a/tekton/release-cheat-sheet.md +++ b/tekton/release-cheat-sheet.md @@ -9,7 +9,7 @@ The automated steps below become operational only after the release Pipeline tra 1. Confirm the target version, LTS designation, end-of-life date, and compatibility ranges with the maintainers. 2. Confirm all intended changes are merged and their release-note blocks are complete. 3. From a clean checkout of the candidate commit, run generation, lint, unit, and end-to-end checks. -4. Run a release candidate with `releaseAsLatest=false` through the Tekton-controlled release Pipeline so it cannot update the `latest` image tag or artifacts. +4. Stage the release candidate through the Tekton-controlled release Pipeline. It publishes only versioned images and artifacts; it cannot update `latest`. 5. Verify both generated manifests reference only `ghcr.io/tektoncd/tekton-kueue` images by immutable digest. 6. Install and upgrade the candidate on the supported Kubernetes, Tekton Pipelines, and Kueue versions. @@ -23,7 +23,7 @@ The automated steps below become operational only after the release Pipeline tra ## Publish a patch release 1. Cherry-pick an approved fix to the supported release branch. -2. After the cherry-pick passes CI on the release branch, run the **Patch Release** GitHub workflow with the branch, next `vX.Y.Z` version, and whether it should update `latest`. +2. After the cherry-pick passes CI on the release branch, run the **Patch Release** GitHub workflow with the branch and next `vX.Y.Z` version. This stages a draft and does not update `latest`. 3. Monitor and verify the release as for an initial release. The scheduled patch workflow scans `release-v0.3.x` and newer for unreleased commits. Older release lines require manual dispatch. A release operator must still verify every resulting release before announcing it. @@ -41,6 +41,11 @@ Verify and record: Do not announce the release until every promised artifact is downloadable and independently verifiable. +Staging does not authorize publication or promotion. Those actions require +maintainer approval and verification of the finished release. Automated +approval-gated finalization is a separate rollout; staging alone leaves the +GitHub release as a draft and preserves the existing `latest` artifacts. + ## After the release 1. Update [`releases.md`](../releases.md) with the release date, exact end-of-life date, compatibility ranges, and release links. diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index b146f225d..3e0ca03d1 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -27,9 +27,6 @@ spec: - name: releaseBucket description: bucket where the release is stored default: tekton-releases - - name: releaseAsLatest - description: Whether to tag and publish this release as latest - default: "false" - name: platforms description: Linux platforms to publish default: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le @@ -163,7 +160,7 @@ spec: - name: imageRegistryUser value: $(params.imageRegistryUser) - name: releaseAsLatest - value: $(params.releaseAsLatest) + value: "false" - name: serviceAccountPath value: $(params.serviceAccountImagesPath) - name: platforms @@ -208,41 +205,6 @@ spec: - name: recursive value: "true" - - name: publish-to-bucket-latest - runAfter: [publish-images] - when: - - input: "$(params.releaseAsLatest)" - operator: in - values: ["true"] - taskRef: - resolver: bundles - params: - - name: bundle - value: ghcr.io/tektoncd/catalog/upstream/tasks/oracle-cloud-storage-upload@sha256:f1dd28da10dfb72ebbb3a5258e9f1ba6bfe82ee587311d708a7be3207f69e5e6 - - name: name - value: oracle-cloud-storage-upload - - name: kind - value: task - workspaces: - - name: credentials - workspace: release-secret - - name: source - workspace: workarea - subPath: bucket - params: - - name: path - value: $(params.versionTag) - - name: bucketName - value: $(params.releaseBucket) - - name: objectPrefix - value: $(params.repoName)/latest/ - - name: replaceExistingFiles - value: "true" - - name: recursive - value: "true" - - name: deleteExtraFiles - value: "true" - - name: report-bucket runAfter: [publish-to-bucket] params: From 12946417b0d00bb42fcb981345e8fc25f5e30c62 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:15:20 +0530 Subject: [PATCH 4/7] feat: smoke test finished release artifacts in an isolated cluster Install the immutable manifest and image, require webhook mutation, admission and PipelineRun success, and retain diagnostics while cleaning only the owned cluster. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- .github/workflows/release-smoke.yaml | 56 ++++++++ hack/smoke-release.sh | 201 +++++++++++++++++++++++++++ hack/smoke-release_test.sh | 181 ++++++++++++++++++++++++ test/release-smoke/README.md | 23 +++ test/release-smoke/release.yaml | 40 ++++++ 5 files changed, 501 insertions(+) create mode 100644 .github/workflows/release-smoke.yaml create mode 100644 hack/smoke-release.sh create mode 100644 hack/smoke-release_test.sh create mode 100644 test/release-smoke/README.md create mode 100644 test/release-smoke/release.yaml diff --git a/.github/workflows/release-smoke.yaml b/.github/workflows/release-smoke.yaml new file mode 100644 index 000000000..1b98f10a1 --- /dev/null +++ b/.github/workflows/release-smoke.yaml @@ -0,0 +1,56 @@ +name: Released Artifact Smoke + +"on": + pull_request: + paths: + - ".github/workflows/release-smoke.yaml" + - ".github/workflows/finalize-release.yaml" + - "hack/*release*.sh" + - "tekton/publish.yaml" + - "tekton/release-pipeline.yaml" + - "test/release-smoke/**" + +permissions: + contents: read + +concurrency: + group: release-smoke-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Install pinned Kind + run: | + GOTOOLCHAIN=local go install sigs.k8s.io/kind@v0.32.0 + - name: Check smoke script behavior + run: | + bash -n hack/smoke-release.sh hack/smoke-release_test.sh + bash hack/smoke-release_test.sh + - name: Install and exercise the immutable release fixture + shell: bash + run: | + # A pinned public fixture tests installation, not the production trust bootstrap. + curl -fsSL https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/v0.4.3/release.yaml \ + -o "$RUNNER_TEMP/release.yaml" + printf '%s %s\n' \ + 340e21dbed798284de3e665e816e02f29f024af0cd32efa3cc98bbb7f9830270 \ + "$RUNNER_TEMP/release.yaml" | sha256sum -c - + bash hack/smoke-release.sh "$RUNNER_TEMP/release.yaml" \ + ghcr.io/tektoncd/tekton-kueue@sha256:6d89cbd8b47cebebc1c05c9e45292433e755372db68e0b66d5dd68d6ede9cb17 \ + "$RUNNER_TEMP/release-smoke" + - name: Save smoke diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-smoke-${{ github.run_id }} + path: ${{ runner.temp }}/release-smoke + if-no-files-found: ignore diff --git a/hack/smoke-release.sh b/hack/smoke-release.sh new file mode 100644 index 000000000..d7a91c2e5 --- /dev/null +++ b/hack/smoke-release.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release smoke failed: $*" >&2; exit 1; } +[[ $# == 3 ]] || fail "usage: $0 VERIFIED_MANIFEST EXPECTED_IMAGE ARTIFACT_DIR" +manifest=$1 +image=$2 +artifacts=$3 +repository=ghcr.io/tektoncd/tekton-kueue +[[ $image =~ ^ghcr\.io/tektoncd/tekton-kueue@sha256:[0-9a-f]{64}$ ]] || + fail "EXPECTED_IMAGE must be the canonical digest reference" +[[ -f $manifest && -r $manifest && -s $manifest ]] || fail "manifest must be a nonempty readable file" +[[ -n $artifacts && ! -L $artifacts ]] || fail "invalid artifact directory" +if [[ -e $artifacts ]]; then + [[ -d $artifacts && -z $(find "$artifacts" -mindepth 1 -print -quit) ]] || + fail "artifact directory must be empty" +fi +for command in kind kubectl jq openssl; do + command -v "$command" >/dev/null || fail "missing command: $command" +done +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d) +cluster="release-smoke-$(basename "$work" | tr '[:upper:]._' '[:lower:]--')" +context="kind-$cluster" +export KUBECONFIG="$work/kubeconfig" +owned=false + +k() { kubectl --context="$context" --request-timeout=30s "$@"; } +cleanup() { + status=$? + trap - EXIT + set +e + if [[ $owned == true ]]; then + for resource in pods deployments services endpoints events certificates \ + pipelineruns.tekton.dev taskruns.tekton.dev workloads.kueue.x-k8s.io; do + k get "$resource" -A -o yaml > "$artifacts/$resource.yaml" 2>&1 + done + k describe pods -A > "$artifacts/pod-descriptions.txt" 2>&1 + for namespace in cert-manager kueue-system tekton-pipelines tekton-kueue release-smoke; do + pods=$(k get pods -n "$namespace" -o name) + for pod in $pods; do + k logs -n "$namespace" "$pod" --all-containers=true --prefix=true \ + --tail=200 > "$artifacts/$namespace-${pod#*/}.log" 2>&1 + done + done + kind export logs --name "$cluster" "$artifacts/kind" > "$artifacts/kind-export.log" 2>&1 + kind delete cluster --name "$cluster" > "$artifacts/cleanup.log" 2>&1 + if [[ $? != 0 ]]; then + echo "failed to delete owned cluster: $cluster" >&2 + status=1 + fi + fi + rm -rf "$work" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# Parse the finished YAML locally, without discovery or the user's kubeconfig. +# This labels only the in-memory objects, never the installed release. +cp "$manifest" "$work/release.yaml" +kubectl label --local --overwrite -f "$work/release.yaml" -o json release-smoke-validation=true > "$work/manifest.json" +jq -e -s --arg repo "$repository" --arg digest "${image##*@}" ' + [ .[] | if .kind == "List" then .items[] else . end ] as $objects + | [$objects[] | select(.kind == "Deployment")] as $deployments + | ($deployments | length == 2) + and ($deployments | map(.metadata.name) | sort == + ["tekton-kueue-controller-manager", "tekton-kueue-webhook"]) + and all($deployments[]; + .apiVersion == "apps/v1" and .metadata.namespace == "tekton-kueue" + and (.spec.template.spec.containers | length == 1) + and ((.spec.template.spec.initContainers // []) | length == 0)) + and ([$objects[] | .. | objects | select(has("image")) | .image] | + length == 2 and all(.[]; + type == "string" and + (split("@") | length == 2 and .[1] == $digest + and (.[0] == $repo or + (.[0] | startswith($repo + ":") and + (ltrimstr($repo + ":") | test("^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$"))))))) +' "$work/manifest.json" >/dev/null || fail "manifest controller/webhook images do not match EXPECTED_IMAGE" + +mkdir -p "$artifacts" +artifacts=$(cd "$artifacts" && pwd) +cp "$work/release.yaml" "$artifacts/release.yaml" +printf '%s\n' "$image" > "$artifacts/expected-image.txt" +printf '%s\n' "$cluster" > "$artifacts/cluster-name.txt" +# Install only the snapshot checked above, even if the caller changes its file. +exec > >(tee "$artifacts/smoke.log") 2>&1 +owned=true # Creation can fail after provisioning nodes; still clean up our name. +kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 180s --retain \ + --image kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95 + +k apply --server-side -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml +k wait --for=condition=Available deployment --all -n cert-manager --timeout=300s +k apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml \ + -f "$root/hack/kueue-config.yaml" +k rollout restart deployment/kueue-controller-manager -n kueue-system +k rollout status deployment/kueue-controller-manager -n kueue-system --timeout=300s +k apply --server-side -f https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml +k wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=300s +k apply --server-side -f "$work/release.yaml" +k wait --for=condition=Ready certificate --all -n tekton-kueue --timeout=300s +k rollout status deployment/tekton-kueue-controller-manager -n tekton-kueue --timeout=300s +k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +k wait --for=jsonpath='{.webhooks[0].clientConfig.caBundle}' \ + mutatingwebhookconfiguration/tekton-kueue-mutating-webhook-configuration --timeout=300s + +# Configure one CPU resource mutation. Restart after configuration so the new +# webhook starts with the ConfigMap already present, rather than racing reload. +k apply -f - <<'YAML' +apiVersion: v1 +kind: ConfigMap +metadata: + name: tekton-kueue-config + namespace: tekton-kueue +data: + config.yaml: | + queueName: pipelines-queue + cel: + expressions: + - resource("cpu", 1) +--- +apiVersion: v1 +kind: Namespace +metadata: + name: release-smoke +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: ResourceFlavor +metadata: + name: release-smoke +spec: {} +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: ClusterQueue +metadata: + name: release-smoke +spec: + namespaceSelector: {} + resourceGroups: + - coveredResources: ["cpu", "tekton.dev/pipelineruns"] + flavors: + - name: release-smoke + resources: + - name: cpu + nominalQuota: 1 + - name: tekton.dev/pipelineruns + nominalQuota: 1 +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: LocalQueue +metadata: + name: pipelines-queue + namespace: release-smoke +spec: + clusterQueue: release-smoke +YAML +k rollout restart deployment/tekton-kueue-webhook -n tekton-kueue +k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +k wait --for=condition=Active clusterqueue/release-smoke --timeout=120s +k wait --for=condition=Active localqueue/pipelines-queue -n release-smoke --timeout=120s +k create -f - -o json > "$artifacts/created-pipelinerun.json" <<'YAML' +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: release-smoke + namespace: release-smoke +spec: + timeouts: + pipeline: 5m0s + pipelineSpec: + tasks: + - name: hello + taskSpec: + steps: + - name: hello + image: registry.access.redhat.com/ubi9/ubi@sha256:66233eebd72bb5baa25190d4f55e1dc3fff3a9b77186c1f91a0abdb274452072 + command: ["/bin/echo", "release smoke"] +YAML +# The create response is the admission result, before controller updates. +jq -e '.spec.status == "Pending" + and .metadata.labels["kueue.x-k8s.io/queue-name"] == "pipelines-queue" + and .metadata.annotations["kueue.konflux-ci.dev/requests-cpu"] == "1" + and (.metadata.uid | type == "string" and length > 0) +' "$artifacts/created-pipelinerun.json" >/dev/null || fail "webhook mutation missing" +uid=$(jq -r .metadata.uid "$artifacts/created-pipelinerun.json") +# Kueue v0.16.6 GetWorkloadNameForOwnerWithGVK hashes kind, group, name, UID. +hash=$(printf 'PipelineRun\ntekton.dev\nrelease-smoke\n%s' "$uid" | openssl dgst -sha1 -r) +workload="workload/pipelinerun-release-smoke-${hash:0:5}" +k wait --for=create "$workload" -n release-smoke --timeout=120s +k get "$workload" -n release-smoke -o json > "$artifacts/created-workload.json" +jq -e --arg uid "$uid" ' + any(.metadata.ownerReferences[]; .uid == $uid and .kind == "PipelineRun" and .controller == true) + and .spec.queueName == "pipelines-queue" + and .spec.podSets[0].template.spec.containers[0].resources.requests.cpu == "1" + and .spec.podSets[0].template.spec.containers[0].resources.requests["tekton.dev/pipelineruns"] == "1" +' "$artifacts/created-workload.json" >/dev/null || fail "Workload does not represent the mutated PipelineRun" +k wait --for=condition=Admitted "$workload" -n release-smoke --timeout=300s +k wait --for=condition=Succeeded pipelinerun/release-smoke -n release-smoke --timeout=360s +echo "PASS published release $image" diff --git a/hack/smoke-release_test.sh b/hack/smoke-release_test.sh new file mode 100644 index 000000000..856d0b36c --- /dev/null +++ b/hack/smoke-release_test.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The same executable supplies both boundary fakes. Local YAML parsing remains +# real kubectl, using the smoke script's isolated kubeconfig and --local. +if [[ ${SMOKE_FAKE:-} == true ]]; then + command=$(basename "$0") + [[ -n ${KUBECONFIG:-} && $KUBECONFIG != "$USER_KUBECONFIG" ]] || exit 90 + jq -cn --arg command "$command" --arg kubeconfig "$KUBECONFIG" \ + --args '$ARGS.positional as $args | {command:$command,kubeconfig:$kubeconfig,args:$args}' \ + -- "$@" >> "$CALLS" + if [[ $command == kind ]]; then + case "$*" in + "create cluster "*) + [[ $* == *"--kubeconfig $KUBECONFIG"* && $* == *"--wait 180s"* ]] || exit 91 + [[ $MODE != creation ]] || exit 42 + ;; + "delete cluster "*) + [[ $MODE != cleanup ]] || exit 42 + ;; + "export logs "*) ;; + *) exit 92 ;; + esac + exit 0 + fi + if [[ $1 == label ]]; then + [[ $* == *"--local"* ]] || exit 93 + exec "$REAL_KUBECTL" "$@" + fi + [[ $1 == --context=kind-release-smoke-* && $2 == --request-timeout=30s ]] || exit 94 + shift 2 + case "$1" in + apply) + if [[ $* == *"-f -"* ]]; then + # Parse actual fixtures offline; invalid YAML cannot silently pass. + "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ + > "$CASE_DIR/fixtures.json" + elif [[ $* == *"/release.yaml"* && $* != *https:* ]]; then + for ((i=1; i<=$#; i++)); do + if [[ ${!i} == -f ]]; then + j=$((i + 1)) + cmp "$VERIFIED_MANIFEST" "${!j}" + fi + done + fi + ;; + rollout) + if [[ $MODE == readiness && $* == *"status deployment/tekton-kueue-webhook"* ]]; then exit 42; fi + ;; + wait) + [[ $* == *--timeout=* ]] || exit 95 + if [[ $MODE == run && $* == *condition=Succeeded* ]]; then exit 42; fi + if [[ $MODE == admission && $* == *condition=Admitted* ]]; then exit 42; fi + if [[ $MODE == workload && $* == *--for=create* ]]; then exit 42; fi + ;; + create) + "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ + > "$CASE_DIR/pipelinerun.json" + jq -n --arg mode "$MODE" '{ + metadata:{uid:"smoke-uid", + labels:{"kueue.x-k8s.io/queue-name":"pipelines-queue"}, + annotations:{"kueue.konflux-ci.dev/requests-cpu":(if $mode == "mutation" then "2" else "1" end)}}, + spec:{status:"Pending"}}' + ;; + get) + if [[ $2 == workload/* ]]; then + jq -n '{metadata:{ownerReferences:[{uid:"smoke-uid",kind:"PipelineRun",controller:true}]}, + spec:{queueName:"pipelines-queue",podSets:[{template:{spec:{containers:[ + {resources:{requests:{cpu:"1","tekton.dev/pipelineruns":"1"}}} + ]}}}]}}' + elif [[ $* != *"-o name"* ]]; then + printf 'kind: List\nitems: []\n' + fi + ;; + describe|logs) ;; + *) exit 96 ;; + esac + exit 0 +fi + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'status=$?; if [[ $status == 0 ]]; then rm -rf "$tmp"; else echo "test artifacts: $tmp" >&2; fi' EXIT +export REAL_KUBECTL +REAL_KUBECTL=$(command -v kubectl) +mkdir "$tmp/bin" +cp "${BASH_SOURCE[0]}" "$tmp/bin/kubectl" +cp "${BASH_SOURCE[0]}" "$tmp/bin/kind" +chmod +x "$tmp/bin/kubectl" "$tmp/bin/kind" +export PATH="$tmp/bin:$PATH" SMOKE_FAKE=true +export USER_KUBECONFIG="$tmp/user-kubeconfig" +printf 'untouched user configuration\n' > "$USER_KUBECONFIG" +cp "$USER_KUBECONFIG" "$tmp/user-original" +image=ghcr.io/tektoncd/tekton-kueue@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export CASE_DIR CALLS MODE VERIFIED_MANIFEST +cases=0 + +check() { + jq -e -s "$1" "$CALLS" >/dev/null || { echo "failed assertion in $MODE: $1" >&2; exit 1; } +} +run_case() { + MODE=$1 + CASE_DIR="$tmp/$MODE" + mkdir "$CASE_DIR" + CALLS="$CASE_DIR/calls.jsonl" + : > "$CALLS" + VERIFIED_MANIFEST="$CASE_DIR/release.yaml" + local expected=$image + cp "$root/test/release-smoke/release.yaml" "$VERIFIED_MANIFEST" + case "$MODE" in + digest) expected=ghcr.io/tektoncd/tekton-kueue:latest ;; + mismatch) expected=${image/aaaa/bbbb} ;; + malformed) printf 'not: [valid yaml\n' > "$VERIFIED_MANIFEST" ;; + missing) rm "$VERIFIED_MANIFEST" ;; + incomplete) + # A valid document without the required controller/webhook pair. + printf 'apiVersion: v1\nkind: Namespace\nmetadata:\n name: test\n' > "$VERIFIED_MANIFEST" + ;; + artifacts) mkdir "$CASE_DIR/artifacts"; touch "$CASE_DIR/artifacts/keep" ;; + esac + local status=0 + KUBECONFIG="$USER_KUBECONFIG" bash "$root/hack/smoke-release.sh" \ + "$VERIFIED_MANIFEST" "$expected" "$CASE_DIR/artifacts" > "$CASE_DIR/output" 2>&1 || status=$? + case "$MODE" in + success|success-again) [[ $status == 0 ]] || { echo "success flow failed; see $CASE_DIR/output" >&2; "$REAL_KUBECTL" version --client; exit 1; } ;; + *) [[ $status != 0 ]] || { echo "$MODE unexpectedly passed" >&2; exit 1; } ;; + esac + cmp "$USER_KUBECONFIG" "$tmp/user-original" + case "$MODE" in + digest|mismatch|malformed|missing|incomplete|artifacts) + check 'all(.[]; .command != "kind")' + ;; + *) + check ' + [.[] | select(.command == "kind" and .args[0] == "create")][0] as $create + | ($create.args | index("--name")) as $nameIndex + | $create.args[$nameIndex + 1] as $name + | ($name | startswith("release-smoke-")) + and all(.[] | select(.command == "kind"); + (.args | index("--name")) as $i | .args[$i + 1] == $name) + and any(.[]; .command == "kind" and .args[0] == "delete") + and all(.[] | select(.command == "kubectl" and .args[0] != "label"); + .args[0] == ("--context=kind-" + $name) and .kubeconfig == $create.kubeconfig) + ' + [[ -s $CASE_DIR/artifacts/events.yaml && -f $CASE_DIR/artifacts/cleanup.log ]] || + { echo "missing failure diagnostics" >&2; exit 1; } + # The private kubeconfig directory must disappear even after failures. + local kubeconfig + kubeconfig=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .kubeconfig' "$CALLS") + [[ ! -e $(dirname "$kubeconfig") ]] + if [[ $MODE == success || $MODE == success-again ]]; then + check ' + any(.[]; .args | index("--for=condition=Ready")) + and any(.[]; .args | index("deployment/tekton-kueue-controller-manager")) + and any(.[]; .args | index("deployment/tekton-kueue-webhook")) + and any(.[]; .args | index("--for=create")) + and any(.[]; .args | index("--for=condition=Admitted")) + and any(.[]; .args | index("--for=condition=Succeeded")) + and any(.[]; .args | index("https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml")) + and any(.[]; .args | index("https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml")) + and any(.[]; .args | index("https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml")) + ' + cmp "$VERIFIED_MANIFEST" "$CASE_DIR/artifacts/release.yaml" + jq -e '.apiVersion == "tekton.dev/v1" and + (.spec.pipelineSpec.tasks[0].taskSpec.steps[0].image | contains("@sha256:"))' \ + "$CASE_DIR/pipelinerun.json" >/dev/null + fi + ;; + esac + cases=$((cases + 1)) + echo "PASS $MODE" +} + +for mode in digest mismatch malformed missing incomplete artifacts success success-again \ + creation readiness mutation workload admission run cleanup; do + run_case "$mode" +done +first=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success/calls.jsonl") +second=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success-again/calls.jsonl") +[[ $first != "$second" ]] +echo "PASS $cases deterministic release smoke cases" diff --git a/test/release-smoke/README.md b/test/release-smoke/README.md new file mode 100644 index 000000000..13d5a9b6b --- /dev/null +++ b/test/release-smoke/README.md @@ -0,0 +1,23 @@ +# Finished-release smoke test + +Run the published artifact verifier first, using an independently configured +signing public key. Pass its saved manifest and immutable image reference to: + +```sh +bash hack/smoke-release.sh VERIFIED_MANIFEST EXPECTED_IMAGE ARTIFACT_DIR +``` + +The script requires Bash, Kind, kubectl, jq, OpenSSL and a working container +provider. It creates its own cluster and kubeconfig, installs the manifest +without rebuilding its image, checks webhook mutation and Kueue admission, +and waits for a PipelineRun to succeed. It saves diagnostics and deletes only +the cluster it created, including on failure. The artifact directory must be +empty. + +The pull-request workflow exercises the existing public v0.4.3 manifest, +pinned by SHA-256 and image digest, on an isolated runner. This installation +check is not a signing-key trust bootstrap. The finalization workflow will +call the same script with the candidate that passed signature verification. + +`release.yaml` in this directory is an offline parsing fixture for +`bash hack/smoke-release_test.sh`; it is not an installable release. diff --git a/test/release-smoke/release.yaml b/test/release-smoke/release.yaml new file mode 100644 index 000000000..f45caa1db --- /dev/null +++ b/test/release-smoke/release.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tekton-kueue +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tekton-kueue-controller-manager + namespace: tekton-kueue +spec: + selector: + matchLabels: + app: controller + template: + metadata: + labels: + app: controller + spec: + containers: + - name: manager + image: ghcr.io/tektoncd/tekton-kueue:v0.4.3@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tekton-kueue-webhook + namespace: tekton-kueue +spec: + selector: + matchLabels: + app: webhook + template: + metadata: + labels: + app: webhook + spec: + containers: + - name: webhook + image: ghcr.io/tektoncd/tekton-kueue@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa From 972ca8b655d2ca59fb270e9b1327546b734a2b12 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:28:20 +0530 Subject: [PATCH 5/7] fix: assert the released Tekton pending status in smoke tests Use PipelineRunPending from the actual admission response and reject the incorrect Pending value in the deterministic boundary test. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- hack/smoke-release.sh | 2 +- hack/smoke-release_test.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/smoke-release.sh b/hack/smoke-release.sh index d7a91c2e5..f3cd989de 100644 --- a/hack/smoke-release.sh +++ b/hack/smoke-release.sh @@ -179,7 +179,7 @@ spec: command: ["/bin/echo", "release smoke"] YAML # The create response is the admission result, before controller updates. -jq -e '.spec.status == "Pending" +jq -e '.spec.status == "PipelineRunPending" and .metadata.labels["kueue.x-k8s.io/queue-name"] == "pipelines-queue" and .metadata.annotations["kueue.konflux-ci.dev/requests-cpu"] == "1" and (.metadata.uid | type == "string" and length > 0) diff --git a/hack/smoke-release_test.sh b/hack/smoke-release_test.sh index 856d0b36c..05fcbda92 100644 --- a/hack/smoke-release_test.sh +++ b/hack/smoke-release_test.sh @@ -60,7 +60,7 @@ if [[ ${SMOKE_FAKE:-} == true ]]; then metadata:{uid:"smoke-uid", labels:{"kueue.x-k8s.io/queue-name":"pipelines-queue"}, annotations:{"kueue.konflux-ci.dev/requests-cpu":(if $mode == "mutation" then "2" else "1" end)}}, - spec:{status:"Pending"}}' + spec:{status:(if $mode == "wrong-pending" then "Pending" else "PipelineRunPending" end)}}' ;; get) if [[ $2 == workload/* ]]; then @@ -172,7 +172,7 @@ run_case() { } for mode in digest mismatch malformed missing incomplete artifacts success success-again \ - creation readiness mutation workload admission run cleanup; do + creation readiness mutation wrong-pending workload admission run cleanup; do run_case "$mode" done first=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success/calls.jsonl") From e440a65278d296d1d1efd0d59e471e1ee04923b8 Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 18:39:43 +0530 Subject: [PATCH 6/7] fix: await webhook replacement endpoints before release admission Watch old Pod deletion and the Service endpoint switch after rollout instead of relying only on Deployment availability. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- hack/smoke-release.sh | 13 +++++++++++++ hack/smoke-release_test.sh | 12 ++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/hack/smoke-release.sh b/hack/smoke-release.sh index f3cd989de..a70b59e4e 100644 --- a/hack/smoke-release.sh +++ b/hack/smoke-release.sh @@ -156,8 +156,21 @@ metadata: spec: clusterQueue: release-smoke YAML +k get pods -n tekton-kueue -l app.kubernetes.io/name=tekton-kueue-webhook -o json \ + > "$work/old-webhook-pods.json" k rollout restart deployment/tekton-kueue-webhook -n tekton-kueue k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +# Deployment availability alone can precede removal of the old Service endpoint. +# Watch the old Pods disappear and the endpoint switch before admission. +while IFS= read -r pod; do + k wait --for=delete "pod/$pod" -n tekton-kueue --timeout=300s +done < <(jq -r '.items[].metadata.name' "$work/old-webhook-pods.json") +k get pods -n tekton-kueue -l app.kubernetes.io/name=tekton-kueue-webhook -o json \ + > "$work/webhook-pods.json" +pod=$(jq -er '[.items[] | select(.metadata.deletionTimestamp == null)] | + if length == 1 then .[0].metadata.name else error("expected one webhook Pod") end' "$work/webhook-pods.json") +k wait --for=jsonpath='{.subsets[0].addresses[0].targetRef.name}'="$pod" \ + endpoints/tekton-kueue-webhook-service -n tekton-kueue --timeout=120s k wait --for=condition=Active clusterqueue/release-smoke --timeout=120s k wait --for=condition=Active localqueue/pipelines-queue -n release-smoke --timeout=120s k create -f - -o json > "$artifacts/created-pipelinerun.json" <<'YAML' diff --git a/hack/smoke-release_test.sh b/hack/smoke-release_test.sh index 05fcbda92..546d84953 100644 --- a/hack/smoke-release_test.sh +++ b/hack/smoke-release_test.sh @@ -46,12 +46,14 @@ if [[ ${SMOKE_FAKE:-} == true ]]; then ;; rollout) if [[ $MODE == readiness && $* == *"status deployment/tekton-kueue-webhook"* ]]; then exit 42; fi + if [[ $* == *"restart deployment/tekton-kueue-webhook"* ]]; then touch "$CASE_DIR/restarted"; fi ;; wait) [[ $* == *--timeout=* ]] || exit 95 if [[ $MODE == run && $* == *condition=Succeeded* ]]; then exit 42; fi if [[ $MODE == admission && $* == *condition=Admitted* ]]; then exit 42; fi if [[ $MODE == workload && $* == *--for=create* ]]; then exit 42; fi + if [[ $MODE == endpoint && $* == *endpoints/tekton-kueue-webhook-service* ]]; then exit 42; fi ;; create) "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ @@ -63,7 +65,11 @@ if [[ ${SMOKE_FAKE:-} == true ]]; then spec:{status:(if $mode == "wrong-pending" then "Pending" else "PipelineRunPending" end)}}' ;; get) - if [[ $2 == workload/* ]]; then + if [[ $2 == pods && $* == *"-o json"* ]]; then + name=old-webhook + [[ ! -e $CASE_DIR/restarted ]] || name=new-webhook + jq -n --arg name "$name" '{items:[{metadata:{name:$name}}]}' + elif [[ $2 == workload/* ]]; then jq -n '{metadata:{ownerReferences:[{uid:"smoke-uid",kind:"PipelineRun",controller:true}]}, spec:{queueName:"pipelines-queue",podSets:[{template:{spec:{containers:[ {resources:{requests:{cpu:"1","tekton.dev/pipelineruns":"1"}}} @@ -156,6 +162,8 @@ run_case() { and any(.[]; .args | index("--for=create")) and any(.[]; .args | index("--for=condition=Admitted")) and any(.[]; .args | index("--for=condition=Succeeded")) + and any(.[]; .args | index("pod/old-webhook")) + and any(.[]; .args | index("endpoints/tekton-kueue-webhook-service")) and any(.[]; .args | index("https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml")) and any(.[]; .args | index("https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml")) and any(.[]; .args | index("https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml")) @@ -172,7 +180,7 @@ run_case() { } for mode in digest mismatch malformed missing incomplete artifacts success success-again \ - creation readiness mutation wrong-pending workload admission run cleanup; do + creation readiness mutation wrong-pending endpoint workload admission run cleanup; do run_case "$mode" done first=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success/calls.jsonl") From 2d2f04b6f8ee4746bc2a7052c607d95bd581d70b Mon Sep 17 00:00:00 2001 From: Vibhav Bobade Date: Tue, 17 Nov 2026 22:05:15 +0530 Subject: [PATCH 7/7] fix: verify release tags against the approved source Peel existing tags and fail on source conflicts or lookup errors. Permit missing tags only for candidates that have not been published. Assisted-by: OpenAI Signed-off-by: Vibhav Bobade --- hack/verify-release-artifacts.sh | 3 +++ hack/verify-release-artifacts_test.sh | 18 ++++++++++++-- hack/verify-release-tag.sh | 34 +++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 hack/verify-release-tag.sh diff --git a/hack/verify-release-artifacts.sh b/hack/verify-release-artifacts.sh index 90848df82..845f29232 100644 --- a/hack/verify-release-artifacts.sh +++ b/hack/verify-release-artifacts.sh @@ -164,6 +164,9 @@ jq -e -s --arg digest "${digest#sha256:}" --arg commit "$commit" ' fail "no verified provenance matching source and digest" echo "PASS configured-key signature and SLSA provenance" +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +bash "$root/hack/verify-release-tag.sh" "$version" "$commit" "$(jq -r '.draft != false' "$work/release.json")" + jq -n --argjson release_id "$release_id" --arg version "$version" --arg commit "$commit" \ --arg image "$image" --arg digest "$digest" --slurpfile release "$work/release.json" \ '{release_id: $release_id, version: $version, commit: $commit, image: $image, diff --git a/hack/verify-release-artifacts_test.sh b/hack/verify-release-artifacts_test.sh index f3e038824..b7367e189 100644 --- a/hack/verify-release-artifacts_test.sh +++ b/hack/verify-release-artifacts_test.sh @@ -69,13 +69,26 @@ cat > "$work/bin/fake" <<'EOF' set -euo pipefail case "${0##*/}" in gh) + if [[ $* == "api --paginate --slurp repos/tektoncd/tekton-kueue/git/matching-refs/tags/$VERSION" ]]; then + [[ $SCENARIO != tag-api-failure ]] || exit 42 + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + if $scenario == "missing-tag" or $scenario == "published-missing-tag" then [[]] + else [[{ref:("refs/tags/" + $version),object:{ + type:(if $scenario == "annotated-tag" or $scenario == "wrong-annotated-tag" then "tag" else "commit" end), + sha:(if $scenario == "wrong-tag" then "f"*40 else $commit end)}}]] end' + exit + elif [[ $* == "api repos/tektoncd/tekton-kueue/git/tags/$COMMIT" ]]; then + jq -n --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {object:{type:"commit",sha:(if $scenario == "wrong-annotated-tag" then "f"*40 else $commit end)}}' + exit + fi [[ $1 == api && $2 == --method && $3 == GET ]] || exit 90 if [[ $4 == repos/tektoncd/tekton-kueue/releases/123 ]]; then if [[ $SCENARIO == malformed-metadata ]]; then printf '{\n'; exit; fi jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' {id:123,tag_name:$version,target_commitish:$commit,draft:true,assets:[ {id:11,name:"release.yaml"},{id:12,name:"release.notags.yaml"},{id:13,name:"SHA256SUMS"}]} - | if $scenario == "published" then .draft = false + | if $scenario == "published" or $scenario == "published-missing-tag" then .draft = false elif $scenario == "metadata-id" then .id = 124 elif $scenario == "metadata-tag" then .tag_name = "v0.4.2" elif $scenario == "metadata-commit" then .target_commitish = "main" @@ -196,6 +209,7 @@ run_verifier() { passed=0 for SCENARIO in success published build-config-source multiple-attestations array-attestations \ + annotated-tag missing-tag wrong-tag wrong-annotated-tag published-missing-tag tag-api-failure \ checksum traversal absolute extra-checksum duplicate-checksum missing-checksum empty-checksum \ bucket-release.yaml bucket-release.notags.yaml bucket-SHA256SUMS \ malformed-metadata metadata-id metadata-tag metadata-commit metadata-draft metadata-assets \ @@ -210,7 +224,7 @@ for SCENARIO in success published build-config-source multiple-attestations arra status=0 run_verifier > "$work/log" 2>&1 || status=$? case "$SCENARIO" in - success|published|build-config-source|multiple-attestations|array-attestations) + success|published|build-config-source|multiple-attestations|array-attestations|annotated-tag|missing-tag) if [[ $status != 0 ]]; then cat "$work/log" >&2; exit 1; fi for file in release.yaml release.notags.yaml SHA256SUMS; do cmp "$FIXTURES/$file" "$work/output/$file" diff --git a/hack/verify-release-tag.sh b/hack/verify-release-tag.sh new file mode 100644 index 000000000..1aa05b686 --- /dev/null +++ b/hack/verify-release-tag.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release tag verification failed: $*" >&2; exit 1; } +[[ $# == 3 ]] || fail "usage: $0 VERSION COMMIT ALLOW_MISSING" +version=$1 +commit=$2 +allow_missing=$3 +repository=${GITHUB_REPOSITORY:-tektoncd/tekton-kueue} +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ && $commit =~ ^[0-9a-f]{40}$ && + $repository =~ ^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$ && + ( $allow_missing == true || $allow_missing == false ) ]] || fail "invalid candidate" +object=$(gh api --paginate --slurp "repos/$repository/git/matching-refs/tags/$version" | + jq -c --arg ref "refs/tags/$version" ' + [.[][] | select(.ref == $ref)] + | if length <= 1 then .[0].object // null else error("ambiguous tag") end') +if [[ $object == null ]]; then + [[ $allow_missing == true ]] || fail "published release tag is missing" + exit 0 +fi +while :; do + sha=$(jq -r .sha <<< "$object") + [[ $sha =~ ^[0-9a-f]{40}$ ]] || fail "invalid tag object" + case $(jq -r .type <<< "$object") in + commit) + [[ $sha == "$commit" ]] || fail "tag points to another source" + exit 0 + ;; + tag) + object=$(gh api "repos/$repository/git/tags/$sha" | jq -ce '.object') + ;; + *) fail "tag does not resolve to a commit" ;; + esac +done