diff --git a/.github/workflows/patch-release.yaml b/.github/workflows/patch-release.yaml index 2f7ae9923..561ed2de9 100644 --- a/.github/workflows/patch-release.yaml +++ b/.github/workflows/patch-release.yaml @@ -11,11 +11,6 @@ name: Patch Release description: "Version to release (for example, v0.5.1)" required: true type: string - release_as_latest: - description: "Publish as latest release" - required: false - type: boolean - default: true schedule: - cron: "0 10 * * 4" @@ -47,23 +42,6 @@ jobs: id: scan shell: bash run: | - latest_branch="" - latest_major=0 - latest_minor=0 - for ref in $(git branch -r --list 'origin/release-v*'); do - branch="${ref#origin/}" - if [[ "$branch" =~ ^release-v([0-9]+)\.([0-9]+)\.x$ ]]; then - major="${BASH_REMATCH[1]}" - minor="${BASH_REMATCH[2]}" - if [ "$major" -gt "$latest_major" ] || { [ "$major" -eq "$latest_major" ] && [ "$minor" -gt "$latest_minor" ]; }; then - latest_major=$major - latest_minor=$minor - latest_branch=$branch - fi - fi - done - echo "::notice::Latest release branch: ${latest_branch}" - min_major="${MIN_RELEASE_VERSION%%.*}" min_minor="${MIN_RELEASE_VERSION##*.}" releases=() @@ -93,9 +71,7 @@ jobs: fi next_version=$(echo "$last_tag" | awk -F. '{printf "%s.%s.%d", $1, $2, $3+1}') - is_latest=false - [ "$branch" = "$latest_branch" ] && is_latest=true - releases+=("{\"branch\":\"${branch}\",\"version\":\"${next_version}\",\"release_as_latest\":\"${is_latest}\"}") + releases+=("{\"branch\":\"${branch}\",\"version\":\"${next_version}\"}") done if [ ${#releases[@]} -eq 0 ]; then @@ -133,7 +109,6 @@ jobs: PAC_INCOMING_SECRET: ${{ secrets.PAC_INCOMING_SECRET }} RELEASE_BRANCH: ${{ matrix.release.branch }} RELEASE_VERSION: ${{ matrix.release.version }} - RELEASE_AS_LATEST: ${{ matrix.release.release_as_latest }} run: bash ./hack/trigger-patch-release.sh trigger-manual-release: @@ -184,5 +159,4 @@ jobs: PAC_INCOMING_SECRET: ${{ secrets.PAC_INCOMING_SECRET }} RELEASE_BRANCH: ${{ inputs.branch }} RELEASE_VERSION: ${{ inputs.version }} - RELEASE_AS_LATEST: ${{ inputs.release_as_latest }} run: bash ./hack/trigger-patch-release.sh diff --git a/.github/workflows/release-smoke.yaml b/.github/workflows/release-smoke.yaml new file mode 100644 index 000000000..1b98f10a1 --- /dev/null +++ b/.github/workflows/release-smoke.yaml @@ -0,0 +1,56 @@ +name: Released Artifact Smoke + +"on": + pull_request: + paths: + - ".github/workflows/release-smoke.yaml" + - ".github/workflows/finalize-release.yaml" + - "hack/*release*.sh" + - "tekton/publish.yaml" + - "tekton/release-pipeline.yaml" + - "test/release-smoke/**" + +permissions: + contents: read + +concurrency: + group: release-smoke-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Install pinned Kind + run: | + GOTOOLCHAIN=local go install sigs.k8s.io/kind@v0.32.0 + - name: Check smoke script behavior + run: | + bash -n hack/smoke-release.sh hack/smoke-release_test.sh + bash hack/smoke-release_test.sh + - name: Install and exercise the immutable release fixture + shell: bash + run: | + # A pinned public fixture tests installation, not the production trust bootstrap. + curl -fsSL https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/v0.4.3/release.yaml \ + -o "$RUNNER_TEMP/release.yaml" + printf '%s %s\n' \ + 340e21dbed798284de3e665e816e02f29f024af0cd32efa3cc98bbb7f9830270 \ + "$RUNNER_TEMP/release.yaml" | sha256sum -c - + bash hack/smoke-release.sh "$RUNNER_TEMP/release.yaml" \ + ghcr.io/tektoncd/tekton-kueue@sha256:6d89cbd8b47cebebc1c05c9e45292433e755372db68e0b66d5dd68d6ede9cb17 \ + "$RUNNER_TEMP/release-smoke" + - name: Save smoke diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-smoke-${{ github.run_id }} + path: ${{ runner.temp }}/release-smoke + if-no-files-found: ignore diff --git a/.github/workflows/release-verification.yaml b/.github/workflows/release-verification.yaml new file mode 100644 index 000000000..0d7b03d54 --- /dev/null +++ b/.github/workflows/release-verification.yaml @@ -0,0 +1,23 @@ +name: Release artifact verification + +on: + pull_request: + paths: + - hack/verify-release-artifacts.sh + - hack/verify-release-artifacts_test.sh + - .github/workflows/release-verification.yaml + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check verifier syntax and behavior + run: | + bash -n hack/verify-release-artifacts.sh hack/verify-release-artifacts_test.sh + bash hack/verify-release-artifacts_test.sh diff --git a/.tekton/release-patch.yaml b/.tekton/release-patch.yaml index e67af9ea1..ccfde1333 100644 --- a/.tekton/release-patch.yaml +++ b/.tekton/release-patch.yaml @@ -43,8 +43,6 @@ spec: value: tekton-robot - name: releaseBucket value: tekton-releases - - name: releaseAsLatest - value: "{{ release_as_latest }}" - name: platforms value: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountImagesPath diff --git a/.tekton/release.yaml b/.tekton/release.yaml index f6337bc36..310c69208 100644 --- a/.tekton/release.yaml +++ b/.tekton/release.yaml @@ -46,8 +46,6 @@ spec: value: tekton-robot - name: releaseBucket value: tekton-releases - - name: releaseAsLatest - value: "true" - name: platforms value: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountImagesPath diff --git a/hack/smoke-release.sh b/hack/smoke-release.sh new file mode 100644 index 000000000..a70b59e4e --- /dev/null +++ b/hack/smoke-release.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release smoke failed: $*" >&2; exit 1; } +[[ $# == 3 ]] || fail "usage: $0 VERIFIED_MANIFEST EXPECTED_IMAGE ARTIFACT_DIR" +manifest=$1 +image=$2 +artifacts=$3 +repository=ghcr.io/tektoncd/tekton-kueue +[[ $image =~ ^ghcr\.io/tektoncd/tekton-kueue@sha256:[0-9a-f]{64}$ ]] || + fail "EXPECTED_IMAGE must be the canonical digest reference" +[[ -f $manifest && -r $manifest && -s $manifest ]] || fail "manifest must be a nonempty readable file" +[[ -n $artifacts && ! -L $artifacts ]] || fail "invalid artifact directory" +if [[ -e $artifacts ]]; then + [[ -d $artifacts && -z $(find "$artifacts" -mindepth 1 -print -quit) ]] || + fail "artifact directory must be empty" +fi +for command in kind kubectl jq openssl; do + command -v "$command" >/dev/null || fail "missing command: $command" +done +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d) +cluster="release-smoke-$(basename "$work" | tr '[:upper:]._' '[:lower:]--')" +context="kind-$cluster" +export KUBECONFIG="$work/kubeconfig" +owned=false + +k() { kubectl --context="$context" --request-timeout=30s "$@"; } +cleanup() { + status=$? + trap - EXIT + set +e + if [[ $owned == true ]]; then + for resource in pods deployments services endpoints events certificates \ + pipelineruns.tekton.dev taskruns.tekton.dev workloads.kueue.x-k8s.io; do + k get "$resource" -A -o yaml > "$artifacts/$resource.yaml" 2>&1 + done + k describe pods -A > "$artifacts/pod-descriptions.txt" 2>&1 + for namespace in cert-manager kueue-system tekton-pipelines tekton-kueue release-smoke; do + pods=$(k get pods -n "$namespace" -o name) + for pod in $pods; do + k logs -n "$namespace" "$pod" --all-containers=true --prefix=true \ + --tail=200 > "$artifacts/$namespace-${pod#*/}.log" 2>&1 + done + done + kind export logs --name "$cluster" "$artifacts/kind" > "$artifacts/kind-export.log" 2>&1 + kind delete cluster --name "$cluster" > "$artifacts/cleanup.log" 2>&1 + if [[ $? != 0 ]]; then + echo "failed to delete owned cluster: $cluster" >&2 + status=1 + fi + fi + rm -rf "$work" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# Parse the finished YAML locally, without discovery or the user's kubeconfig. +# This labels only the in-memory objects, never the installed release. +cp "$manifest" "$work/release.yaml" +kubectl label --local --overwrite -f "$work/release.yaml" -o json release-smoke-validation=true > "$work/manifest.json" +jq -e -s --arg repo "$repository" --arg digest "${image##*@}" ' + [ .[] | if .kind == "List" then .items[] else . end ] as $objects + | [$objects[] | select(.kind == "Deployment")] as $deployments + | ($deployments | length == 2) + and ($deployments | map(.metadata.name) | sort == + ["tekton-kueue-controller-manager", "tekton-kueue-webhook"]) + and all($deployments[]; + .apiVersion == "apps/v1" and .metadata.namespace == "tekton-kueue" + and (.spec.template.spec.containers | length == 1) + and ((.spec.template.spec.initContainers // []) | length == 0)) + and ([$objects[] | .. | objects | select(has("image")) | .image] | + length == 2 and all(.[]; + type == "string" and + (split("@") | length == 2 and .[1] == $digest + and (.[0] == $repo or + (.[0] | startswith($repo + ":") and + (ltrimstr($repo + ":") | test("^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$"))))))) +' "$work/manifest.json" >/dev/null || fail "manifest controller/webhook images do not match EXPECTED_IMAGE" + +mkdir -p "$artifacts" +artifacts=$(cd "$artifacts" && pwd) +cp "$work/release.yaml" "$artifacts/release.yaml" +printf '%s\n' "$image" > "$artifacts/expected-image.txt" +printf '%s\n' "$cluster" > "$artifacts/cluster-name.txt" +# Install only the snapshot checked above, even if the caller changes its file. +exec > >(tee "$artifacts/smoke.log") 2>&1 +owned=true # Creation can fail after provisioning nodes; still clean up our name. +kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 180s --retain \ + --image kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95 + +k apply --server-side -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml +k wait --for=condition=Available deployment --all -n cert-manager --timeout=300s +k apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml \ + -f "$root/hack/kueue-config.yaml" +k rollout restart deployment/kueue-controller-manager -n kueue-system +k rollout status deployment/kueue-controller-manager -n kueue-system --timeout=300s +k apply --server-side -f https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml +k wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=300s +k apply --server-side -f "$work/release.yaml" +k wait --for=condition=Ready certificate --all -n tekton-kueue --timeout=300s +k rollout status deployment/tekton-kueue-controller-manager -n tekton-kueue --timeout=300s +k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +k wait --for=jsonpath='{.webhooks[0].clientConfig.caBundle}' \ + mutatingwebhookconfiguration/tekton-kueue-mutating-webhook-configuration --timeout=300s + +# Configure one CPU resource mutation. Restart after configuration so the new +# webhook starts with the ConfigMap already present, rather than racing reload. +k apply -f - <<'YAML' +apiVersion: v1 +kind: ConfigMap +metadata: + name: tekton-kueue-config + namespace: tekton-kueue +data: + config.yaml: | + queueName: pipelines-queue + cel: + expressions: + - resource("cpu", 1) +--- +apiVersion: v1 +kind: Namespace +metadata: + name: release-smoke +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: ResourceFlavor +metadata: + name: release-smoke +spec: {} +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: ClusterQueue +metadata: + name: release-smoke +spec: + namespaceSelector: {} + resourceGroups: + - coveredResources: ["cpu", "tekton.dev/pipelineruns"] + flavors: + - name: release-smoke + resources: + - name: cpu + nominalQuota: 1 + - name: tekton.dev/pipelineruns + nominalQuota: 1 +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: LocalQueue +metadata: + name: pipelines-queue + namespace: release-smoke +spec: + clusterQueue: release-smoke +YAML +k get pods -n tekton-kueue -l app.kubernetes.io/name=tekton-kueue-webhook -o json \ + > "$work/old-webhook-pods.json" +k rollout restart deployment/tekton-kueue-webhook -n tekton-kueue +k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +# Deployment availability alone can precede removal of the old Service endpoint. +# Watch the old Pods disappear and the endpoint switch before admission. +while IFS= read -r pod; do + k wait --for=delete "pod/$pod" -n tekton-kueue --timeout=300s +done < <(jq -r '.items[].metadata.name' "$work/old-webhook-pods.json") +k get pods -n tekton-kueue -l app.kubernetes.io/name=tekton-kueue-webhook -o json \ + > "$work/webhook-pods.json" +pod=$(jq -er '[.items[] | select(.metadata.deletionTimestamp == null)] | + if length == 1 then .[0].metadata.name else error("expected one webhook Pod") end' "$work/webhook-pods.json") +k wait --for=jsonpath='{.subsets[0].addresses[0].targetRef.name}'="$pod" \ + endpoints/tekton-kueue-webhook-service -n tekton-kueue --timeout=120s +k wait --for=condition=Active clusterqueue/release-smoke --timeout=120s +k wait --for=condition=Active localqueue/pipelines-queue -n release-smoke --timeout=120s +k create -f - -o json > "$artifacts/created-pipelinerun.json" <<'YAML' +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: release-smoke + namespace: release-smoke +spec: + timeouts: + pipeline: 5m0s + pipelineSpec: + tasks: + - name: hello + taskSpec: + steps: + - name: hello + image: registry.access.redhat.com/ubi9/ubi@sha256:66233eebd72bb5baa25190d4f55e1dc3fff3a9b77186c1f91a0abdb274452072 + command: ["/bin/echo", "release smoke"] +YAML +# The create response is the admission result, before controller updates. +jq -e '.spec.status == "PipelineRunPending" + and .metadata.labels["kueue.x-k8s.io/queue-name"] == "pipelines-queue" + and .metadata.annotations["kueue.konflux-ci.dev/requests-cpu"] == "1" + and (.metadata.uid | type == "string" and length > 0) +' "$artifacts/created-pipelinerun.json" >/dev/null || fail "webhook mutation missing" +uid=$(jq -r .metadata.uid "$artifacts/created-pipelinerun.json") +# Kueue v0.16.6 GetWorkloadNameForOwnerWithGVK hashes kind, group, name, UID. +hash=$(printf 'PipelineRun\ntekton.dev\nrelease-smoke\n%s' "$uid" | openssl dgst -sha1 -r) +workload="workload/pipelinerun-release-smoke-${hash:0:5}" +k wait --for=create "$workload" -n release-smoke --timeout=120s +k get "$workload" -n release-smoke -o json > "$artifacts/created-workload.json" +jq -e --arg uid "$uid" ' + any(.metadata.ownerReferences[]; .uid == $uid and .kind == "PipelineRun" and .controller == true) + and .spec.queueName == "pipelines-queue" + and .spec.podSets[0].template.spec.containers[0].resources.requests.cpu == "1" + and .spec.podSets[0].template.spec.containers[0].resources.requests["tekton.dev/pipelineruns"] == "1" +' "$artifacts/created-workload.json" >/dev/null || fail "Workload does not represent the mutated PipelineRun" +k wait --for=condition=Admitted "$workload" -n release-smoke --timeout=300s +k wait --for=condition=Succeeded pipelinerun/release-smoke -n release-smoke --timeout=360s +echo "PASS published release $image" diff --git a/hack/smoke-release_test.sh b/hack/smoke-release_test.sh new file mode 100644 index 000000000..546d84953 --- /dev/null +++ b/hack/smoke-release_test.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The same executable supplies both boundary fakes. Local YAML parsing remains +# real kubectl, using the smoke script's isolated kubeconfig and --local. +if [[ ${SMOKE_FAKE:-} == true ]]; then + command=$(basename "$0") + [[ -n ${KUBECONFIG:-} && $KUBECONFIG != "$USER_KUBECONFIG" ]] || exit 90 + jq -cn --arg command "$command" --arg kubeconfig "$KUBECONFIG" \ + --args '$ARGS.positional as $args | {command:$command,kubeconfig:$kubeconfig,args:$args}' \ + -- "$@" >> "$CALLS" + if [[ $command == kind ]]; then + case "$*" in + "create cluster "*) + [[ $* == *"--kubeconfig $KUBECONFIG"* && $* == *"--wait 180s"* ]] || exit 91 + [[ $MODE != creation ]] || exit 42 + ;; + "delete cluster "*) + [[ $MODE != cleanup ]] || exit 42 + ;; + "export logs "*) ;; + *) exit 92 ;; + esac + exit 0 + fi + if [[ $1 == label ]]; then + [[ $* == *"--local"* ]] || exit 93 + exec "$REAL_KUBECTL" "$@" + fi + [[ $1 == --context=kind-release-smoke-* && $2 == --request-timeout=30s ]] || exit 94 + shift 2 + case "$1" in + apply) + if [[ $* == *"-f -"* ]]; then + # Parse actual fixtures offline; invalid YAML cannot silently pass. + "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ + > "$CASE_DIR/fixtures.json" + elif [[ $* == *"/release.yaml"* && $* != *https:* ]]; then + for ((i=1; i<=$#; i++)); do + if [[ ${!i} == -f ]]; then + j=$((i + 1)) + cmp "$VERIFIED_MANIFEST" "${!j}" + fi + done + fi + ;; + rollout) + if [[ $MODE == readiness && $* == *"status deployment/tekton-kueue-webhook"* ]]; then exit 42; fi + if [[ $* == *"restart deployment/tekton-kueue-webhook"* ]]; then touch "$CASE_DIR/restarted"; fi + ;; + wait) + [[ $* == *--timeout=* ]] || exit 95 + if [[ $MODE == run && $* == *condition=Succeeded* ]]; then exit 42; fi + if [[ $MODE == admission && $* == *condition=Admitted* ]]; then exit 42; fi + if [[ $MODE == workload && $* == *--for=create* ]]; then exit 42; fi + if [[ $MODE == endpoint && $* == *endpoints/tekton-kueue-webhook-service* ]]; then exit 42; fi + ;; + create) + "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ + > "$CASE_DIR/pipelinerun.json" + jq -n --arg mode "$MODE" '{ + metadata:{uid:"smoke-uid", + labels:{"kueue.x-k8s.io/queue-name":"pipelines-queue"}, + annotations:{"kueue.konflux-ci.dev/requests-cpu":(if $mode == "mutation" then "2" else "1" end)}}, + spec:{status:(if $mode == "wrong-pending" then "Pending" else "PipelineRunPending" end)}}' + ;; + get) + if [[ $2 == pods && $* == *"-o json"* ]]; then + name=old-webhook + [[ ! -e $CASE_DIR/restarted ]] || name=new-webhook + jq -n --arg name "$name" '{items:[{metadata:{name:$name}}]}' + elif [[ $2 == workload/* ]]; then + jq -n '{metadata:{ownerReferences:[{uid:"smoke-uid",kind:"PipelineRun",controller:true}]}, + spec:{queueName:"pipelines-queue",podSets:[{template:{spec:{containers:[ + {resources:{requests:{cpu:"1","tekton.dev/pipelineruns":"1"}}} + ]}}}]}}' + elif [[ $* != *"-o name"* ]]; then + printf 'kind: List\nitems: []\n' + fi + ;; + describe|logs) ;; + *) exit 96 ;; + esac + exit 0 +fi + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'status=$?; if [[ $status == 0 ]]; then rm -rf "$tmp"; else echo "test artifacts: $tmp" >&2; fi' EXIT +export REAL_KUBECTL +REAL_KUBECTL=$(command -v kubectl) +mkdir "$tmp/bin" +cp "${BASH_SOURCE[0]}" "$tmp/bin/kubectl" +cp "${BASH_SOURCE[0]}" "$tmp/bin/kind" +chmod +x "$tmp/bin/kubectl" "$tmp/bin/kind" +export PATH="$tmp/bin:$PATH" SMOKE_FAKE=true +export USER_KUBECONFIG="$tmp/user-kubeconfig" +printf 'untouched user configuration\n' > "$USER_KUBECONFIG" +cp "$USER_KUBECONFIG" "$tmp/user-original" +image=ghcr.io/tektoncd/tekton-kueue@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export CASE_DIR CALLS MODE VERIFIED_MANIFEST +cases=0 + +check() { + jq -e -s "$1" "$CALLS" >/dev/null || { echo "failed assertion in $MODE: $1" >&2; exit 1; } +} +run_case() { + MODE=$1 + CASE_DIR="$tmp/$MODE" + mkdir "$CASE_DIR" + CALLS="$CASE_DIR/calls.jsonl" + : > "$CALLS" + VERIFIED_MANIFEST="$CASE_DIR/release.yaml" + local expected=$image + cp "$root/test/release-smoke/release.yaml" "$VERIFIED_MANIFEST" + case "$MODE" in + digest) expected=ghcr.io/tektoncd/tekton-kueue:latest ;; + mismatch) expected=${image/aaaa/bbbb} ;; + malformed) printf 'not: [valid yaml\n' > "$VERIFIED_MANIFEST" ;; + missing) rm "$VERIFIED_MANIFEST" ;; + incomplete) + # A valid document without the required controller/webhook pair. + printf 'apiVersion: v1\nkind: Namespace\nmetadata:\n name: test\n' > "$VERIFIED_MANIFEST" + ;; + artifacts) mkdir "$CASE_DIR/artifacts"; touch "$CASE_DIR/artifacts/keep" ;; + esac + local status=0 + KUBECONFIG="$USER_KUBECONFIG" bash "$root/hack/smoke-release.sh" \ + "$VERIFIED_MANIFEST" "$expected" "$CASE_DIR/artifacts" > "$CASE_DIR/output" 2>&1 || status=$? + case "$MODE" in + success|success-again) [[ $status == 0 ]] || { echo "success flow failed; see $CASE_DIR/output" >&2; "$REAL_KUBECTL" version --client; exit 1; } ;; + *) [[ $status != 0 ]] || { echo "$MODE unexpectedly passed" >&2; exit 1; } ;; + esac + cmp "$USER_KUBECONFIG" "$tmp/user-original" + case "$MODE" in + digest|mismatch|malformed|missing|incomplete|artifacts) + check 'all(.[]; .command != "kind")' + ;; + *) + check ' + [.[] | select(.command == "kind" and .args[0] == "create")][0] as $create + | ($create.args | index("--name")) as $nameIndex + | $create.args[$nameIndex + 1] as $name + | ($name | startswith("release-smoke-")) + and all(.[] | select(.command == "kind"); + (.args | index("--name")) as $i | .args[$i + 1] == $name) + and any(.[]; .command == "kind" and .args[0] == "delete") + and all(.[] | select(.command == "kubectl" and .args[0] != "label"); + .args[0] == ("--context=kind-" + $name) and .kubeconfig == $create.kubeconfig) + ' + [[ -s $CASE_DIR/artifacts/events.yaml && -f $CASE_DIR/artifacts/cleanup.log ]] || + { echo "missing failure diagnostics" >&2; exit 1; } + # The private kubeconfig directory must disappear even after failures. + local kubeconfig + kubeconfig=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .kubeconfig' "$CALLS") + [[ ! -e $(dirname "$kubeconfig") ]] + if [[ $MODE == success || $MODE == success-again ]]; then + check ' + any(.[]; .args | index("--for=condition=Ready")) + and any(.[]; .args | index("deployment/tekton-kueue-controller-manager")) + and any(.[]; .args | index("deployment/tekton-kueue-webhook")) + and any(.[]; .args | index("--for=create")) + and any(.[]; .args | index("--for=condition=Admitted")) + and any(.[]; .args | index("--for=condition=Succeeded")) + and any(.[]; .args | index("pod/old-webhook")) + and any(.[]; .args | index("endpoints/tekton-kueue-webhook-service")) + and any(.[]; .args | index("https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml")) + and any(.[]; .args | index("https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml")) + and any(.[]; .args | index("https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml")) + ' + cmp "$VERIFIED_MANIFEST" "$CASE_DIR/artifacts/release.yaml" + jq -e '.apiVersion == "tekton.dev/v1" and + (.spec.pipelineSpec.tasks[0].taskSpec.steps[0].image | contains("@sha256:"))' \ + "$CASE_DIR/pipelinerun.json" >/dev/null + fi + ;; + esac + cases=$((cases + 1)) + echo "PASS $MODE" +} + +for mode in digest mismatch malformed missing incomplete artifacts success success-again \ + creation readiness mutation wrong-pending endpoint workload admission run cleanup; do + run_case "$mode" +done +first=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success/calls.jsonl") +second=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success-again/calls.jsonl") +[[ $first != "$second" ]] +echo "PASS $cases deterministic release smoke cases" diff --git a/hack/trigger-patch-release.sh b/hack/trigger-patch-release.sh index 38a22fcfa..563423efd 100644 --- a/hack/trigger-patch-release.sh +++ b/hack/trigger-patch-release.sh @@ -7,7 +7,6 @@ set -euo pipefail : "${PAC_CONTROLLER_URL:?}" : "${PAC_INCOMING_SECRET:?}" : "${PAC_REPOSITORY_NAME:?}" -: "${RELEASE_AS_LATEST:?}" : "${RELEASE_BRANCH:?}" : "${RELEASE_VERSION:?}" @@ -23,8 +22,7 @@ jq -n '{ pipelinerun: "release-patch", secret: env.PAC_INCOMING_SECRET, params: { - version: env.RELEASE_VERSION, - release_as_latest: env.RELEASE_AS_LATEST + version: env.RELEASE_VERSION } }' | curl --fail-with-body --silent --show-error \ -H "Content-Type: application/json" \ @@ -97,7 +95,8 @@ draft_url=$(jq -r '.html_url' <<<"$draft") echo echo "- External PipelineRun: ${details_url}" echo "- Draft release: ${draft_url}" - echo "- Publication: manual review required" + echo "- Finalization: approval, artifact verification, and installation smoke test required" + echo "- Latest image and bucket promotion: disabled during staging" } >> "$GITHUB_STEP_SUMMARY" -echo "::warning title=Draft release awaiting publication::Review and publish ${draft_url}" +echo "::warning title=Draft release awaiting finalization::Review ${draft_url}; publication and latest promotion require separate approval" diff --git a/hack/verify-release-artifacts.sh b/hack/verify-release-artifacts.sh new file mode 100644 index 000000000..845f29232 --- /dev/null +++ b/hack/verify-release-artifacts.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fail() { + echo "release verification failed: $*" >&2 + exit 1 +} + +[[ $# == 5 ]] || fail "usage: $0 VERSION EXPECTED_COMMIT RELEASE_ID PUBLIC_KEY_FILE OUTPUT_DIR" +version=$1 +commit=$2 +release_id=$3 +key=$4 +output=$5 +repository=${GITHUB_REPOSITORY:-tektoncd/tekton-kueue} +image_repository=ghcr.io/tektoncd/tekton-kueue +source_url=https://github.com/tektoncd/tekton-kueue + +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "invalid version" +[[ $commit =~ ^[0-9a-f]{40}$ ]] || fail "invalid commit" +[[ $release_id =~ ^[1-9][0-9]*$ ]] || fail "invalid release ID" +[[ $repository =~ ^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$ && + ${repository##*/} != . && ${repository##*/} != .. ]] || fail "invalid GitHub repository" +[[ -n ${GH_TOKEN:-} ]] || fail "GH_TOKEN is required" +[[ -f $key && -s $key && -r $key ]] || fail "a nonempty configured public key file is required" +[[ -n $output && ! -L $output ]] || fail "invalid output directory" +if [[ -e $output ]]; then + [[ -d $output ]] || fail "output must be a directory" + [[ -z $(find "$output" -mindepth 1 -print -quit) ]] || fail "output directory must be empty" +fi +for command in gh jq curl crane cosign openssl awk cmp; do + command -v "$command" >/dev/null || fail "missing command: $command" +done +openssl pkey -pubin -in "$key" -noout >/dev/null 2>&1 || fail "invalid configured public key" +if command -v sha256sum >/dev/null; then + checksum=(sha256sum) +else + command -v shasum >/dev/null || fail "sha256sum or shasum is required" + checksum=(shasum -a 256) +fi + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +gh api --method GET "repos/$repository/releases/$release_id" > "$work/release.json" +jq -e --arg version "$version" --arg commit "$commit" --arg id "$release_id" ' + .id == ($id | tonumber) and .tag_name == $version and .target_commitish == $commit + and (.draft | type == "boolean") and (.assets | type == "array") + and ([.assets[] | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS")] + | length == 3 and (map(.name) | unique | length == 3) + and all(.[]; (.id | type == "number") and .id > 0 and .id == (.id | floor)) + and (map(.id) | unique | length == 3)) +' "$work/release.json" >/dev/null || fail "release metadata does not match candidate" + +for file in release.yaml release.notags.yaml SHA256SUMS; do + asset_id=$(jq -r --arg name "$file" '.assets[] | select(.name == $name) | .id' "$work/release.json") + gh api --method GET -H "Accept: application/octet-stream" \ + "repos/$repository/releases/assets/$asset_id" > "$work/$file" + [[ -s $work/$file ]] || fail "empty asset: $file" +done + +# Parse before hashing: never pass downloaded filenames to a checksum tool. +declare -A seen=() +while IFS= read -r line || [[ -n $line ]]; do + [[ $line =~ ^([0-9a-f]{64})[[:space:]][\ \*](release\.yaml|release\.notags\.yaml)$ ]] || + fail "unsafe or malformed SHA256SUMS" + hash=${BASH_REMATCH[1]} + file=${BASH_REMATCH[2]} + [[ -z ${seen[$file]:-} ]] || fail "duplicate checksum filename" + seen[$file]=1 + actual=$("${checksum[@]}" "$work/$file") + [[ ${actual%% *} == "$hash" ]] || fail "checksum mismatch: $file" +done < "$work/SHA256SUMS" +[[ ${#seen[@]} == 2 ]] || fail "SHA256SUMS must contain exactly the two manifests" +for file in release.yaml release.notags.yaml SHA256SUMS; do + curl --fail --silent --show-error --location \ + "https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$version/$file" \ + --output "$work/bucket-$file" + cmp -s "$work/$file" "$work/bucket-$file" || fail "bucket mismatch: $file" +done +echo "PASS release metadata, checksums and bucket copies" + +# Kustomize emits block YAML with top-level metadata and scalar image fields. +# Require the controller and webhook deployments, not just an image somewhere. +for file in release.yaml release.notags.yaml; do + awk ' + /^---/ { kind = ""; name = "" } + /^kind: / { kind = $2 } + /^metadata:/ { metadata = 1; next } + /^[^ ]/ { metadata = 0 } + metadata && /^ name: / { name = $2 } + $1 == "image:" { print kind "\t" name "\t" $2 } + ' "$work/$file" > "$work/images" + count=0 + controller=0 + webhook=0 + while IFS=$'\t' read -r kind name reference; do + [[ $kind == Deployment ]] || fail "unexpected image outside a Deployment" + case "$name" in + tekton-kueue-controller-manager) controller=$((controller + 1)) ;; + tekton-kueue-webhook) webhook=$((webhook + 1)) ;; + *) fail "unexpected image deployment" ;; + esac + # Both variants must have the same digest; only release.yaml may have the version tag. + prefix=$image_repository + if [[ $file == release.yaml ]]; then + prefix="$prefix:$version" + fi + [[ $reference == "$prefix"@sha256:* ]] || fail "unexpected image repository or tag" + candidate=${reference##*@} + [[ $candidate =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid image digest" + [[ ${digest:-$candidate} == "$candidate" ]] || fail "inconsistent manifest image digests" + digest=$candidate + count=$((count + 1)) + done < "$work/images" + [[ $count == 2 && $controller == 1 && $webhook == 1 ]] || + fail "both controller and webhook images are required" +done +image="$image_repository@$digest" + +crane manifest "$image" > "$work/index.json" +jq -e ' + [.manifests[].platform | .os + "/" + .architecture] | sort == + ["linux/amd64", "linux/arm64", "linux/ppc64le", "linux/s390x"] +' "$work/index.json" >/dev/null || fail "unexpected image platforms" +for platform in linux/amd64 linux/arm64 linux/ppc64le linux/s390x; do + crane config --platform "$platform" "$image" > "$work/config.json" + jq -e --arg source "$source_url" --arg platform "$platform" ' + (.os + "/" + .architecture) == $platform + and .config.Labels["org.opencontainers.image.source"] == $source + and .config.Labels["org.opencontainers.image.url"] == $source + ' "$work/config.json" >/dev/null || fail "incorrect OCI labels or platform: $platform" +done +echo "PASS manifest images, four platforms and OCI labels" + +cosign verify --key "$key" "$image" > "$work/signatures.json" +jq -e -s --arg digest "$digest" ' + [ .[] | if type == "array" then .[] else . end + | select(.critical.image["docker-manifest-digest"] == $digest) ] | length > 0 +' \ + "$work/signatures.json" >/dev/null || fail "no verified image signature" +cosign verify-attestation --key "$key" --type https://slsa.dev/provenance/v1 \ + "$image" > "$work/attestations.json" +# cosign emits verified DSSE envelopes as JSON objects (or an array). +jq -e -s --arg digest "${digest#sha256:}" --arg commit "$commit" ' + def repository: + # Chains uses git+https URIs with an optional ref suffix. + test("^(git\\+)?https://github\\.com/tektoncd/tekton-kueue(\\.git)?(@[^[:space:]]+)?$"); + def source: + (.uri // "" | repository) + and ((.digest.sha1 // .digest.gitCommit) == $commit); + [ .[] | if type == "array" then .[] else . end + | .payload as $payload | $payload | @base64d | fromjson + | select(._type == "https://in-toto.io/Statement/v1" + and .predicateType == "https://slsa.dev/provenance/v1") + | select(any(.subject[]; .digest.sha256 == $digest)) + | select( + any(.predicate.buildDefinition.resolvedDependencies[]?; source) + or (.predicate.buildDefinition.externalParameters.buildConfigSource + | (.repository // "" | repository) and .ref == ("sha1:" + $commit))) + | $payload + ] | if length > 0 then . else error("no matching verified provenance") end +' "$work/attestations.json" > "$work/verified-payloads.json" || + fail "no verified provenance matching source and digest" +echo "PASS configured-key signature and SLSA provenance" + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +bash "$root/hack/verify-release-tag.sh" "$version" "$commit" "$(jq -r '.draft != false' "$work/release.json")" + +jq -n --argjson release_id "$release_id" --arg version "$version" --arg commit "$commit" \ + --arg image "$image" --arg digest "$digest" --slurpfile release "$work/release.json" \ + '{release_id: $release_id, version: $version, commit: $commit, image: $image, + digest: $digest, draft: $release[0].draft, + assets: [$release[0].assets[] + | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS") + | {name, id}] | sort_by(.name)}' > "$work/release-info.json" +mkdir -p "$output" +for file in release.yaml release.notags.yaml SHA256SUMS release-info.json verified-payloads.json; do + # Refuse to overwrite even if another process populated the output during verification. + (set -o noclobber; cat "$work/$file" > "$output/$file") +done +echo "PASS $version artifacts saved to $output" diff --git a/hack/verify-release-artifacts_test.sh b/hack/verify-release-artifacts_test.sh new file mode 100644 index 000000000..b7367e189 --- /dev/null +++ b/hack/verify-release-artifacts_test.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +export FIXTURES="$work/fixtures" +mkdir -p "$work/bin" "$FIXTURES" +export PATH="$work/bin:$PATH" +export GH_TOKEN=test-token +unset GITHUB_REPOSITORY +export VERSION=v0.4.3 +export COMMIT=0123456789abcdef0123456789abcdef01234567 +export DIGEST=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export IMAGE="ghcr.io/tektoncd/tekton-kueue@sha256:$DIGEST" + +# A fixed Ed25519 public key; openssl (not a fake) validates this boundary. +cat > "$work/key.pub" <<'EOF' +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo= +-----END PUBLIC KEY----- +EOF +write_artifacts() { +for variant in release.yaml release.notags.yaml; do + reference=$IMAGE + [[ $variant != release.yaml ]] || reference="ghcr.io/tektoncd/tekton-kueue:$VERSION@sha256:$DIGEST" + for deployment in controller-manager webhook; do + name=$deployment + candidate=$reference + case "${SCENARIO:-success}" in + wrong-image-repository) candidate=${candidate/tektoncd\/tekton-kueue/other\/tekton-kueue} ;; + inconsistent-image) [[ $deployment != webhook ]] || candidate=${candidate/$DIGEST/${DIGEST//a/b}} ;; + missing-webhook) [[ $deployment != webhook ]] || continue ;; + wrong-deployment) [[ $deployment != webhook ]] || name=other ;; + tag-in-notags) [[ $variant != release.notags.yaml ]] || candidate="ghcr.io/tektoncd/tekton-kueue:$VERSION@sha256:$DIGEST" ;; + esac + cat < "$FIXTURES/$variant" +done +( + cd "$FIXTURES" + if command -v sha256sum >/dev/null; then + sha256sum release.yaml release.notags.yaml + else + shasum -a 256 release.yaml release.notags.yaml + fi +) > "$FIXTURES/SHA256SUMS" +} + +# Only network/registry/signing executables are replaced. JSON parsing, hashing, +# key parsing, manifest extraction and byte comparisons use the real commands. +cat > "$work/bin/fake" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "${0##*/}" in + gh) + if [[ $* == "api --paginate --slurp repos/tektoncd/tekton-kueue/git/matching-refs/tags/$VERSION" ]]; then + [[ $SCENARIO != tag-api-failure ]] || exit 42 + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + if $scenario == "missing-tag" or $scenario == "published-missing-tag" then [[]] + else [[{ref:("refs/tags/" + $version),object:{ + type:(if $scenario == "annotated-tag" or $scenario == "wrong-annotated-tag" then "tag" else "commit" end), + sha:(if $scenario == "wrong-tag" then "f"*40 else $commit end)}}]] end' + exit + elif [[ $* == "api repos/tektoncd/tekton-kueue/git/tags/$COMMIT" ]]; then + jq -n --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {object:{type:"commit",sha:(if $scenario == "wrong-annotated-tag" then "f"*40 else $commit end)}}' + exit + fi + [[ $1 == api && $2 == --method && $3 == GET ]] || exit 90 + if [[ $4 == repos/tektoncd/tekton-kueue/releases/123 ]]; then + if [[ $SCENARIO == malformed-metadata ]]; then printf '{\n'; exit; fi + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {id:123,tag_name:$version,target_commitish:$commit,draft:true,assets:[ + {id:11,name:"release.yaml"},{id:12,name:"release.notags.yaml"},{id:13,name:"SHA256SUMS"}]} + | if $scenario == "published" or $scenario == "published-missing-tag" then .draft = false + elif $scenario == "metadata-id" then .id = 124 + elif $scenario == "metadata-tag" then .tag_name = "v0.4.2" + elif $scenario == "metadata-commit" then .target_commitish = "main" + elif $scenario == "metadata-draft" then .draft = "true" + elif $scenario == "metadata-assets" then .assets = null + elif $scenario == "duplicate-asset" then .assets += [.assets[0]] + elif $scenario == "missing-asset" then .assets |= .[0:2] + elif $scenario == "unsafe-asset-id" then .assets[0].id = "../evil" + elif $scenario == "duplicate-asset-id" then .assets[1].id = 11 + else . end' + else + [[ $4 == -H && $5 == "Accept: application/octet-stream" ]] || exit 90 + case "$6" in + repos/tektoncd/tekton-kueue/releases/assets/11) file=release.yaml ;; + repos/tektoncd/tekton-kueue/releases/assets/12) file=release.notags.yaml ;; + repos/tektoncd/tekton-kueue/releases/assets/13) file=SHA256SUMS ;; + *) exit 90 ;; + esac + if [[ $file == SHA256SUMS ]]; then + case "$SCENARIO" in + checksum) printf '%064d release.yaml\n' 0; exit ;; + traversal) printf '%s ../release.yaml\n' "$DIGEST"; exit ;; + absolute) printf '%s /tmp/release.yaml\n' "$DIGEST"; exit ;; + extra-checksum) cat "$FIXTURES/$file"; printf '%s extra.yaml\n' "$DIGEST"; exit ;; + duplicate-checksum) cat "$FIXTURES/$file" "$FIXTURES/$file"; exit ;; + missing-checksum) awk 'NR == 1' "$FIXTURES/$file"; exit ;; + empty-checksum) exit ;; + esac + fi + cat "$FIXTURES/$file" + fi + ;; + curl) + [[ $# == 7 && $1 == --fail && $2 == --silent && $3 == --show-error && + $4 == --location && $6 == --output ]] || exit 90 + prefix="https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$VERSION/" + [[ $5 == "$prefix"* ]] || exit 90 + file=${5#"$prefix"} + case "$file" in release.yaml|release.notags.yaml|SHA256SUMS) ;; *) exit 90 ;; esac + cat "$FIXTURES/$file" > "$7" + [[ $SCENARIO != "bucket-$file" ]] || printf '\nchanged\n' >> "$7" + ;; + crane) + if [[ $1 == manifest && $2 == "$IMAGE" && $# == 2 ]]; then + jq -n --arg scenario "$SCENARIO" ' + ["amd64","arm64","ppc64le","s390x"] + | if $scenario == "missing-platform" then .[0:3] + elif $scenario == "duplicate-platform" then . + ["amd64"] + elif $scenario == "wrong-platform" then .[3] = "riscv64" + else . end + | {schemaVersion:2, manifests:map({digest:("sha256:" + ("b" * 64)), + platform:{os:"linux",architecture:.}})}' + elif [[ $1 == config && $2 == --platform && $4 == "$IMAGE" && $# == 4 ]]; then + jq -n --arg platform "$3" --arg scenario "$SCENARIO" ' + {os:"linux",architecture:($platform | split("/")[1]),config:{Labels:{ + "org.opencontainers.image.source":"https://github.com/tektoncd/tekton-kueue", + "org.opencontainers.image.url":"https://github.com/tektoncd/tekton-kueue"}}} + | if $platform == "linux/s390x" then + if $scenario == "bad-source-label" then .config.Labels["org.opencontainers.image.source"] = "https://example.com" + elif $scenario == "bad-url-label" then del(.config.Labels["org.opencontainers.image.url"]) + elif $scenario == "bad-config-platform" then .os = "windows" + else . end + else . end' + else exit 90; fi + ;; + cosign) + [[ $2 == --key && -s $3 ]] || exit 90 + case "$1" in + verify) + [[ $# == 4 && $4 == "$IMAGE" ]] || exit 90 + [[ $SCENARIO != signature-failure ]] || exit 1 + [[ $SCENARIO != empty-signature ]] || exit + printf '[{"critical":{"image":{"docker-manifest-digest":"sha256:%s"}}}]\n' "$DIGEST" + ;; + verify-attestation) + [[ $# == 6 && $4 == --type && $5 == https://slsa.dev/provenance/v1 && $6 == "$IMAGE" ]] || exit 90 + [[ $SCENARIO != attestation-failure ]] || exit 1 + [[ $SCENARIO != empty-attestation ]] || exit + if [[ $SCENARIO == malformed-attestation ]]; then + printf '{"payload":"invalid base64"}\n' + exit + fi + jq -n --arg digest "$DIGEST" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {_type:"https://in-toto.io/Statement/v1",predicateType:"https://slsa.dev/provenance/v1", + subject:[{name:"ghcr.io/tektoncd/tekton-kueue",digest:{sha256:$digest}}], + predicate:{buildDefinition:{resolvedDependencies:[ + {uri:"git+https://github.com/tektoncd/tekton-kueue.git@refs/tags/v0.4.3",digest:{sha1:$commit}}]}}} + | if $scenario == "bad-source" then .predicate.buildDefinition.resolvedDependencies[0].uri = "https://github.com/tektoncd/tekton-kueue-evil" + elif $scenario == "bad-commit" then .predicate.buildDefinition.resolvedDependencies[0].digest.sha1 = ("f" * 40) + elif $scenario == "bad-digest" then .subject[0].digest.sha256 = ("b" * 64) + elif $scenario == "bad-predicate" then .predicateType = "https://slsa.dev/provenance/v0.2" + elif $scenario == "build-config-source" then + .predicate.buildDefinition.externalParameters.buildConfigSource = + {repository:"git+https://github.com/tektoncd/tekton-kueue",ref:("sha1:" + $commit),path:"tekton/release-pipeline.yaml"} + | del(.predicate.buildDefinition.resolvedDependencies) + else . end + | if $scenario == "multiple-attestations" then + (.subject[0].digest.sha256 = ("b" * 64)), . + else . end + | {payloadType:"application/vnd.in-toto+json",payload:(tojson | @base64),signatures:[{sig:"verified-by-fake-cosign"}]} + | if $scenario == "array-attestations" then [.] else . end' + ;; + *) exit 90 ;; + esac + ;; + *) exit 90 ;; +esac +EOF +chmod +x "$work/bin/fake" +for executable in gh curl crane cosign; do + ln -s fake "$work/bin/$executable" +done + +run_verifier() { + bash "$root/hack/verify-release-artifacts.sh" \ + "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +} + +passed=0 +for SCENARIO in success published build-config-source multiple-attestations array-attestations \ + annotated-tag missing-tag wrong-tag wrong-annotated-tag published-missing-tag tag-api-failure \ + checksum traversal absolute extra-checksum duplicate-checksum missing-checksum empty-checksum \ + bucket-release.yaml bucket-release.notags.yaml bucket-SHA256SUMS \ + malformed-metadata metadata-id metadata-tag metadata-commit metadata-draft metadata-assets \ + duplicate-asset missing-asset unsafe-asset-id duplicate-asset-id \ + wrong-image-repository inconsistent-image missing-webhook wrong-deployment tag-in-notags \ + missing-platform duplicate-platform wrong-platform bad-source-label bad-url-label bad-config-platform \ + signature-failure empty-signature attestation-failure empty-attestation malformed-attestation \ + bad-source bad-commit bad-digest bad-predicate; do + export SCENARIO + rm -rf "$work/output" + write_artifacts + status=0 + run_verifier > "$work/log" 2>&1 || status=$? + case "$SCENARIO" in + success|published|build-config-source|multiple-attestations|array-attestations|annotated-tag|missing-tag) + if [[ $status != 0 ]]; then cat "$work/log" >&2; exit 1; fi + for file in release.yaml release.notags.yaml SHA256SUMS; do + cmp "$FIXTURES/$file" "$work/output/$file" + done + jq -e --arg commit "$COMMIT" --arg image "$IMAGE" --arg scenario "$SCENARIO" \ + --arg version "$VERSION" --arg digest "sha256:$DIGEST" ' + . == {release_id:123,version:$version,commit:$commit,image:$image,digest:$digest, + draft:($scenario != "published"),assets:[ + {name:"SHA256SUMS",id:13},{name:"release.notags.yaml",id:12}, + {name:"release.yaml",id:11}]}' "$work/output/release-info.json" >/dev/null + jq -e --arg digest "$DIGEST" ' + [.[] | @base64d | fromjson + | select(any(.subject[]; .digest.sha256 == $digest))] + | length > 0' "$work/output/verified-payloads.json" >/dev/null + ;; + *) + [[ $status != 0 && ! -e $work/output/release-info.json ]] || { + echo "unexpected verification success: $SCENARIO" >&2; exit 1; + } + ;; + esac + passed=$((passed + 1)) +done + +export SCENARIO=success +expect_failure() { + if "$@" > "$work/log" 2>&1; then + echo "unexpected boundary acceptance" >&2 + exit 1 + fi + passed=$((passed + 1)) +} +verifier="$root/hack/verify-release-artifacts.sh" +rm -rf "$work/output" +expect_failure bash "$verifier" +expect_failure bash "$verifier" ../v0.4.3 "$COMMIT" 123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "${COMMIT^^}" 123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" ../123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/missing.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "" +expect_failure env GH_TOKEN= bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +expect_failure env GITHUB_REPOSITORY=../repo bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +touch "$work/empty.pub" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/empty.pub" "$work/output" +printf 'not a public key\n' > "$work/invalid.pub" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/invalid.pub" "$work/output" +mkdir "$work/output" +printf 'existing artifact\n' > "$work/output/release.yaml" +expect_failure run_verifier +[[ $(< "$work/output/release.yaml") == "existing artifact" ]] +echo "PASS $passed release artifact verification cases" diff --git a/hack/verify-release-config.sh b/hack/verify-release-config.sh index 9513a9676..0aad70ffa 100644 --- a/hack/verify-release-config.sh +++ b/hack/verify-release-config.sh @@ -25,12 +25,33 @@ test -n "$go_image" grep -Fq "apk add --no-cache bash make" "$root/tekton/publish.yaml" grep -Fq 'org.opencontainers.image.source=https://$PACKAGE' "$root/tekton/publish.yaml" grep -Fq 'org.opencontainers.image.url=https://$PACKAGE' "$root/tekton/publish.yaml" +if grep -Fq '/latest/' "$root/tekton/release-pipeline.yaml" || + grep -Eq 'crane (cp|copy|tag).*latest' "$root/tekton/publish.yaml" || + grep -Fq '$(params.releaseAsLatest)' "$root/tekton/release-pipeline.yaml"; then + echo "release staging must not promote latest" >&2 + exit 1 +fi +for caller in "$root/.tekton/release.yaml" "$root/.tekton/release-patch.yaml"; do + if grep -Fq 'name: releaseAsLatest' "$caller"; then + echo "release entry points must not accept latest promotion" >&2 + exit 1 + fi +done "$container_tool" run --rm --platform "$platform" \ -v "$root:/workspace/source" \ "$publish_image" sh -ec ' apk add --no-cache bash make >/dev/null cd /workspace/source + # Exercise the shipped guard, not a copy of its condition. + awk '\'' + $0 == " - name: publish-image-and-manifests" { step = 1 } + step && $0 == " script: |" { script = 1; next } + script { sub(/^ /, ""); print } + '\'' tekton/publish.yaml > /tmp/publish.sh + status=0 + VERSION_TAG=v0.0.0 RELEASE_AS_LATEST=true sh /tmp/publish.sh || status=$? + test "$status" -eq 64 make release \ KUSTOMIZE=/go/bin/kustomize \ RELEASE_DIR=/tmp/release \ diff --git a/hack/verify-release-tag.sh b/hack/verify-release-tag.sh new file mode 100644 index 000000000..1aa05b686 --- /dev/null +++ b/hack/verify-release-tag.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release tag verification failed: $*" >&2; exit 1; } +[[ $# == 3 ]] || fail "usage: $0 VERSION COMMIT ALLOW_MISSING" +version=$1 +commit=$2 +allow_missing=$3 +repository=${GITHUB_REPOSITORY:-tektoncd/tekton-kueue} +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ && $commit =~ ^[0-9a-f]{40}$ && + $repository =~ ^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$ && + ( $allow_missing == true || $allow_missing == false ) ]] || fail "invalid candidate" +object=$(gh api --paginate --slurp "repos/$repository/git/matching-refs/tags/$version" | + jq -c --arg ref "refs/tags/$version" ' + [.[][] | select(.ref == $ref)] + | if length <= 1 then .[0].object // null else error("ambiguous tag") end') +if [[ $object == null ]]; then + [[ $allow_missing == true ]] || fail "published release tag is missing" + exit 0 +fi +while :; do + sha=$(jq -r .sha <<< "$object") + [[ $sha =~ ^[0-9a-f]{40}$ ]] || fail "invalid tag object" + case $(jq -r .type <<< "$object") in + commit) + [[ $sha == "$commit" ]] || fail "tag points to another source" + exit 0 + ;; + tag) + object=$(gh api "repos/$repository/git/tags/$sha" | jq -ce '.object') + ;; + *) fail "tag does not resolve to a commit" ;; + esac +done diff --git a/tekton/publish.yaml b/tekton/publish.yaml index bc4413fae..e339d0644 100644 --- a/tekton/publish.yaml +++ b/tekton/publish.yaml @@ -31,7 +31,8 @@ spec: - name: imageRegistryUser default: tekton-robot - name: releaseAsLatest - default: "true" + description: Must be false; latest promotion belongs to release finalization. + default: "false" - name: platforms default: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountPath @@ -90,17 +91,17 @@ spec: #!/usr/bin/env sh set -eu - apk add --no-cache bash make - printf '%s\n' "$VERSION_TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' || { echo "invalid release version: $VERSION_TAG" >&2 exit 1 } case "$RELEASE_AS_LATEST" in - true|false) ;; - *) echo "releaseAsLatest must be true or false" >&2; exit 1 ;; + false) ;; + *) echo "latest promotion requires the release finalization workflow" >&2; exit 64 ;; esac + apk add --no-cache bash make + OUTPUT_RELEASE_DIR="$OUTPUT_ROOT/$VERSION_TAG" git config --global --add safe.directory "$PROJECT_ROOT" mkdir -p "$HOME/.docker" "$OUTPUT_RELEASE_DIR/template" @@ -139,15 +140,3 @@ spec: *) echo "unexpected release image: $image" >&2; exit 1 ;; esac printf '%s,' "$image" > "$(results.IMAGES.path)" - - - name: tag-latest - image: cgr.dev/chainguard/crane:latest-dev@sha256:dd9a345764e47517b80391648020b78c0ae6d4db0e392c7013b71e3817ed544c - script: | - #!/bin/sh - set -eu - - [ "$RELEASE_AS_LATEST" = "true" ] || exit 0 - mkdir -p "$HOME/.docker" - cp /workspace/docker-config.json "$HOME/.docker/config.json" - image=$(cat /workspace/built-images) - crane cp "$image" "${image%@*}:latest" diff --git a/tekton/release-cheat-sheet.md b/tekton/release-cheat-sheet.md index b6f6e8322..da31d8e9d 100644 --- a/tekton/release-cheat-sheet.md +++ b/tekton/release-cheat-sheet.md @@ -9,7 +9,7 @@ The automated steps below become operational only after the release Pipeline tra 1. Confirm the target version, LTS designation, end-of-life date, and compatibility ranges with the maintainers. 2. Confirm all intended changes are merged and their release-note blocks are complete. 3. From a clean checkout of the candidate commit, run generation, lint, unit, and end-to-end checks. -4. Run a release candidate with `releaseAsLatest=false` through the Tekton-controlled release Pipeline so it cannot update the `latest` image tag or artifacts. +4. Stage the release candidate through the Tekton-controlled release Pipeline. It publishes only versioned images and artifacts; it cannot update `latest`. 5. Verify both generated manifests reference only `ghcr.io/tektoncd/tekton-kueue` images by immutable digest. 6. Install and upgrade the candidate on the supported Kubernetes, Tekton Pipelines, and Kueue versions. @@ -23,7 +23,7 @@ The automated steps below become operational only after the release Pipeline tra ## Publish a patch release 1. Cherry-pick an approved fix to the supported release branch. -2. After the cherry-pick passes CI on the release branch, run the **Patch Release** GitHub workflow with the branch, next `vX.Y.Z` version, and whether it should update `latest`. +2. After the cherry-pick passes CI on the release branch, run the **Patch Release** GitHub workflow with the branch and next `vX.Y.Z` version. This stages a draft and does not update `latest`. 3. Monitor and verify the release as for an initial release. The scheduled patch workflow scans `release-v0.3.x` and newer for unreleased commits. Older release lines require manual dispatch. A release operator must still verify every resulting release before announcing it. @@ -41,6 +41,11 @@ Verify and record: Do not announce the release until every promised artifact is downloadable and independently verifiable. +Staging does not authorize publication or promotion. Those actions require +maintainer approval and verification of the finished release. Automated +approval-gated finalization is a separate rollout; staging alone leaves the +GitHub release as a draft and preserves the existing `latest` artifacts. + ## After the release 1. Update [`releases.md`](../releases.md) with the release date, exact end-of-life date, compatibility ranges, and release links. diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index 9518f7034..3e0ca03d1 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -27,9 +27,6 @@ spec: - name: releaseBucket description: bucket where the release is stored default: tekton-releases - - name: releaseAsLatest - description: Whether to tag and publish this release as latest - default: "false" - name: platforms description: Linux platforms to publish default: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le @@ -163,7 +160,7 @@ spec: - name: imageRegistryUser value: $(params.imageRegistryUser) - name: releaseAsLatest - value: $(params.releaseAsLatest) + value: "false" - name: serviceAccountPath value: $(params.serviceAccountImagesPath) - name: platforms @@ -208,41 +205,6 @@ spec: - name: recursive value: "true" - - name: publish-to-bucket-latest - runAfter: [publish-images] - when: - - input: "$(params.releaseAsLatest)" - operator: in - values: ["true"] - taskRef: - resolver: bundles - params: - - name: bundle - value: ghcr.io/tektoncd/catalog/upstream/tasks/oracle-cloud-storage-upload@sha256:f1dd28da10dfb72ebbb3a5258e9f1ba6bfe82ee587311d708a7be3207f69e5e6 - - name: name - value: oracle-cloud-storage-upload - - name: kind - value: task - workspaces: - - name: credentials - workspace: release-secret - - name: source - workspace: workarea - subPath: bucket - params: - - name: path - value: $(params.versionTag) - - name: bucketName - value: $(params.releaseBucket) - - name: objectPrefix - value: $(params.repoName)/latest/ - - name: replaceExistingFiles - value: "true" - - name: recursive - value: "true" - - name: deleteExtraFiles - value: "true" - - name: report-bucket runAfter: [publish-to-bucket] params: @@ -365,7 +327,7 @@ spec: for uuid in ${CANDIDATES}; do KIND=$(curl -fsS "${REKOR_HOST}/api/v1/log/entries/${uuid}" \ | jq -r '.[].body' | base64 -d 2>/dev/null | jq -r '.kind' 2>/dev/null || true) - if [ "${KIND}" = "intoto" ]; then + if [ "${KIND}" = "intoto" ] || [ "${KIND}" = "dsse" ]; then REKOR_UUID="${uuid}" break fi @@ -373,19 +335,8 @@ spec: fi if [ -z "${REKOR_UUID}" ]; then - echo "WARNING: Could not find an in-toto attestation entry; falling back to the transparency annotation" - TRANSPARENCY=$(kubectl get taskrun "${TASKRUN_NAME}" -n "${NAMESPACE}" \ - -o jsonpath='{.metadata.annotations.chains\.tekton\.dev/transparency}' 2>/dev/null || echo "") - if [ -n "${TRANSPARENCY}" ]; then - REKOR_UUID=$(curl -fsS "${TRANSPARENCY}" | jq -r 'keys[0] // empty' || true) - if [ -z "${REKOR_UUID}" ]; then - REKOR_UUID=$(echo "${TRANSPARENCY}" | grep -oE 'logIndex=[0-9]+' | cut -d= -f2) - fi - fi - fi - - if [ -z "${REKOR_UUID}" ]; then - REKOR_UUID="unknown" + echo "ERROR: Could not find an intoto or dsse attestation entry for ${IMAGE_DIGEST}" + exit 1 fi echo "Rekor UUID: ${REKOR_UUID}" printf '%s' "${REKOR_UUID}" > "$(results.rekor-uuid.path)" @@ -491,7 +442,7 @@ spec: - name: url value: https://github.com/tektoncd/plumbing - name: revision - value: 894710e292fcb6b17276af01a2bc318107eef8a7 + value: df071e78c5b1a58867435efc7a9b295688d58969 - name: pathInRepo value: tekton/resources/release/base/github_release_oci.yaml params: diff --git a/test/release-smoke/README.md b/test/release-smoke/README.md new file mode 100644 index 000000000..13d5a9b6b --- /dev/null +++ b/test/release-smoke/README.md @@ -0,0 +1,23 @@ +# Finished-release smoke test + +Run the published artifact verifier first, using an independently configured +signing public key. Pass its saved manifest and immutable image reference to: + +```sh +bash hack/smoke-release.sh VERIFIED_MANIFEST EXPECTED_IMAGE ARTIFACT_DIR +``` + +The script requires Bash, Kind, kubectl, jq, OpenSSL and a working container +provider. It creates its own cluster and kubeconfig, installs the manifest +without rebuilding its image, checks webhook mutation and Kueue admission, +and waits for a PipelineRun to succeed. It saves diagnostics and deletes only +the cluster it created, including on failure. The artifact directory must be +empty. + +The pull-request workflow exercises the existing public v0.4.3 manifest, +pinned by SHA-256 and image digest, on an isolated runner. This installation +check is not a signing-key trust bootstrap. The finalization workflow will +call the same script with the candidate that passed signature verification. + +`release.yaml` in this directory is an offline parsing fixture for +`bash hack/smoke-release_test.sh`; it is not an installable release. diff --git a/test/release-smoke/release.yaml b/test/release-smoke/release.yaml new file mode 100644 index 000000000..f45caa1db --- /dev/null +++ b/test/release-smoke/release.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tekton-kueue +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tekton-kueue-controller-manager + namespace: tekton-kueue +spec: + selector: + matchLabels: + app: controller + template: + metadata: + labels: + app: controller + spec: + containers: + - name: manager + image: ghcr.io/tektoncd/tekton-kueue:v0.4.3@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tekton-kueue-webhook + namespace: tekton-kueue +spec: + selector: + matchLabels: + app: webhook + template: + metadata: + labels: + app: webhook + spec: + containers: + - name: webhook + image: ghcr.io/tektoncd/tekton-kueue@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa