diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0fd1474a..bb8b15fbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,9 +90,27 @@ jobs: - name: Run actionlint run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 + release-automation: + name: Release automation tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check release verification, smoke and finalization + run: | + bash -n hack/verify-release-artifacts.sh hack/verify-release-artifacts_test.sh \ + hack/smoke-release.sh hack/smoke-release_test.sh \ + hack/finalize-release.sh hack/finalize-release_test.sh \ + hack/recover-release.sh hack/recover-release_test.sh + bash hack/verify-release-artifacts_test.sh + bash hack/smoke-release_test.sh + bash hack/finalize-release_test.sh + bash hack/recover-release_test.sh + ci-summary: name: CI summary - needs: [build, test, lint, e2e, dependency-review, workflow-lint] + needs: [build, test, lint, e2e, dependency-review, workflow-lint, release-automation] runs-on: ubuntu-latest if: always() steps: @@ -104,6 +122,7 @@ jobs: E2E_RESULT: ${{ needs.e2e.result }} DEPENDENCY_REVIEW_RESULT: ${{ needs.dependency-review.result }} WORKFLOW_LINT_RESULT: ${{ needs.workflow-lint.result }} + RELEASE_AUTOMATION_RESULT: ${{ needs.release-automation.result }} run: | results=( "build=${BUILD_RESULT}" @@ -112,6 +131,7 @@ jobs: "e2e=${E2E_RESULT}" "dependency-review=${DEPENDENCY_REVIEW_RESULT}" "workflow-lint=${WORKFLOW_LINT_RESULT}" + "release-automation=${RELEASE_AUTOMATION_RESULT}" ) failed=0 for result in "${results[@]}"; do diff --git a/.github/workflows/finalize-release.yaml b/.github/workflows/finalize-release.yaml new file mode 100644 index 000000000..2edb71784 --- /dev/null +++ b/.github/workflows/finalize-release.yaml @@ -0,0 +1,118 @@ +name: Finalize Release + +"on": + workflow_dispatch: + inputs: + operation: + description: Publish a verified candidate or repair only its attestation notes + required: true + type: choice + options: [finalize, repair] + version: + description: Candidate version, for example v0.4.4 + required: true + type: string + source_commit: + description: Full source commit approved for this release + required: true + type: string + release_id: + description: Existing GitHub release ID + required: true + type: string + image_digest: + description: Exact approved sha256 image digest + required: true + type: string + promote_latest: + description: Also promote the verified image and manifests to latest + required: false + default: false + type: boolean + +permissions: + contents: read + +concurrency: + group: release-finalization + cancel-in-progress: false + +jobs: + finalize: + if: github.repository == 'tektoncd/tekton-kueue' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: + name: kueue-release-finalization + env: + GOTOOLCHAIN: local + GH_TOKEN: ${{ secrets.CHATOPS_TOKEN }} + GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} + OCI_API_KEY: ${{ secrets.OCI_API_KEY }} + OCI_FINGERPRINT: ${{ secrets.OCI_FINGERPRINT }} + OCI_TENANCY_OCID: ${{ secrets.OCI_TENANCY_OCID }} + OCI_USER_OCID: ${{ secrets.OCI_USER_OCID }} + OCI_REGION: ${{ secrets.OCI_REGION }} + RELEASE_SIGNING_PUBLIC_KEY: ${{ vars.RELEASE_SIGNING_PUBLIC_KEY }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Require configured signing trust + shell: bash + run: | + test -n "$RELEASE_SIGNING_PUBLIC_KEY" + printf '%s\n' "$RELEASE_SIGNING_PUBLIC_KEY" > "$RUNNER_TEMP/release-signing.pub" + openssl pkey -pubin -in "$RUNNER_TEMP/release-signing.pub" -noout + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + + - name: Install pinned verification tools + shell: bash + run: | + export GOBIN="$RUNNER_TEMP/release-tools" + mkdir -p "$GOBIN" + go install github.com/sigstore/cosign/v3/cmd/cosign@v3.0.6 + go install github.com/google/go-containerregistry/cmd/crane@v0.20.6 + go install sigs.k8s.io/kind@v0.32.0 + printf '%s\n' "$GOBIN" >> "$GITHUB_PATH" + + - name: Install pinned OCI client + if: inputs.promote_latest + shell: bash + run: | + python3 -m venv "$RUNNER_TEMP/oci-client" + "$RUNNER_TEMP/oci-client/bin/pip" install oci-cli==3.74.2 + printf '%s\n' "$RUNNER_TEMP/oci-client/bin" >> "$GITHUB_PATH" + + - name: Verify, smoke-test and finalize + shell: bash + env: + OPERATION: ${{ inputs.operation }} + VERSION: ${{ inputs.version }} + COMMIT: ${{ inputs.source_commit }} + RELEASE_ID: ${{ inputs.release_id }} + DIGEST: ${{ inputs.image_digest }} + PROMOTE_LATEST: ${{ inputs.promote_latest }} + run: | + bash hack/finalize-release.sh "$OPERATION" "$VERSION" "$COMMIT" \ + "$RELEASE_ID" "$DIGEST" "$RUNNER_TEMP/release-signing.pub" \ + "$PROMOTE_LATEST" "$RUNNER_TEMP/release-finalization" + { + printf '### Release %s\n\n' "$VERSION" + printf -- '- Operation: %s\n' "$OPERATION" + printf -- '- Source: %s\n' "$COMMIT" + printf -- '- Image: %s\n' "$DIGEST" + printf -- '- Release: https://github.com/tektoncd/tekton-kueue/releases/tag/%s\n' "$VERSION" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Save isolated smoke diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-smoke-${{ github.run_id }} + path: ${{ runner.temp }}/release-finalization/smoke + if-no-files-found: ignore diff --git a/.github/workflows/patch-release.yaml b/.github/workflows/patch-release.yaml index 2f7ae9923..561ed2de9 100644 --- a/.github/workflows/patch-release.yaml +++ b/.github/workflows/patch-release.yaml @@ -11,11 +11,6 @@ name: Patch Release description: "Version to release (for example, v0.5.1)" required: true type: string - release_as_latest: - description: "Publish as latest release" - required: false - type: boolean - default: true schedule: - cron: "0 10 * * 4" @@ -47,23 +42,6 @@ jobs: id: scan shell: bash run: | - latest_branch="" - latest_major=0 - latest_minor=0 - for ref in $(git branch -r --list 'origin/release-v*'); do - branch="${ref#origin/}" - if [[ "$branch" =~ ^release-v([0-9]+)\.([0-9]+)\.x$ ]]; then - major="${BASH_REMATCH[1]}" - minor="${BASH_REMATCH[2]}" - if [ "$major" -gt "$latest_major" ] || { [ "$major" -eq "$latest_major" ] && [ "$minor" -gt "$latest_minor" ]; }; then - latest_major=$major - latest_minor=$minor - latest_branch=$branch - fi - fi - done - echo "::notice::Latest release branch: ${latest_branch}" - min_major="${MIN_RELEASE_VERSION%%.*}" min_minor="${MIN_RELEASE_VERSION##*.}" releases=() @@ -93,9 +71,7 @@ jobs: fi next_version=$(echo "$last_tag" | awk -F. '{printf "%s.%s.%d", $1, $2, $3+1}') - is_latest=false - [ "$branch" = "$latest_branch" ] && is_latest=true - releases+=("{\"branch\":\"${branch}\",\"version\":\"${next_version}\",\"release_as_latest\":\"${is_latest}\"}") + releases+=("{\"branch\":\"${branch}\",\"version\":\"${next_version}\"}") done if [ ${#releases[@]} -eq 0 ]; then @@ -133,7 +109,6 @@ jobs: PAC_INCOMING_SECRET: ${{ secrets.PAC_INCOMING_SECRET }} RELEASE_BRANCH: ${{ matrix.release.branch }} RELEASE_VERSION: ${{ matrix.release.version }} - RELEASE_AS_LATEST: ${{ matrix.release.release_as_latest }} run: bash ./hack/trigger-patch-release.sh trigger-manual-release: @@ -184,5 +159,4 @@ jobs: PAC_INCOMING_SECRET: ${{ secrets.PAC_INCOMING_SECRET }} RELEASE_BRANCH: ${{ inputs.branch }} RELEASE_VERSION: ${{ inputs.version }} - RELEASE_AS_LATEST: ${{ inputs.release_as_latest }} run: bash ./hack/trigger-patch-release.sh diff --git a/.github/workflows/recover-release.yaml b/.github/workflows/recover-release.yaml new file mode 100644 index 000000000..4f3a1aef9 --- /dev/null +++ b/.github/workflows/recover-release.yaml @@ -0,0 +1,65 @@ +name: Recover Release Draft + +"on": + workflow_dispatch: + inputs: + version: + description: Existing versioned candidate, for example v0.4.4 + required: true + type: string + source_commit: + description: Full approved source commit + required: true + type: string + image_digest: + description: Exact approved sha256 image digest + required: true + type: string + +permissions: + contents: read + +concurrency: + group: release-finalization + cancel-in-progress: false + +jobs: + recover: + if: github.repository == 'tektoncd/tekton-kueue' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: + name: kueue-release-finalization + env: + GH_TOKEN: ${{ secrets.CHATOPS_TOKEN }} + RELEASE_SIGNING_PUBLIC_KEY: ${{ vars.RELEASE_SIGNING_PUBLIC_KEY }} + GOTOOLCHAIN: local + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Require configured signing trust + run: | + test -n "$RELEASE_SIGNING_PUBLIC_KEY" + printf '%s\n' "$RELEASE_SIGNING_PUBLIC_KEY" > "$RUNNER_TEMP/release-signing.pub" + openssl pkey -pubin -in "$RUNNER_TEMP/release-signing.pub" -noout + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Install pinned verification tools + run: | + export GOBIN="$RUNNER_TEMP/release-tools" + mkdir -p "$GOBIN" + go install github.com/sigstore/cosign/v3/cmd/cosign@v3.0.6 + go install github.com/google/go-containerregistry/cmd/crane@v0.20.6 + printf '%s\n' "$GOBIN" >> "$GITHUB_PATH" + - name: Verify and resume the existing draft + env: + VERSION: ${{ inputs.version }} + COMMIT: ${{ inputs.source_commit }} + DIGEST: ${{ inputs.image_digest }} + run: | + bash hack/recover-release.sh "$VERSION" "$COMMIT" "$DIGEST" \ + "$RUNNER_TEMP/release-signing.pub" "$RUNNER_TEMP/release-recovery" + printf 'Recovered verified draft for %s. Run Finalize Release separately after approval.\n' \ + "$VERSION" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release-smoke.yaml b/.github/workflows/release-smoke.yaml new file mode 100644 index 000000000..1b98f10a1 --- /dev/null +++ b/.github/workflows/release-smoke.yaml @@ -0,0 +1,56 @@ +name: Released Artifact Smoke + +"on": + pull_request: + paths: + - ".github/workflows/release-smoke.yaml" + - ".github/workflows/finalize-release.yaml" + - "hack/*release*.sh" + - "tekton/publish.yaml" + - "tekton/release-pipeline.yaml" + - "test/release-smoke/**" + +permissions: + contents: read + +concurrency: + group: release-smoke-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + smoke: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + - name: Install pinned Kind + run: | + GOTOOLCHAIN=local go install sigs.k8s.io/kind@v0.32.0 + - name: Check smoke script behavior + run: | + bash -n hack/smoke-release.sh hack/smoke-release_test.sh + bash hack/smoke-release_test.sh + - name: Install and exercise the immutable release fixture + shell: bash + run: | + # A pinned public fixture tests installation, not the production trust bootstrap. + curl -fsSL https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/v0.4.3/release.yaml \ + -o "$RUNNER_TEMP/release.yaml" + printf '%s %s\n' \ + 340e21dbed798284de3e665e816e02f29f024af0cd32efa3cc98bbb7f9830270 \ + "$RUNNER_TEMP/release.yaml" | sha256sum -c - + bash hack/smoke-release.sh "$RUNNER_TEMP/release.yaml" \ + ghcr.io/tektoncd/tekton-kueue@sha256:6d89cbd8b47cebebc1c05c9e45292433e755372db68e0b66d5dd68d6ede9cb17 \ + "$RUNNER_TEMP/release-smoke" + - name: Save smoke diagnostics + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-smoke-${{ github.run_id }} + path: ${{ runner.temp }}/release-smoke + if-no-files-found: ignore diff --git a/.github/workflows/release-verification.yaml b/.github/workflows/release-verification.yaml new file mode 100644 index 000000000..0d7b03d54 --- /dev/null +++ b/.github/workflows/release-verification.yaml @@ -0,0 +1,23 @@ +name: Release artifact verification + +on: + pull_request: + paths: + - hack/verify-release-artifacts.sh + - hack/verify-release-artifacts_test.sh + - .github/workflows/release-verification.yaml + +permissions: + contents: read + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Check verifier syntax and behavior + run: | + bash -n hack/verify-release-artifacts.sh hack/verify-release-artifacts_test.sh + bash hack/verify-release-artifacts_test.sh diff --git a/.tekton/release-patch.yaml b/.tekton/release-patch.yaml index e67af9ea1..ccfde1333 100644 --- a/.tekton/release-patch.yaml +++ b/.tekton/release-patch.yaml @@ -43,8 +43,6 @@ spec: value: tekton-robot - name: releaseBucket value: tekton-releases - - name: releaseAsLatest - value: "{{ release_as_latest }}" - name: platforms value: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountImagesPath diff --git a/.tekton/release.yaml b/.tekton/release.yaml index f6337bc36..310c69208 100644 --- a/.tekton/release.yaml +++ b/.tekton/release.yaml @@ -46,8 +46,6 @@ spec: value: tekton-robot - name: releaseBucket value: tekton-releases - - name: releaseAsLatest - value: "true" - name: platforms value: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountImagesPath diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 000000000..cdf570641 --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,151 @@ +# Release staging and finalization + +Releases have two separate stages. The release pipeline builds and signs a +versioned candidate, uploads its manifests under `previous//`, and +creates a GitHub draft. It does not update the image's `latest` tag or the +bucket's `latest/` manifests. + +The **Finalize Release** workflow verifies that candidate, installs its exact +published image in a disposable cluster, and then publishes it. Updating +`latest` is an explicit choice in this workflow, after maintainer approval. +It never builds another image. + +## Required environment + +Provision `kueue-release-finalization` using the standalone module +in `tektoncd/plumbing` under `terraform/kueue-release-finalization/`. The environment +requires review by `tekton-kueue-maintainers`, disallows self-review and +administrator bypass, and allows only the protected workflow on `main`. +Merging the workflow alone does not configure these protections. The +finalizer also checks the environment's actual protection before mutations. + +Set the environment variable `RELEASE_SIGNING_PUBLIC_KEY` through the required +Terraform signing-key input. Obtain the key from independently managed +release infrastructure and confirm its fingerprint with the release +maintainers. Do not bootstrap trust from the candidate's signature or Rekor +entry. The historical `tektoncd/chains/tekton.pub` does not match the key used +by the OCI `v0.4.3` release; it is not a fallback. + +The workflow uses these existing credential names. Environment-scoped secrets +can override organization secrets where the existing credential is unsuitable: + +| Name | Required access | +| --- | --- | +| `CHATOPS_TOKEN` | `tekton-robot` GitHub credential with repository contents write access, including draft releases | +| `GHCR_TOKEN` | `tekton-robot` package credential that can promote this repository's image | +| `OCI_API_KEY`, `OCI_FINGERPRINT`, `OCI_TENANCY_OCID`, `OCI_USER_OCID`, `OCI_REGION` | Existing release-storage identity, authorized to read the namespace and write `tekton-kueue/latest/` | + +The finalizer verifies token ownership through GitHub, rather than assuming a +secret name proves its owner. Missing or incorrect credentials stop the run; +there is no personal-account fallback. Registry and storage credentials are +needed only when `promote_latest` is selected. Temporary OCI and Docker +credential files are removed on exit. + +For key rotation, independently authenticate the replacement key, update the +managed environment variable, and test an existing candidate before allowing +publication. Keep historical public keys available for verifying old releases; +do not silently trust whatever key a candidate supplies. + +## Approving a candidate + +Run **Finalize Release** from `main`. Supply: + +| Input | Meaning | +| --- | --- | +| `operation` | `finalize` to publish, or `repair` to correct only the attestation notes | +| `version` | The existing candidate's `vX.Y.Z` version | +| `source_commit` | Its full, approved source commit | +| `release_id` | The existing GitHub release's numeric ID | +| `image_digest` | The exact approved `sha256:` digest | +| `promote_latest` | Also update the image, bucket and GitHub latest release; default is false | + +Check the source commit and digest before approving the environment deployment. +The workflow runs only the trusted `main` scripts; it does not check out or +execute scripts supplied by the candidate branch. + +Before any promotion or publication, the finalizer: + +1. Downloads the three release assets by ID, checks both manifest hashes, and + compares the assets byte-for-byte with the versioned bucket objects. +2. Confirms that the controller and webhook reference the same expected image + digest, that all four supported Linux platforms exist, and that their OCI + repository labels are correct. +3. Verifies the image signature and SLSA v1 provenance against the configured + key, including the provenance's source commit and subject digest. +4. For `finalize`, installs the finished manifests with supported pinned + dependencies in a uniquely named Kind cluster. It checks controller and + webhook readiness, resource mutation, Kueue admission, and PipelineRun + completion. The cluster and its private kubeconfig are cleaned up on every + exit; diagnostics are retained as workflow artifacts. +5. Checks that the release's tag, source, draft state and asset IDs have not + changed since verification. +6. Corrects the attestation section using the Rekor entry whose payload hash + matches the verified statement. + +For `finalize`, optional latest promotion happens only after these gates. +The finalizer refuses to promote an older version over the current latest, +checks the resulting image digest and bucket contents, and publishes the +existing GitHub release with an explicit tag and source commit. + +## Repair and retries + +`repair` performs artifact and signature verification but does not install a +cluster, publish a draft, or promote image/bucket latest. It preserves the +release's publication state and GitHub latest selection while replacing only +the attestation section. `promote_latest` must be false for this operation. + +A failed verification never falls back to an image-signature entry, an +untrusted key, or another account. A promotion can fail after an approved +image or object has already moved; the GitHub release is not published until +all requested promotion checks pass. Rerun with the same approved candidate +to complete that promotion without rebuilding it. + +Do not delete versioned artifacts to work around a partial release. Preserve +them and their original source/signing evidence for recovery. + +## Recovering a partial draft + +Run **Recover Release Draft** on `main` with the approved version, full source +commit and image digest. It uses the same protected environment, independent +signing key and robot credential as finalization. It can create a missing draft +or attach missing files to an existing draft without rebuilding. + +Recovery requires a complete, signed versioned bucket bundle: `release.yaml`, +`release.notags.yaml` and `SHA256SUMS`. It verifies the bundle and provenance +before any draft mutation. Existing tags must resolve to the approved commit, +and existing assets must match the verified bytes. Conflicting or published +releases are rejected; matching assets and existing notes are preserved. +After attaching only missing files, it verifies the recovered draft against +the bucket again. Run **Finalize Release** separately to smoke-test and publish. + +An incomplete bucket bundle or missing signing evidence cannot be reconstructed +safely by this workflow. It stops without rebuilding or deleting anything; +preserve the state for an operator to recover the original artifacts. + +The release pipeline's authenticated preflight rejects any existing tag, +draft, image or versioned object and directs operators to recovery. Its +`github-token/GITHUB_TOKEN` namespace secret must belong to `tekton-robot` and +have repository push permission, so draft lookup cannot silently miss drafts. +Authentication, rate-limit and ambiguous registry/storage failures block staging. + +## Developer checks + +```sh +bash hack/verify-release-artifacts_test.sh +bash hack/smoke-release_test.sh +bash hack/finalize-release_test.sh +bash hack/recover-release_test.sh +``` + +These deterministic boundary tests do not mutate GitHub, the registry or +storage, and do not use the developer's current cluster. Real smoke QA uses +`hack/smoke-release.sh` with an already verified manifest and canonical image +digest; it creates and deletes only its own Kind cluster. The finalization +workflow pins its tool versions, including Kind, rather than relying on older +workstation tools. + +The Patch Release workflow no longer accepts `release_as_latest`, and the +release pipeline no longer accepts `releaseAsLatest`. Select +`promote_latest` during finalization instead. The image-publishing Task keeps +the legacy parameter only to accept `false` from older supported branches; +passing `true` is rejected. diff --git a/hack/finalize-release.sh b/hack/finalize-release.sh new file mode 100644 index 000000000..af8cf6a58 --- /dev/null +++ b/hack/finalize-release.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release finalization failed: $*" >&2; exit 1; } +robot() { + [[ $(gh api user --jq .login) == tekton-robot ]] || fail "GH_TOKEN must belong to tekton-robot" +} + +[[ $# == 8 ]] || + fail "usage: $0 finalize|repair VERSION COMMIT RELEASE_ID DIGEST PUBLIC_KEY_FILE true|false WORK_DIR" +operation=$1 +version=$2 +commit=$3 +release_id=$4 +digest=$5 +key=$6 +latest=$7 +work=$8 +repository=tektoncd/tekton-kueue +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +[[ $operation == finalize || $operation == repair ]] || fail "invalid operation" +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ && + $commit =~ ^[0-9a-f]{40}$ && $release_id =~ ^[1-9][0-9]*$ && + $digest =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid candidate identity" +[[ $latest == true || $latest == false ]] || fail "invalid latest selection" +[[ $operation != repair || $latest == false ]] || fail "repair cannot promote latest" +[[ ${GITHUB_REPOSITORY:-} == "$repository" && + ${GITHUB_REF:-} == refs/heads/main ]] || fail "run the trusted main-branch workflow" +[[ -n ${GH_TOKEN:-} && -f $key && -s $key ]] || fail "bot token and trusted public key are required" +[[ -n $work && ! -L $work ]] || fail "invalid work directory" +if [[ -e $work ]]; then + [[ -d $work && -z $(find "$work" -mindepth 1 -print -quit) ]] || + fail "work directory must be empty" +fi +mkdir -p "$work" +work=$(cd "$work" && pwd) +robot + +# An environment name in a workflow does not itself configure review protection. +gh api "repos/$repository/environments/kueue-release-finalization" > "$work/environment.json" +jq -e ' + .can_admins_bypass == false and + any(.protection_rules[]?; + .type == "required_reviewers" and .prevent_self_review == true and + any(.reviewers[]?; .type == "Team" and .reviewer.id == 18766447)) +' "$work/environment.json" >/dev/null || fail "release approval environment is not protected" + +current=$(gh api "repos/$repository/releases/latest" --jq .tag_name) +[[ $current =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "cannot determine current latest release" +publish_latest=$latest +[[ $current != "$version" ]] || publish_latest=true +if [[ $latest == true ]]; then + for name in GHCR_TOKEN OCI_API_KEY OCI_FINGERPRINT OCI_TENANCY_OCID OCI_USER_OCID OCI_REGION; do + [[ -n ${!name:-} ]] || fail "$name is required for latest promotion" + done + [[ $(GH_TOKEN="$GHCR_TOKEN" gh api user --jq .login) == tekton-robot ]] || + fail "GHCR_TOKEN must belong to tekton-robot" + IFS=. read -r major minor patch <<< "${version#v}" + IFS=. read -r old_major old_minor old_patch <<< "${current#v}" + (( 10#$major > 10#$old_major || + (10#$major == 10#$old_major && 10#$minor > 10#$old_minor) || + (10#$major == 10#$old_major && 10#$minor == 10#$old_minor && 10#$patch >= 10#$old_patch) )) || + fail "refusing to promote an older release as latest" +fi + +bash "$root/hack/verify-release-artifacts.sh" "$version" "$commit" "$release_id" "$key" "$work/verified" +info="$work/verified/release-info.json" +jq -e --arg digest "$digest" '.digest == $digest' "$info" >/dev/null || + fail "verified digest differs from the approved candidate" +image=$(jq -r .image "$info") +if [[ $operation == finalize ]]; then + bash "$root/hack/smoke-release.sh" "$work/verified/release.yaml" "$image" "$work/smoke" +fi + +# Bind the release-note UUID to bytes from an already verified statement. +jq -r '.[]' "$work/verified/verified-payloads.json" | + while IFS= read -r payload; do + hash=$(printf '%s' "$payload" | base64 -d | openssl dgst -sha256 -r) + printf '%s\n' "${hash%% *}" + done > "$work/payload-hashes" +curl -fsS -H 'Content-Type: application/json' \ + -d "{\"hash\":\"$digest\"}" https://rekor.sigstore.dev/api/v1/index/retrieve > "$work/rekor-index.json" +uuid="" +while IFS= read -r candidate; do + [[ $candidate =~ ^[0-9a-f]+$ ]] || fail "invalid Rekor entry ID" + curl -fsS "https://rekor.sigstore.dev/api/v1/log/entries/$candidate" | + jq -r '.[].body' | base64 -d > "$work/rekor-entry.json" + kind=$(jq -r .kind "$work/rekor-entry.json") + [[ $kind == dsse || $kind == intoto ]] || continue + hash=$(jq -r '.spec.payloadHash.value // .spec.content.payloadHash.value // empty' "$work/rekor-entry.json") + if [[ -n $hash ]] && grep -Fxq -- "$hash" "$work/payload-hashes"; then + uuid=$candidate + break + fi +done < <(jq -er '.[]' "$work/rekor-index.json") +[[ -n $uuid ]] || fail "no Rekor entry matches the verified provenance" + +gh api "repos/$repository/releases/$release_id" > "$work/current-release.json" +jq -e --arg version "$version" --arg commit "$commit" --slurpfile info "$info" ' + .id == $info[0].release_id and .tag_name == $version + and .target_commitish == $commit and .draft == $info[0].draft + and ([.assets[] + | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS") + | {name, id}] | sort_by(.name)) == $info[0].assets +' "$work/current-release.json" >/dev/null || fail "release changed after verification" + +cat > "$work/attestation.md" <\n" + elif $end == null or $end <= $start + then error("unsupported release-note structure") + else $body[0:$start] + $section + $body[$end:] end)} + | .make_latest = $latest +' "$work/current-release.json" > "$work/patch.json" + +bash "$root/hack/verify-release-tag.sh" "$version" "$commit" "$(jq -r .draft "$work/current-release.json")" + +if [[ $latest == true ]]; then + # Keep credentials out of the caller's Docker configuration and process args. + export DOCKER_CONFIG="$work/docker" + mkdir -p "$DOCKER_CONFIG" "$work/oci" + trap 'rm -rf "$work/oci" "$work/docker"' EXIT + chmod 700 "$work/oci" + printf '%s\n' "$OCI_API_KEY" > "$work/oci/key.pem" + chmod 600 "$work/oci/key.pem" + for value in "$OCI_USER_OCID" "$OCI_TENANCY_OCID" "$OCI_FINGERPRINT" "$OCI_REGION"; do + [[ $value != *$'\n'* && $value != *$'\r'* ]] || fail "invalid OCI configuration" + done + cat > "$work/oci/config" </dev/null + curl -fsSL "https://infra.tekton.dev/tekton-releases/tekton-kueue/latest/$file" \ + --output "$work/latest-$file" + cmp -s "$work/verified/$file" "$work/latest-$file" || fail "latest bucket verification failed: $file" + done +fi + +robot +gh api --method PATCH "repos/$repository/releases/$release_id" --input "$work/patch.json" > "$work/result.json" +jq -e --arg version "$version" --arg commit "$commit" --arg operation "$operation" \ + --slurpfile previous "$work/current-release.json" --slurpfile patch "$work/patch.json" ' + .tag_name == $version and .target_commitish == $commit and + .body == $patch[0].body and + .draft == (if $operation == "repair" then $previous[0].draft else false end) +' "$work/result.json" >/dev/null || fail "GitHub did not retain the requested release state" +bash "$root/hack/verify-release-tag.sh" "$version" "$commit" "$(jq -r .draft "$work/result.json")" +if [[ $publish_latest == true ]]; then + [[ $(gh api "repos/$repository/releases/latest" --jq .tag_name) == "$version" ]] || + fail "GitHub latest release does not match the finalized candidate" +fi +echo "PASS $operation https://github.com/$repository/releases/tag/$version" diff --git a/hack/finalize-release_test.sh b/hack/finalize-release_test.sh new file mode 100644 index 000000000..4d2c17a97 --- /dev/null +++ b/hack/finalize-release_test.sh @@ -0,0 +1,363 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Copies of this file provide only the separately tested verifier/smoke and +# network executable boundaries. The finalizer itself is copied without edits. +if [[ ${FINALIZER_TEST_CHILD:-} == true ]]; then + command=${0##*/} + event() { + jq -cn --arg event "$1" --arg token "${GH_TOKEN:-}" --args \ + '{event:$event,token:$token,args:$ARGS.positional}' -- "${@:2}" >> "$CALLS" + } + case "$command" in + verify-release-artifacts.sh) + event verify "$@" + [[ $# == 5 && $1 == "$VERSION" && $2 == "$COMMIT" && $3 == 123 && + $4 == "$CASE_DIR/key.pub" && $5 == "$CASE_DIR/work/verified" ]] + openssl pkey -pubin -in "$4" -noout + [[ $MODE != verifier-failure ]] || exit 42 + mkdir -p "$5" + cp "$CASE_DIR"/{release.yaml,release.notags.yaml,SHA256SUMS,verified-payloads.json} "$5/" + jq --arg image "$IMAGE" --arg digest "$DIGEST" --arg version "$VERSION" \ + --arg commit "$COMMIT" --arg mode "$MODE" ' + {release_id:.id,version:$version,commit:$commit,image:$image, + digest:(if $mode == "bad-digest" then "sha256:" + ("b"*64) else $digest end), + draft:.draft,assets:(.assets | sort_by(.name))} + ' "$CASE_DIR/original.json" > "$5/release-info.json" + ;; + smoke-release.sh) + event smoke "$@" + [[ $# == 3 && $1 == "$CASE_DIR/work/verified/release.yaml" && + $2 == "$IMAGE" && $3 == "$CASE_DIR/work/smoke" ]] + cmp "$1" "$CASE_DIR/release.yaml" + [[ $MODE != smoke-failure ]] || exit 42 + mkdir "$3" + cp "$1" "$3/release.yaml" + printf '%s\n' "$2" > "$3/expected-image.txt" + ;; + gh) + [[ $1 == api ]] + shift + if [[ $1 == user ]]; then + [[ $# == 3 && $2 == --jq && $3 == .login ]] + event identity "$@" + if [[ $MODE == wrong-actor ]]; then echo waveywaves; else echo tekton-robot; fi + elif [[ $1 == --paginate ]]; then + [[ $# == 3 && $2 == --slurp && + $3 == "repos/tektoncd/tekton-kueue/git/matching-refs/tags/$VERSION" ]] + event tag + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg mode "$MODE" ' + [[{ref:("refs/tags/" + $version),object:{type:"commit", + sha:(if $mode == "changed-tag" then "f"*40 else $commit end)}}]]' + elif [[ $1 == --method ]]; then + [[ $# == 5 && $2 == PATCH && + $3 == repos/tektoncd/tekton-kueue/releases/123 && $4 == --input ]] + event patch "$@" + [[ $GH_TOKEN == test-bot-token ]] + cp "$5" "$CASE_DIR/sent-patch.json" + [[ $MODE != api-failure ]] || exit 42 + jq -s '.[0] * (.[1] | del(.make_latest))' \ + "$CASE_DIR/release.json" "$5" > "$CASE_DIR/updated.json" + if [[ $MODE == wrong-result ]]; then + jq '.draft = true' "$CASE_DIR/updated.json" > "$CASE_DIR/release.json" + else + cp "$CASE_DIR/updated.json" "$CASE_DIR/release.json" + fi + if [[ $(jq -r .make_latest "$5") == true && $MODE != wrong-latest ]]; then + printf '%s\n' "$VERSION" > "$CASE_DIR/latest" + fi + cat "$CASE_DIR/release.json" + else + case "$1" in + repos/tektoncd/tekton-kueue/environments/kueue-release-finalization) + [[ $# == 1 ]] + event environment + cat "$CASE_DIR/environment.json" + ;; + repos/tektoncd/tekton-kueue/releases/latest) + [[ $# == 3 && $2 == --jq && $3 == .tag_name ]] + event latest + cat "$CASE_DIR/latest" + ;; + repos/tektoncd/tekton-kueue/releases/123) + [[ $# == 1 ]] + event release + cat "$CASE_DIR/release.json" + ;; + *) exit 90 ;; + esac + fi + ;; + curl) + if [[ $# == 6 && $1 == -fsS && $2 == -H && + $3 == 'Content-Type: application/json' && $4 == -d && + $6 == https://rekor.sigstore.dev/api/v1/index/retrieve ]]; then + jq -e --arg digest "$DIGEST" '.hash == $digest' <<< "$5" >/dev/null + event rekor-index + printf '["abcdef"]\n' + elif [[ $# == 2 && $1 == -fsS && + $2 == https://rekor.sigstore.dev/api/v1/log/entries/abcdef ]]; then + event rekor-entry + hash=$(openssl dgst -sha256 -r "$CASE_DIR/payload.json") + hash=${hash%% *} + [[ $MODE != rekor-mismatch ]] || hash=${hash//?/0} + jq -cn --arg hash "$hash" \ + '{abcdef:{body:({kind:"dsse",spec:{payloadHash:{value:$hash}}}|tojson|@base64)}}' + elif [[ $# == 4 && $1 == -fsSL && $3 == --output ]]; then + prefix=https://infra.tekton.dev/tekton-releases/tekton-kueue/latest/ + [[ $2 == "$prefix"* ]] + file=${2#"$prefix"} + case "$file" in release.yaml|release.notags.yaml|SHA256SUMS) ;; *) exit 90 ;; esac + event bucket-read "$file" + cp "$CASE_DIR/$file" "$4" + [[ $MODE != bucket-failure ]] || printf 'changed\n' >> "$4" + else exit 90; fi + ;; + crane) + [[ $DOCKER_CONFIG == "$CASE_DIR/work/docker" && -d $DOCKER_CONFIG ]] + case "$1" in + auth) + [[ $# == 6 && $2 == login && $3 == ghcr.io && + $4 == --username && $5 == tekton-robot && $6 == --password-stdin ]] + token=$(cat) + [[ $token == test-registry-token ]] + event login "$@" + printf '{}\n' > "$DOCKER_CONFIG/config.json" + ;; + cp) + [[ $# == 3 && $2 == "$IMAGE" && $3 == ghcr.io/tektoncd/tekton-kueue:latest ]] + event image-copy "$@" + ;; + digest) + [[ $# == 2 && $2 == ghcr.io/tektoncd/tekton-kueue:latest ]] + event image-digest "$@" + if [[ $MODE == image-failure ]]; then printf 'sha256:%064d\n' 0; else echo "$DIGEST"; fi + ;; + *) exit 90 ;; + esac + ;; + oci) + [[ $1 == --config-file && $2 == "$CASE_DIR/work/oci/config" ]] + [[ -s $2 && -s $CASE_DIR/work/oci/key.pem ]] + if [[ $(uname -s) == Darwin ]]; then mode=(-f %Lp); else mode=(-c %a); fi + [[ $(stat "${mode[@]}" "$2") == 600 ]] + [[ $(stat "${mode[@]}" "$CASE_DIR/work/oci") == 700 ]] + shift 2 + if [[ $* == 'os ns get --query data --raw-output' ]]; then + event namespace "$@" + echo test-namespace + elif [[ $# == 12 && $1 == os && $2 == object && $3 == put && + $4 == --namespace-name && $5 == test-namespace && + $6 == --bucket-name && $7 == tekton-releases && $8 == --name && + ${10} == --file && ${12} == --force ]]; then + file=${9#tekton-kueue/latest/} + case "$file" in release.yaml|release.notags.yaml|SHA256SUMS) ;; *) exit 90 ;; esac + [[ $9 == "tekton-kueue/latest/$file" && ${11} == "$CASE_DIR/work/verified/$file" ]] + cmp "${11}" "$CASE_DIR/$file" + event bucket-put "$file" + else exit 90; fi + ;; + *) exit 90 ;; + esac + exit 0 +fi + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'status=$?; if [[ $status == 0 ]]; then rm -rf "$tmp"; else echo "test artifacts: $tmp" >&2; fi' EXIT +mkdir -p "$tmp/checkout/hack" "$tmp/bin" +mkdir "$tmp/caller-docker" +printf 'caller configuration\n' > "$tmp/caller-docker/config.json" +cp "$tmp/caller-docker/config.json" "$tmp/caller-config-original" +export DOCKER_CONFIG="$tmp/caller-docker" +cp "$root/hack/finalize-release.sh" "$tmp/checkout/hack/" +cp "$root/hack/verify-release-tag.sh" "$tmp/checkout/hack/" +for name in verify-release-artifacts.sh smoke-release.sh; do + cp "${BASH_SOURCE[0]}" "$tmp/checkout/hack/$name" +done +for name in gh curl crane oci; do + cp "${BASH_SOURCE[0]}" "$tmp/bin/$name" + chmod +x "$tmp/bin/$name" +done +export PATH="$tmp/bin:$PATH" FINALIZER_TEST_CHILD=true +export GITHUB_REPOSITORY=tektoncd/tekton-kueue GITHUB_REF=refs/heads/main +export GH_TOKEN=test-bot-token GHCR_TOKEN=test-registry-token +export OCI_API_KEY=test-private-key OCI_FINGERPRINT=test-fingerprint +export OCI_TENANCY_OCID=test-tenancy OCI_USER_OCID=test-user OCI_REGION=test-region +export VERSION=v0.4.3 COMMIT=0123456789abcdef0123456789abcdef01234567 +export DIGEST=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export IMAGE="ghcr.io/tektoncd/tekton-kueue@$DIGEST" +export CASE_DIR CALLS MODE +cases=0 +check() { + jq -e -s "$1" "$CALLS" >/dev/null || { + echo "failed event assertion: $MODE: $1" >&2; exit 1; + } +} +run_case() { + MODE=$1 + CASE_DIR="$tmp/$MODE" + CALLS="$CASE_DIR/calls.jsonl" + mkdir "$CASE_DIR" + : > "$CALLS" + cat > "$CASE_DIR/key.pub" <<'EOF' +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo= +-----END PUBLIC KEY----- +EOF + local operation=finalize latest=false version=$VERSION ref=refs/heads/main + case "$MODE" in + success|older-latest|image-failure|bucket-failure|wrong-latest) latest=true ;; + repair-*) operation=repair ;; + invalid-input) version=../v0.4.3 ;; + bad-ref) ref=refs/heads/untrusted ;; + esac + printf 'v0.4.2\n' > "$CASE_DIR/latest" + [[ $MODE != older-latest ]] || printf 'v0.5.0\n' > "$CASE_DIR/latest" + [[ $MODE != repair-published ]] || printf '%s\n' "$VERSION" > "$CASE_DIR/latest" + jq -n --arg mode "$MODE" '{ + can_admins_bypass:($mode == "admin-bypass"), + protection_rules:(if $mode == "unprotected" then [] else + [{type:"required_reviewers",prevent_self_review:($mode != "self-review"), + reviewers:[{type:"Team",reviewer:{id:18766447}}]}] end)}' > "$CASE_DIR/environment.json" + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg mode "$MODE" '{ + id:123,tag_name:$version,target_commitish:$commit,draft:($mode != "repair-published"), + body:(if $mode == "generated-notes" then "Generated release notes\n" + else "Release notes\n\n## Attestation\nold\n\n\nKeep this\n" end), + assets:[{name:"release.yaml",id:11},{name:"release.notags.yaml",id:12},{name:"SHA256SUMS",id:13}]} + ' > "$CASE_DIR/original.json" + jq --arg mode "$MODE" ' + if $mode == "wrong-source" then .target_commitish = "main" + elif $mode == "changed-assets" then .assets[0].id = 99 + else . end' "$CASE_DIR/original.json" > "$CASE_DIR/release.json" + printf 'image: %s\n' "$IMAGE" > "$CASE_DIR/release.yaml" + cp "$CASE_DIR/release.yaml" "$CASE_DIR/release.notags.yaml" + (cd "$CASE_DIR"; openssl dgst -sha256 -r release.yaml release.notags.yaml) > "$CASE_DIR/SHA256SUMS" + # Hash exactly the decoded verified bytes, not the encoded envelope or JSON + # pretty-printing. Both finalizer and Rekor fixture use real SHA-256. + jq -cn --arg commit "$COMMIT" --arg digest "${DIGEST#sha256:}" '{ + _type:"https://in-toto.io/Statement/v1",predicateType:"https://slsa.dev/provenance/v1", + subject:[{name:"ghcr.io/tektoncd/tekton-kueue",digest:{sha256:$digest}}], + predicate:{buildDefinition:{resolvedDependencies:[{ + uri:"git+https://github.com/tektoncd/tekton-kueue.git",digest:{sha1:$commit}}]}}} + ' > "$CASE_DIR/payload.json" + jq -n --rawfile payload "$CASE_DIR/payload.json" '[$payload | @base64]' > "$CASE_DIR/verified-payloads.json" + local status=0 + GITHUB_REF=$ref bash "$tmp/checkout/hack/finalize-release.sh" \ + "$operation" "$version" "$COMMIT" 123 "$DIGEST" "$CASE_DIR/key.pub" "$latest" \ + "$CASE_DIR/work" > "$CASE_DIR/output" 2>&1 || status=$? + if [[ $status == 90 ]]; then + echo "unexpected executable boundary: $MODE; see $CASE_DIR/output" >&2; exit 1; + fi + case "$MODE" in + success|repair-draft|repair-published|generated-notes) + [[ $status == 0 ]] || { echo "failed $MODE; see $CASE_DIR/output" >&2; exit 1; } + local expected_latest=$latest + [[ $MODE != repair-published ]] || expected_latest=true + jq -e --arg version "$VERSION" --arg commit "$COMMIT" --arg latest "$expected_latest" \ + --arg operation "$operation" --slurpfile original "$CASE_DIR/original.json" ' + .tag_name == $version and .target_commitish == $commit and .make_latest == $latest + and .draft == (if $operation == "repair" then $original[0].draft else false end) + and (if $original[0].body | contains("## Attestation") then + (.body | startswith("Release notes\n\n") and endswith("\nKeep this\n")) + else .body | startswith($original[0].body) end) + and (.body | contains("abcdef"))' "$CASE_DIR/sent-patch.json" >/dev/null + jq -e -s '.[0] == (.[1] * (.[2] | del(.make_latest)))' \ + "$CASE_DIR/release.json" "$CASE_DIR/original.json" "$CASE_DIR/sent-patch.json" >/dev/null + check 'map(.event) as $e | ($e|index("verify")) < ($e|index("rekor-entry")) + and ($e|index("rekor-entry")) < ($e|index("release")) + and ($e|index("release")) < ($e|index("patch")) + and all(.[] | select(.event == "patch"); .token == "test-bot-token") + and ([.[] | select(.event == "identity")][-1].token == "test-bot-token")' + if [[ $operation == repair ]]; then + check 'all(.[]; .event != "smoke" and .event != "image-copy" and .event != "bucket-put" and .event != "login")' + elif [[ $latest == true ]]; then + check 'map(.event) as $e | ($e|index("verify")) < ($e|index("smoke")) + and ($e|index("smoke")) < ($e|index("image-copy")) + and ($e|index("image-copy")) < ($e|index("image-digest")) + and ($e|index("image-digest")) < ($e|index("bucket-put")) + and ($e|rindex("bucket-read")) < ($e|index("patch")) + and ([.[] | select(.event == "bucket-put") | .args[0]] | sort) == + ["SHA256SUMS","release.notags.yaml","release.yaml"]' + else + check 'map(.event) as $e | ($e|index("verify")) < ($e|index("smoke")) + and ($e|index("smoke")) < ($e|index("patch")) + and all(.[]; .event != "image-copy" and .event != "bucket-put" and .event != "login")' + fi + if [[ $MODE == generated-notes ]]; then + # Recovery-generated notes must remain repairable after publication. + cp "$CASE_DIR/release.json" "$CASE_DIR/original.json" + mv "$CASE_DIR/work" "$CASE_DIR/first-work" + bash "$tmp/checkout/hack/finalize-release.sh" repair "$VERSION" "$COMMIT" 123 \ + "$DIGEST" "$CASE_DIR/key.pub" false "$CASE_DIR/work" >> "$CASE_DIR/output" 2>&1 + jq -e -s '.[0].body == .[1].body and .[0].draft == false' \ + "$CASE_DIR/release.json" "$CASE_DIR/original.json" >/dev/null + fi + ;; + *) + [[ $status != 0 ]] || { echo "unexpected success: $MODE" >&2; exit 1; } + case "$MODE" in + verifier-failure|smoke-failure|api-failure) [[ $status == 42 ]] ;; + *) [[ $status == 1 ]] ;; + esac + case "$MODE" in + api-failure|wrong-result|wrong-latest) + check 'any(.[]; .event == "patch")' + [[ $MODE != api-failure ]] || cmp "$CASE_DIR/original.json" "$CASE_DIR/release.json" + ;; + *) + check 'all(.[]; .event != "patch")' + cmp "$CASE_DIR/release.json" "$CASE_DIR/work/current-release.json" 2>/dev/null || + [[ ! -e $CASE_DIR/work/current-release.json ]] + ;; + esac + case "$MODE" in + image-failure|bucket-failure) + check 'any(.[]; .event == "image-copy")' + cmp "$CASE_DIR/original.json" "$CASE_DIR/release.json" + ;; + api-failure|wrong-result|wrong-latest) ;; + *) + check 'all(.[]; .event != "image-copy" and .event != "bucket-put" and .event != "login")' + ;; + esac + case "$MODE" in + wrong-actor|unprotected|self-review|admin-bypass|older-latest|invalid-input|bad-ref) + check 'all(.[]; .event != "verify" and .event != "smoke")' ;; + verifier-failure|bad-digest) + check 'any(.[]; .event == "verify") and all(.[]; .event != "smoke")' ;; + smoke-failure) check 'any(.[]; .event == "smoke")' ;; + rekor-mismatch) check 'any(.[]; .event == "rekor-entry")' ;; + wrong-source|changed-assets) check 'any(.[]; .event == "release")' ;; + image-failure) + check 'any(.[]; .event == "image-digest") and all(.[]; .event != "bucket-put")' ;; + bucket-failure) check 'any(.[]; .event == "bucket-read")' ;; + esac + ;; + esac + [[ ! -e $CASE_DIR/work/docker && ! -e $CASE_DIR/work/oci ]] + cmp "$tmp/caller-docker/config.json" "$tmp/caller-config-original" + cases=$((cases + 1)) + echo "PASS $MODE" +} + +# Fifteen focused groups; related protection, repair and API outcomes share a +# group rather than multiplying every failure across finalize/repair/latest. +run_case success +run_case generated-notes +for mode in repair-draft repair-published; do run_case "$mode"; done +run_case wrong-actor +for mode in unprotected self-review admin-bypass; do run_case "$mode"; done +run_case bad-digest +run_case verifier-failure +run_case smoke-failure +run_case wrong-source +run_case changed-assets +run_case changed-tag +run_case rekor-mismatch +run_case older-latest +run_case image-failure +run_case bucket-failure +for mode in api-failure wrong-result wrong-latest; do run_case "$mode"; done +for mode in invalid-input bad-ref; do run_case "$mode"; done +echo "PASS $cases deterministic release finalization cases (16 focused groups)" diff --git a/hack/recover-release.sh b/hack/recover-release.sh new file mode 100644 index 000000000..951966c35 --- /dev/null +++ b/hack/recover-release.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release recovery failed: $*" >&2; exit 1; } +robot() { + [[ $(gh api user --jq .login) == tekton-robot ]] || fail "GH_TOKEN must belong to tekton-robot" +} +[[ $# == 5 ]] || fail "usage: $0 VERSION COMMIT DIGEST PUBLIC_KEY_FILE WORK_DIR" +version=$1 +commit=$2 +digest=$3 +key=$4 +work=$5 +repository=tektoncd/tekton-kueue +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ && $commit =~ ^[0-9a-f]{40}$ && + $digest =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid candidate identity" +[[ ${GITHUB_REPOSITORY:-} == "$repository" && ${GITHUB_REF:-} == refs/heads/main ]] || + fail "run the trusted main-branch workflow" +[[ -n ${GH_TOKEN:-} && -s $key && -r $key ]] || fail "robot token and trusted key are required" +[[ -n $work && ! -L $work ]] || fail "invalid work directory" +if [[ -e $work ]]; then + [[ -d $work && -z $(find "$work" -mindepth 1 -print -quit) ]] || fail "work directory must be empty" +fi +mkdir -p "$work" +work=$(cd "$work" && pwd) +robot +[[ $(gh api "repos/$repository" --jq .permissions.push) == true ]] || + fail "repository push permission is required to inspect drafts" +gh api "repos/$repository/environments/kueue-release-finalization" > "$work/environment.json" +jq -e ' + .can_admins_bypass == false and + any(.protection_rules[]?; + .type == "required_reviewers" and .prevent_self_review == true and + any(.reviewers[]?; .type == "Team" and .reviewer.id == 18766447)) +' "$work/environment.json" >/dev/null || fail "release approval environment is not protected" + +# No GitHub mutation is allowed until a complete, signed existing bundle passes. +bash "$root/hack/verify-release-artifacts.sh" "$version" "$commit" bucket "$key" "$work/verified" +jq -e --arg digest "$digest" '.digest == $digest' "$work/verified/release-info.json" >/dev/null || + fail "verified digest differs from approved candidate" + +check_tag() { + bash "$root/hack/verify-release-tag.sh" "$version" "$commit" true +} + +find_release() { + gh api --paginate --slurp "repos/$repository/releases?per_page=100" > "$work/releases.json" + jq --arg version "$version" '[.[][] | select(.tag_name == $version)]' \ + "$work/releases.json" > "$work/candidates.json" + jq -e 'length <= 1' "$work/candidates.json" >/dev/null || fail "multiple matching releases" + release_id=$(jq -r '.[0].id // empty' "$work/candidates.json") + [[ -z $release_id || $release_id =~ ^[1-9][0-9]*$ ]] || fail "invalid release ID" +} + +refresh() { + local file asset_id + check_tag + gh api "repos/$repository/releases/$release_id" > "$work/current.json" + jq -e --arg id "$release_id" --arg version "$version" --arg commit "$commit" ' + .id == ($id | tonumber) and .tag_name == $version + and .target_commitish == $commit and .draft == true + and (.assets | type == "array") + and ([.assets[] | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS")] + | (map(.name) | unique | length) == length + and all(.[]; (.id | type == "number") and .id > 0 and .id == (.id | floor)) + and (map(.id) | unique | length) == length) + ' "$work/current.json" >/dev/null || fail "release source, publication state or assets conflict" + for file in release.yaml release.notags.yaml SHA256SUMS; do + asset_id=$(jq -r --arg name "$file" '.assets[] | select(.name == $name) | .id' "$work/current.json") + if [[ -n $asset_id ]]; then + gh api --method GET -H 'Accept: application/octet-stream' \ + "repos/$repository/releases/assets/$asset_id" > "$work/existing-$file" + cmp -s "$work/existing-$file" "$work/verified/$file" || fail "conflicting existing asset: $file" + fi + done +} + +check_tag +find_release +if [[ -z $release_id ]]; then + gh api --method POST "repos/$repository/releases/generate-notes" \ + -f tag_name="$version" -f target_commitish="$commit" > "$work/notes.json" + jq -e '.body | type == "string" and length > 0' "$work/notes.json" >/dev/null || + fail "cannot generate release notes" + # Recheck absence immediately before creating; never replace another draft. + check_tag + find_release + if [[ -z $release_id ]]; then + jq --arg version "$version" --arg commit "$commit" ' + {tag_name:$version,target_commitish:$commit,draft:true, + name:$version,body:.body,make_latest:"false"}' "$work/notes.json" > "$work/create.json" + robot + gh api --method POST "repos/$repository/releases" --input "$work/create.json" > "$work/created.json" + release_id=$(jq -er '.id | select(type == "number" and . > 0 and . == floor)' "$work/created.json") + fi +fi +for file in release.yaml release.notags.yaml SHA256SUMS; do + refresh + if ! jq -e --arg name "$file" 'any(.assets[]; .name == $name)' "$work/current.json" >/dev/null; then + robot + # No --clobber: a concurrent attachment must fail, never replace bytes. + gh release upload "$version" "$work/verified/$file" --repo "$repository" + fi +done +refresh +bash "$root/hack/verify-release-artifacts.sh" "$version" "$commit" "$release_id" "$key" "$work/recovered" +jq -e --arg digest "$digest" '.digest == $digest and .draft == true' \ + "$work/recovered/release-info.json" >/dev/null || fail "recovered draft verification failed" +printf 'Verified draft ID: %s\nFinalize separately after approval; no rebuild or latest promotion occurred.\n' "$release_id" diff --git a/hack/recover-release_test.sh b/hack/recover-release_test.sh new file mode 100644 index 000000000..26cbe4f37 --- /dev/null +++ b/hack/recover-release_test.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${RECOVERY_TEST_CHILD:-} == true ]]; then + event() { + jq -cn --arg event "$1" --args '{event:$event,args:$ARGS.positional}' \ + -- "${@:2}" >> "$CALLS" + } + case ${0##*/} in + verify-release-artifacts.sh) + [[ $# == 5 && $1 == "$VERSION" && $2 == "$COMMIT" && $4 == "$CASE_DIR/key.pub" ]] + openssl pkey -pubin -in "$4" -noout + event verify "$3" + [[ $MODE != verification-failure && $MODE != incomplete-bundle ]] || exit 42 + [[ $3 == bucket || $3 == 123 ]] || exit 90 + [[ $MODE != final-verification-failure || $3 == bucket ]] || exit 42 + mkdir -p "$5" + cp "$CASE_DIR"/{release.yaml,release.notags.yaml,SHA256SUMS} "$5/" + jq -n --arg digest "$DIGEST" --arg mode "$MODE" --arg id "$3" ' + {digest:(if $mode == "wrong-digest" then "sha256:" + ("b"*64) else $digest end), + draft:($id != "bucket"),release_id:(if $id == "bucket" then null else 123 end)} + ' > "$5/release-info.json" + ;; + gh) + if [[ $1 == release ]]; then + [[ $# == 6 && $2 == upload && $3 == "$VERSION" && + $5 == --repo && $6 == tektoncd/tekton-kueue ]] + file=${4##*/} + cmp "$4" "$CASE_DIR/$file" + # Mimic the real non-clobber upload contract. + if jq -e --arg name "$file" 'any(.assets[]; .name == $name)' "$CASE_DIR/state.json" >/dev/null; then + exit 90 + fi + [[ $MODE != upload-failure ]] || exit 42 + event upload "$file" + case $file in release.yaml) id=11 ;; release.notags.yaml) id=12 ;; SHA256SUMS) id=13 ;; *) exit 90 ;; esac + jq --arg name "$file" --argjson id "$id" '.assets += [{name:$name,id:$id}]' \ + "$CASE_DIR/state.json" > "$CASE_DIR/next.json" + mv "$CASE_DIR/next.json" "$CASE_DIR/state.json" + exit + fi + [[ $1 == api ]] || exit 90 + shift + case $1 in + user) + [[ $# == 3 && $2 == --jq && $3 == .login ]] + if [[ $MODE == wrong-actor ]]; then echo waveywaves; else echo tekton-robot; fi + ;; + repos/tektoncd/tekton-kueue) + [[ $# == 3 && $2 == --jq && $3 == .permissions.push ]] + if [[ $MODE == no-permission ]]; then echo false; else echo true; fi + ;; + repos/tektoncd/tekton-kueue/environments/kueue-release-finalization) + jq -n --arg mode "$MODE" '{can_admins_bypass:($mode == "admin-bypass"), + protection_rules:(if $mode == "unprotected" then [] else [{ + type:"required_reviewers",prevent_self_review:($mode != "self-review"), + reviewers:[{type:"Team",reviewer:{id:18766447}}]}] end)}' + ;; + --paginate) + [[ $# == 3 && $2 == --slurp ]] || exit 90 + case $3 in + repos/tektoncd/tekton-kueue/git/matching-refs/tags/*) + [[ $MODE != tag-api-failure ]] || exit 42 + if [[ $MODE == no-tag ]]; then echo '[[]]'; else + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg mode "$MODE" ' + [[{ref:("refs/tags/" + $version),object:{ + type:(if $mode == "annotated-tag" then "tag" else "commit" end), + sha:(if $mode == "wrong-tag" then "f"*40 + elif $mode == "annotated-tag" then "e"*40 else $commit end)}}]]' + fi + ;; + 'repos/tektoncd/tekton-kueue/releases?per_page=100') + [[ $MODE != lookup-failure ]] || exit 42 + jq --arg mode "$MODE" ' + if . == null then [[]] elif $mode == "duplicate-release" then [[.,.]] + else [[],[.]] end' "$CASE_DIR/state.json" + ;; + *) exit 90 ;; + esac + ;; + repos/tektoncd/tekton-kueue/git/tags/*) + [[ $MODE == annotated-tag ]] + jq -n --arg commit "$COMMIT" '{object:{type:"commit",sha:$commit}}' + ;; + repos/tektoncd/tekton-kueue/releases/123) + cat "$CASE_DIR/state.json" + ;; + --method) + if [[ $2 == POST && $3 == repos/tektoncd/tekton-kueue/releases/generate-notes ]]; then + [[ $# == 7 && $4 == -f && $5 == "tag_name=$VERSION" && + $6 == -f && $7 == "target_commitish=$COMMIT" ]] + [[ $MODE != notes-failure ]] || exit 42 + event notes + jq -n '{body:"Generated notes from existing commits"}' + elif [[ $2 == POST && $3 == repos/tektoncd/tekton-kueue/releases ]]; then + [[ $# == 5 && $4 == --input && $(< "$CASE_DIR/state.json") == null ]] + jq -e --arg version "$VERSION" --arg commit "$COMMIT" ' + .tag_name == $version and .target_commitish == $commit + and .draft == true and .make_latest == "false" and (.body | length > 0) + ' "$5" >/dev/null + [[ $MODE != create-failure ]] || exit 42 + event create + jq '. + {id:123,assets:[]}' "$5" > "$CASE_DIR/state.json" + cat "$CASE_DIR/state.json" + elif [[ $2 == GET && $3 == -H && $4 == 'Accept: application/octet-stream' ]]; then + [[ $# == 5 ]] + case ${5##*/} in 11) file=release.yaml ;; 12) file=release.notags.yaml ;; 13) file=SHA256SUMS ;; *) exit 90 ;; esac + cat "$CASE_DIR/$file" + [[ $MODE != conflicting-asset ]] || printf 'conflict\n' + else exit 90; fi + ;; + *) exit 90 ;; + esac + ;; + *) exit 90 ;; + esac + exit +fi + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'status=$?; if [[ $status == 0 ]]; then rm -rf "$tmp"; else echo "test artifacts: $tmp" >&2; fi' EXIT +mkdir -p "$tmp/checkout/hack" "$tmp/bin" +cp "$root/hack/recover-release.sh" "$tmp/checkout/hack/" +cp "$root/hack/verify-release-tag.sh" "$tmp/checkout/hack/" +cp "${BASH_SOURCE[0]}" "$tmp/checkout/hack/verify-release-artifacts.sh" +cp "${BASH_SOURCE[0]}" "$tmp/bin/gh" +chmod +x "$tmp/bin/gh" +export PATH="$tmp/bin:$PATH" RECOVERY_TEST_CHILD=true GH_TOKEN=test-robot-token +export GITHUB_REPOSITORY=tektoncd/tekton-kueue GITHUB_REF=refs/heads/main +export VERSION=v0.4.3 COMMIT=0123456789abcdef0123456789abcdef01234567 +export DIGEST=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export MODE CASE_DIR CALLS +cases=0 +for MODE in missing-draft no-tag partial-assets complete annotated-tag \ + wrong-actor no-permission unprotected self-review admin-bypass verification-failure \ + incomplete-bundle wrong-digest wrong-tag tag-api-failure lookup-failure duplicate-release \ + wrong-source published duplicate-asset duplicate-asset-id conflicting-asset notes-failure create-failure \ + upload-failure final-verification-failure invalid-input bad-ref missing-key invalid-key; do + CASE_DIR="$tmp/$MODE" + mkdir "$CASE_DIR" + CALLS="$CASE_DIR/calls.jsonl" + : > "$CALLS" + cat > "$CASE_DIR/key.pub" <<'EOF' +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo= +-----END PUBLIC KEY----- +EOF + [[ $MODE != invalid-key ]] || printf 'not a public key\n' > "$CASE_DIR/key.pub" + printf 'verified manifest\n' > "$CASE_DIR/release.yaml" + cp "$CASE_DIR/release.yaml" "$CASE_DIR/release.notags.yaml" + (cd "$CASE_DIR"; openssl dgst -sha256 -r release.yaml release.notags.yaml) > "$CASE_DIR/SHA256SUMS" + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg mode "$MODE" ' + if (["missing-draft","no-tag","notes-failure","create-failure"] | index($mode)) != null + then null else + {id:123,tag_name:$version, + target_commitish:(if $mode == "wrong-source" then "main" else $commit end), + draft:($mode != "published"),body:"Keep existing notes",assets: + (if $mode == "complete" then + [{name:"release.yaml",id:11},{name:"release.notags.yaml",id:12},{name:"SHA256SUMS",id:13}] + elif $mode == "duplicate-asset" then [{name:"release.yaml",id:11},{name:"release.yaml",id:14}] + elif $mode == "duplicate-asset-id" then [{name:"release.yaml",id:11},{name:"release.notags.yaml",id:11}] + else [{name:"release.yaml",id:11}] end)} + end' > "$CASE_DIR/state.json" + cp "$CASE_DIR/state.json" "$CASE_DIR/original.json" + version=$VERSION + ref=refs/heads/main + key="$CASE_DIR/key.pub" + [[ $MODE != invalid-input ]] || version=../v0.4.3 + [[ $MODE != bad-ref ]] || ref=refs/heads/untrusted + [[ $MODE != missing-key ]] || key="$CASE_DIR/missing.pub" + status=0 + GITHUB_REF=$ref bash "$tmp/checkout/hack/recover-release.sh" \ + "$version" "$COMMIT" "$DIGEST" "$key" "$CASE_DIR/work" > "$CASE_DIR/output" 2>&1 || status=$? + case $MODE in + missing-draft|no-tag|partial-assets|complete|annotated-tag) + [[ $status == 0 ]] || { echo "failed $MODE; see $CASE_DIR/output" >&2; exit 1; } + jq -e '.draft == true and ([.assets[].name] | sort == + ["SHA256SUMS","release.notags.yaml","release.yaml"])' "$CASE_DIR/state.json" >/dev/null + if [[ $MODE != missing-draft && $MODE != no-tag ]]; then + jq -e -s '.[0].body == .[1].body' "$CASE_DIR/state.json" "$CASE_DIR/original.json" >/dev/null + fi + jq -e -s '.[0] == {event:"verify",args:["bucket"]} + and .[-1] == {event:"verify",args:["123"]}' "$CALLS" >/dev/null + writes=$(jq -s '[.[] | select(.event == "create" or .event == "upload")] | length' "$CALLS") + bash "$tmp/checkout/hack/recover-release.sh" "$VERSION" "$COMMIT" "$DIGEST" \ + "$key" "$CASE_DIR/again" >> "$CASE_DIR/output" 2>&1 + [[ $(jq -s '[.[] | select(.event == "create" or .event == "upload")] | length' "$CALLS") == "$writes" ]] + ;; + *) + [[ $status != 0 && $status != 90 ]] || { echo "unexpected $MODE status $status; see $CASE_DIR/output" >&2; exit 1; } + case $MODE in + upload-failure|final-verification-failure) ;; + *) jq -e -s 'all(.[]; .event != "create" and .event != "upload")' "$CALLS" >/dev/null ;; + esac + ;; + esac + cases=$((cases + 1)) + echo "PASS $MODE" +done +echo "PASS $cases deterministic release recovery cases" diff --git a/hack/smoke-release.sh b/hack/smoke-release.sh new file mode 100644 index 000000000..a70b59e4e --- /dev/null +++ b/hack/smoke-release.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release smoke failed: $*" >&2; exit 1; } +[[ $# == 3 ]] || fail "usage: $0 VERIFIED_MANIFEST EXPECTED_IMAGE ARTIFACT_DIR" +manifest=$1 +image=$2 +artifacts=$3 +repository=ghcr.io/tektoncd/tekton-kueue +[[ $image =~ ^ghcr\.io/tektoncd/tekton-kueue@sha256:[0-9a-f]{64}$ ]] || + fail "EXPECTED_IMAGE must be the canonical digest reference" +[[ -f $manifest && -r $manifest && -s $manifest ]] || fail "manifest must be a nonempty readable file" +[[ -n $artifacts && ! -L $artifacts ]] || fail "invalid artifact directory" +if [[ -e $artifacts ]]; then + [[ -d $artifacts && -z $(find "$artifacts" -mindepth 1 -print -quit) ]] || + fail "artifact directory must be empty" +fi +for command in kind kubectl jq openssl; do + command -v "$command" >/dev/null || fail "missing command: $command" +done +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d) +cluster="release-smoke-$(basename "$work" | tr '[:upper:]._' '[:lower:]--')" +context="kind-$cluster" +export KUBECONFIG="$work/kubeconfig" +owned=false + +k() { kubectl --context="$context" --request-timeout=30s "$@"; } +cleanup() { + status=$? + trap - EXIT + set +e + if [[ $owned == true ]]; then + for resource in pods deployments services endpoints events certificates \ + pipelineruns.tekton.dev taskruns.tekton.dev workloads.kueue.x-k8s.io; do + k get "$resource" -A -o yaml > "$artifacts/$resource.yaml" 2>&1 + done + k describe pods -A > "$artifacts/pod-descriptions.txt" 2>&1 + for namespace in cert-manager kueue-system tekton-pipelines tekton-kueue release-smoke; do + pods=$(k get pods -n "$namespace" -o name) + for pod in $pods; do + k logs -n "$namespace" "$pod" --all-containers=true --prefix=true \ + --tail=200 > "$artifacts/$namespace-${pod#*/}.log" 2>&1 + done + done + kind export logs --name "$cluster" "$artifacts/kind" > "$artifacts/kind-export.log" 2>&1 + kind delete cluster --name "$cluster" > "$artifacts/cleanup.log" 2>&1 + if [[ $? != 0 ]]; then + echo "failed to delete owned cluster: $cluster" >&2 + status=1 + fi + fi + rm -rf "$work" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +# Parse the finished YAML locally, without discovery or the user's kubeconfig. +# This labels only the in-memory objects, never the installed release. +cp "$manifest" "$work/release.yaml" +kubectl label --local --overwrite -f "$work/release.yaml" -o json release-smoke-validation=true > "$work/manifest.json" +jq -e -s --arg repo "$repository" --arg digest "${image##*@}" ' + [ .[] | if .kind == "List" then .items[] else . end ] as $objects + | [$objects[] | select(.kind == "Deployment")] as $deployments + | ($deployments | length == 2) + and ($deployments | map(.metadata.name) | sort == + ["tekton-kueue-controller-manager", "tekton-kueue-webhook"]) + and all($deployments[]; + .apiVersion == "apps/v1" and .metadata.namespace == "tekton-kueue" + and (.spec.template.spec.containers | length == 1) + and ((.spec.template.spec.initContainers // []) | length == 0)) + and ([$objects[] | .. | objects | select(has("image")) | .image] | + length == 2 and all(.[]; + type == "string" and + (split("@") | length == 2 and .[1] == $digest + and (.[0] == $repo or + (.[0] | startswith($repo + ":") and + (ltrimstr($repo + ":") | test("^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$"))))))) +' "$work/manifest.json" >/dev/null || fail "manifest controller/webhook images do not match EXPECTED_IMAGE" + +mkdir -p "$artifacts" +artifacts=$(cd "$artifacts" && pwd) +cp "$work/release.yaml" "$artifacts/release.yaml" +printf '%s\n' "$image" > "$artifacts/expected-image.txt" +printf '%s\n' "$cluster" > "$artifacts/cluster-name.txt" +# Install only the snapshot checked above, even if the caller changes its file. +exec > >(tee "$artifacts/smoke.log") 2>&1 +owned=true # Creation can fail after provisioning nodes; still clean up our name. +kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 180s --retain \ + --image kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95 + +k apply --server-side -f https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml +k wait --for=condition=Available deployment --all -n cert-manager --timeout=300s +k apply --server-side -f https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml \ + -f "$root/hack/kueue-config.yaml" +k rollout restart deployment/kueue-controller-manager -n kueue-system +k rollout status deployment/kueue-controller-manager -n kueue-system --timeout=300s +k apply --server-side -f https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml +k wait --for=condition=Available deployment --all -n tekton-pipelines --timeout=300s +k apply --server-side -f "$work/release.yaml" +k wait --for=condition=Ready certificate --all -n tekton-kueue --timeout=300s +k rollout status deployment/tekton-kueue-controller-manager -n tekton-kueue --timeout=300s +k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +k wait --for=jsonpath='{.webhooks[0].clientConfig.caBundle}' \ + mutatingwebhookconfiguration/tekton-kueue-mutating-webhook-configuration --timeout=300s + +# Configure one CPU resource mutation. Restart after configuration so the new +# webhook starts with the ConfigMap already present, rather than racing reload. +k apply -f - <<'YAML' +apiVersion: v1 +kind: ConfigMap +metadata: + name: tekton-kueue-config + namespace: tekton-kueue +data: + config.yaml: | + queueName: pipelines-queue + cel: + expressions: + - resource("cpu", 1) +--- +apiVersion: v1 +kind: Namespace +metadata: + name: release-smoke +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: ResourceFlavor +metadata: + name: release-smoke +spec: {} +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: ClusterQueue +metadata: + name: release-smoke +spec: + namespaceSelector: {} + resourceGroups: + - coveredResources: ["cpu", "tekton.dev/pipelineruns"] + flavors: + - name: release-smoke + resources: + - name: cpu + nominalQuota: 1 + - name: tekton.dev/pipelineruns + nominalQuota: 1 +--- +apiVersion: kueue.x-k8s.io/v1beta2 +kind: LocalQueue +metadata: + name: pipelines-queue + namespace: release-smoke +spec: + clusterQueue: release-smoke +YAML +k get pods -n tekton-kueue -l app.kubernetes.io/name=tekton-kueue-webhook -o json \ + > "$work/old-webhook-pods.json" +k rollout restart deployment/tekton-kueue-webhook -n tekton-kueue +k rollout status deployment/tekton-kueue-webhook -n tekton-kueue --timeout=300s +# Deployment availability alone can precede removal of the old Service endpoint. +# Watch the old Pods disappear and the endpoint switch before admission. +while IFS= read -r pod; do + k wait --for=delete "pod/$pod" -n tekton-kueue --timeout=300s +done < <(jq -r '.items[].metadata.name' "$work/old-webhook-pods.json") +k get pods -n tekton-kueue -l app.kubernetes.io/name=tekton-kueue-webhook -o json \ + > "$work/webhook-pods.json" +pod=$(jq -er '[.items[] | select(.metadata.deletionTimestamp == null)] | + if length == 1 then .[0].metadata.name else error("expected one webhook Pod") end' "$work/webhook-pods.json") +k wait --for=jsonpath='{.subsets[0].addresses[0].targetRef.name}'="$pod" \ + endpoints/tekton-kueue-webhook-service -n tekton-kueue --timeout=120s +k wait --for=condition=Active clusterqueue/release-smoke --timeout=120s +k wait --for=condition=Active localqueue/pipelines-queue -n release-smoke --timeout=120s +k create -f - -o json > "$artifacts/created-pipelinerun.json" <<'YAML' +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + name: release-smoke + namespace: release-smoke +spec: + timeouts: + pipeline: 5m0s + pipelineSpec: + tasks: + - name: hello + taskSpec: + steps: + - name: hello + image: registry.access.redhat.com/ubi9/ubi@sha256:66233eebd72bb5baa25190d4f55e1dc3fff3a9b77186c1f91a0abdb274452072 + command: ["/bin/echo", "release smoke"] +YAML +# The create response is the admission result, before controller updates. +jq -e '.spec.status == "PipelineRunPending" + and .metadata.labels["kueue.x-k8s.io/queue-name"] == "pipelines-queue" + and .metadata.annotations["kueue.konflux-ci.dev/requests-cpu"] == "1" + and (.metadata.uid | type == "string" and length > 0) +' "$artifacts/created-pipelinerun.json" >/dev/null || fail "webhook mutation missing" +uid=$(jq -r .metadata.uid "$artifacts/created-pipelinerun.json") +# Kueue v0.16.6 GetWorkloadNameForOwnerWithGVK hashes kind, group, name, UID. +hash=$(printf 'PipelineRun\ntekton.dev\nrelease-smoke\n%s' "$uid" | openssl dgst -sha1 -r) +workload="workload/pipelinerun-release-smoke-${hash:0:5}" +k wait --for=create "$workload" -n release-smoke --timeout=120s +k get "$workload" -n release-smoke -o json > "$artifacts/created-workload.json" +jq -e --arg uid "$uid" ' + any(.metadata.ownerReferences[]; .uid == $uid and .kind == "PipelineRun" and .controller == true) + and .spec.queueName == "pipelines-queue" + and .spec.podSets[0].template.spec.containers[0].resources.requests.cpu == "1" + and .spec.podSets[0].template.spec.containers[0].resources.requests["tekton.dev/pipelineruns"] == "1" +' "$artifacts/created-workload.json" >/dev/null || fail "Workload does not represent the mutated PipelineRun" +k wait --for=condition=Admitted "$workload" -n release-smoke --timeout=300s +k wait --for=condition=Succeeded pipelinerun/release-smoke -n release-smoke --timeout=360s +echo "PASS published release $image" diff --git a/hack/smoke-release_test.sh b/hack/smoke-release_test.sh new file mode 100644 index 000000000..546d84953 --- /dev/null +++ b/hack/smoke-release_test.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The same executable supplies both boundary fakes. Local YAML parsing remains +# real kubectl, using the smoke script's isolated kubeconfig and --local. +if [[ ${SMOKE_FAKE:-} == true ]]; then + command=$(basename "$0") + [[ -n ${KUBECONFIG:-} && $KUBECONFIG != "$USER_KUBECONFIG" ]] || exit 90 + jq -cn --arg command "$command" --arg kubeconfig "$KUBECONFIG" \ + --args '$ARGS.positional as $args | {command:$command,kubeconfig:$kubeconfig,args:$args}' \ + -- "$@" >> "$CALLS" + if [[ $command == kind ]]; then + case "$*" in + "create cluster "*) + [[ $* == *"--kubeconfig $KUBECONFIG"* && $* == *"--wait 180s"* ]] || exit 91 + [[ $MODE != creation ]] || exit 42 + ;; + "delete cluster "*) + [[ $MODE != cleanup ]] || exit 42 + ;; + "export logs "*) ;; + *) exit 92 ;; + esac + exit 0 + fi + if [[ $1 == label ]]; then + [[ $* == *"--local"* ]] || exit 93 + exec "$REAL_KUBECTL" "$@" + fi + [[ $1 == --context=kind-release-smoke-* && $2 == --request-timeout=30s ]] || exit 94 + shift 2 + case "$1" in + apply) + if [[ $* == *"-f -"* ]]; then + # Parse actual fixtures offline; invalid YAML cannot silently pass. + "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ + > "$CASE_DIR/fixtures.json" + elif [[ $* == *"/release.yaml"* && $* != *https:* ]]; then + for ((i=1; i<=$#; i++)); do + if [[ ${!i} == -f ]]; then + j=$((i + 1)) + cmp "$VERIFIED_MANIFEST" "${!j}" + fi + done + fi + ;; + rollout) + if [[ $MODE == readiness && $* == *"status deployment/tekton-kueue-webhook"* ]]; then exit 42; fi + if [[ $* == *"restart deployment/tekton-kueue-webhook"* ]]; then touch "$CASE_DIR/restarted"; fi + ;; + wait) + [[ $* == *--timeout=* ]] || exit 95 + if [[ $MODE == run && $* == *condition=Succeeded* ]]; then exit 42; fi + if [[ $MODE == admission && $* == *condition=Admitted* ]]; then exit 42; fi + if [[ $MODE == workload && $* == *--for=create* ]]; then exit 42; fi + if [[ $MODE == endpoint && $* == *endpoints/tekton-kueue-webhook-service* ]]; then exit 42; fi + ;; + create) + "$REAL_KUBECTL" label --local --overwrite -f - -o json fixture-validation=true \ + > "$CASE_DIR/pipelinerun.json" + jq -n --arg mode "$MODE" '{ + metadata:{uid:"smoke-uid", + labels:{"kueue.x-k8s.io/queue-name":"pipelines-queue"}, + annotations:{"kueue.konflux-ci.dev/requests-cpu":(if $mode == "mutation" then "2" else "1" end)}}, + spec:{status:(if $mode == "wrong-pending" then "Pending" else "PipelineRunPending" end)}}' + ;; + get) + if [[ $2 == pods && $* == *"-o json"* ]]; then + name=old-webhook + [[ ! -e $CASE_DIR/restarted ]] || name=new-webhook + jq -n --arg name "$name" '{items:[{metadata:{name:$name}}]}' + elif [[ $2 == workload/* ]]; then + jq -n '{metadata:{ownerReferences:[{uid:"smoke-uid",kind:"PipelineRun",controller:true}]}, + spec:{queueName:"pipelines-queue",podSets:[{template:{spec:{containers:[ + {resources:{requests:{cpu:"1","tekton.dev/pipelineruns":"1"}}} + ]}}}]}}' + elif [[ $* != *"-o name"* ]]; then + printf 'kind: List\nitems: []\n' + fi + ;; + describe|logs) ;; + *) exit 96 ;; + esac + exit 0 +fi + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'status=$?; if [[ $status == 0 ]]; then rm -rf "$tmp"; else echo "test artifacts: $tmp" >&2; fi' EXIT +export REAL_KUBECTL +REAL_KUBECTL=$(command -v kubectl) +mkdir "$tmp/bin" +cp "${BASH_SOURCE[0]}" "$tmp/bin/kubectl" +cp "${BASH_SOURCE[0]}" "$tmp/bin/kind" +chmod +x "$tmp/bin/kubectl" "$tmp/bin/kind" +export PATH="$tmp/bin:$PATH" SMOKE_FAKE=true +export USER_KUBECONFIG="$tmp/user-kubeconfig" +printf 'untouched user configuration\n' > "$USER_KUBECONFIG" +cp "$USER_KUBECONFIG" "$tmp/user-original" +image=ghcr.io/tektoncd/tekton-kueue@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export CASE_DIR CALLS MODE VERIFIED_MANIFEST +cases=0 + +check() { + jq -e -s "$1" "$CALLS" >/dev/null || { echo "failed assertion in $MODE: $1" >&2; exit 1; } +} +run_case() { + MODE=$1 + CASE_DIR="$tmp/$MODE" + mkdir "$CASE_DIR" + CALLS="$CASE_DIR/calls.jsonl" + : > "$CALLS" + VERIFIED_MANIFEST="$CASE_DIR/release.yaml" + local expected=$image + cp "$root/test/release-smoke/release.yaml" "$VERIFIED_MANIFEST" + case "$MODE" in + digest) expected=ghcr.io/tektoncd/tekton-kueue:latest ;; + mismatch) expected=${image/aaaa/bbbb} ;; + malformed) printf 'not: [valid yaml\n' > "$VERIFIED_MANIFEST" ;; + missing) rm "$VERIFIED_MANIFEST" ;; + incomplete) + # A valid document without the required controller/webhook pair. + printf 'apiVersion: v1\nkind: Namespace\nmetadata:\n name: test\n' > "$VERIFIED_MANIFEST" + ;; + artifacts) mkdir "$CASE_DIR/artifacts"; touch "$CASE_DIR/artifacts/keep" ;; + esac + local status=0 + KUBECONFIG="$USER_KUBECONFIG" bash "$root/hack/smoke-release.sh" \ + "$VERIFIED_MANIFEST" "$expected" "$CASE_DIR/artifacts" > "$CASE_DIR/output" 2>&1 || status=$? + case "$MODE" in + success|success-again) [[ $status == 0 ]] || { echo "success flow failed; see $CASE_DIR/output" >&2; "$REAL_KUBECTL" version --client; exit 1; } ;; + *) [[ $status != 0 ]] || { echo "$MODE unexpectedly passed" >&2; exit 1; } ;; + esac + cmp "$USER_KUBECONFIG" "$tmp/user-original" + case "$MODE" in + digest|mismatch|malformed|missing|incomplete|artifacts) + check 'all(.[]; .command != "kind")' + ;; + *) + check ' + [.[] | select(.command == "kind" and .args[0] == "create")][0] as $create + | ($create.args | index("--name")) as $nameIndex + | $create.args[$nameIndex + 1] as $name + | ($name | startswith("release-smoke-")) + and all(.[] | select(.command == "kind"); + (.args | index("--name")) as $i | .args[$i + 1] == $name) + and any(.[]; .command == "kind" and .args[0] == "delete") + and all(.[] | select(.command == "kubectl" and .args[0] != "label"); + .args[0] == ("--context=kind-" + $name) and .kubeconfig == $create.kubeconfig) + ' + [[ -s $CASE_DIR/artifacts/events.yaml && -f $CASE_DIR/artifacts/cleanup.log ]] || + { echo "missing failure diagnostics" >&2; exit 1; } + # The private kubeconfig directory must disappear even after failures. + local kubeconfig + kubeconfig=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .kubeconfig' "$CALLS") + [[ ! -e $(dirname "$kubeconfig") ]] + if [[ $MODE == success || $MODE == success-again ]]; then + check ' + any(.[]; .args | index("--for=condition=Ready")) + and any(.[]; .args | index("deployment/tekton-kueue-controller-manager")) + and any(.[]; .args | index("deployment/tekton-kueue-webhook")) + and any(.[]; .args | index("--for=create")) + and any(.[]; .args | index("--for=condition=Admitted")) + and any(.[]; .args | index("--for=condition=Succeeded")) + and any(.[]; .args | index("pod/old-webhook")) + and any(.[]; .args | index("endpoints/tekton-kueue-webhook-service")) + and any(.[]; .args | index("https://github.com/cert-manager/cert-manager/releases/download/v1.19.2/cert-manager.yaml")) + and any(.[]; .args | index("https://github.com/kubernetes-sigs/kueue/releases/download/v0.16.6/manifests.yaml")) + and any(.[]; .args | index("https://github.com/tektoncd/pipeline/releases/download/v1.15.0/release.yaml")) + ' + cmp "$VERIFIED_MANIFEST" "$CASE_DIR/artifacts/release.yaml" + jq -e '.apiVersion == "tekton.dev/v1" and + (.spec.pipelineSpec.tasks[0].taskSpec.steps[0].image | contains("@sha256:"))' \ + "$CASE_DIR/pipelinerun.json" >/dev/null + fi + ;; + esac + cases=$((cases + 1)) + echo "PASS $MODE" +} + +for mode in digest mismatch malformed missing incomplete artifacts success success-again \ + creation readiness mutation wrong-pending endpoint workload admission run cleanup; do + run_case "$mode" +done +first=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success/calls.jsonl") +second=$(jq -r 'select(.command == "kind" and .args[0] == "create") | .args[] | select(startswith("release-smoke-"))' "$tmp/success-again/calls.jsonl") +[[ $first != "$second" ]] +echo "PASS $cases deterministic release smoke cases" diff --git a/hack/trigger-patch-release.sh b/hack/trigger-patch-release.sh index 38a22fcfa..563423efd 100644 --- a/hack/trigger-patch-release.sh +++ b/hack/trigger-patch-release.sh @@ -7,7 +7,6 @@ set -euo pipefail : "${PAC_CONTROLLER_URL:?}" : "${PAC_INCOMING_SECRET:?}" : "${PAC_REPOSITORY_NAME:?}" -: "${RELEASE_AS_LATEST:?}" : "${RELEASE_BRANCH:?}" : "${RELEASE_VERSION:?}" @@ -23,8 +22,7 @@ jq -n '{ pipelinerun: "release-patch", secret: env.PAC_INCOMING_SECRET, params: { - version: env.RELEASE_VERSION, - release_as_latest: env.RELEASE_AS_LATEST + version: env.RELEASE_VERSION } }' | curl --fail-with-body --silent --show-error \ -H "Content-Type: application/json" \ @@ -97,7 +95,8 @@ draft_url=$(jq -r '.html_url' <<<"$draft") echo echo "- External PipelineRun: ${details_url}" echo "- Draft release: ${draft_url}" - echo "- Publication: manual review required" + echo "- Finalization: approval, artifact verification, and installation smoke test required" + echo "- Latest image and bucket promotion: disabled during staging" } >> "$GITHUB_STEP_SUMMARY" -echo "::warning title=Draft release awaiting publication::Review and publish ${draft_url}" +echo "::warning title=Draft release awaiting finalization::Review ${draft_url}; publication and latest promotion require separate approval" diff --git a/hack/verify-release-artifacts.sh b/hack/verify-release-artifacts.sh new file mode 100644 index 000000000..5fff2bd38 --- /dev/null +++ b/hack/verify-release-artifacts.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fail() { + echo "release verification failed: $*" >&2 + exit 1 +} + +[[ $# == 5 ]] || fail "usage: $0 VERSION EXPECTED_COMMIT RELEASE_ID|bucket PUBLIC_KEY_FILE OUTPUT_DIR" +version=$1 +commit=$2 +release_id=$3 +key=$4 +output=$5 +repository=${GITHUB_REPOSITORY:-tektoncd/tekton-kueue} +image_repository=ghcr.io/tektoncd/tekton-kueue +source_url=https://github.com/tektoncd/tekton-kueue + +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] || fail "invalid version" +[[ $commit =~ ^[0-9a-f]{40}$ ]] || fail "invalid commit" +[[ $release_id == bucket || $release_id =~ ^[1-9][0-9]*$ ]] || fail "invalid release ID" +[[ $repository =~ ^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$ && + ${repository##*/} != . && ${repository##*/} != .. ]] || fail "invalid GitHub repository" +[[ -n ${GH_TOKEN:-} ]] || fail "GH_TOKEN is required" +[[ -f $key && -s $key && -r $key ]] || fail "a nonempty configured public key file is required" +[[ -n $output && ! -L $output ]] || fail "invalid output directory" +if [[ -e $output ]]; then + [[ -d $output ]] || fail "output must be a directory" + [[ -z $(find "$output" -mindepth 1 -print -quit) ]] || fail "output directory must be empty" +fi +for command in gh jq curl crane cosign openssl awk cmp; do + command -v "$command" >/dev/null || fail "missing command: $command" +done +openssl pkey -pubin -in "$key" -noout >/dev/null 2>&1 || fail "invalid configured public key" +if command -v sha256sum >/dev/null; then + checksum=(sha256sum) +else + command -v shasum >/dev/null || fail "sha256sum or shasum is required" + checksum=(shasum -a 256) +fi + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +receipt_id=$release_id +if [[ $release_id == bucket ]]; then + receipt_id=null + printf '{"draft":null,"assets":[]}\n' > "$work/release.json" + for file in release.yaml release.notags.yaml SHA256SUMS; do + curl --fail --silent --show-error --location \ + "https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$version/$file" \ + --output "$work/$file" + [[ -s $work/$file ]] || fail "empty bucket artifact: $file" + done +else + gh api --method GET "repos/$repository/releases/$release_id" > "$work/release.json" + jq -e --arg version "$version" --arg commit "$commit" --arg id "$release_id" ' + .id == ($id | tonumber) and .tag_name == $version and .target_commitish == $commit + and (.draft | type == "boolean") and (.assets | type == "array") + and ([.assets[] | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS")] + | length == 3 and (map(.name) | unique | length == 3) + and all(.[]; (.id | type == "number") and .id > 0 and .id == (.id | floor)) + and (map(.id) | unique | length == 3)) + ' "$work/release.json" >/dev/null || fail "release metadata does not match candidate" + + for file in release.yaml release.notags.yaml SHA256SUMS; do + asset_id=$(jq -r --arg name "$file" '.assets[] | select(.name == $name) | .id' "$work/release.json") + gh api --method GET -H "Accept: application/octet-stream" \ + "repos/$repository/releases/assets/$asset_id" > "$work/$file" + [[ -s $work/$file ]] || fail "empty asset: $file" + done +fi + +# Parse before hashing: never pass downloaded filenames to a checksum tool. +declare -A seen=() +while IFS= read -r line || [[ -n $line ]]; do + [[ $line =~ ^([0-9a-f]{64})[[:space:]][\ \*](release\.yaml|release\.notags\.yaml)$ ]] || + fail "unsafe or malformed SHA256SUMS" + hash=${BASH_REMATCH[1]} + file=${BASH_REMATCH[2]} + [[ -z ${seen[$file]:-} ]] || fail "duplicate checksum filename" + seen[$file]=1 + actual=$("${checksum[@]}" "$work/$file") + [[ ${actual%% *} == "$hash" ]] || fail "checksum mismatch: $file" +done < "$work/SHA256SUMS" +[[ ${#seen[@]} == 2 ]] || fail "SHA256SUMS must contain exactly the two manifests" +if [[ $release_id != bucket ]]; then + for file in release.yaml release.notags.yaml SHA256SUMS; do + curl --fail --silent --show-error --location \ + "https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$version/$file" \ + --output "$work/bucket-$file" + cmp -s "$work/$file" "$work/bucket-$file" || fail "bucket mismatch: $file" + done +fi +echo "PASS release metadata, checksums and bucket copies" + +# Kustomize emits block YAML with top-level metadata and scalar image fields. +# Require the controller and webhook deployments, not just an image somewhere. +for file in release.yaml release.notags.yaml; do + awk ' + /^---/ { kind = ""; name = "" } + /^kind: / { kind = $2 } + /^metadata:/ { metadata = 1; next } + /^[^ ]/ { metadata = 0 } + metadata && /^ name: / { name = $2 } + $1 == "image:" { print kind "\t" name "\t" $2 } + ' "$work/$file" > "$work/images" + count=0 + controller=0 + webhook=0 + while IFS=$'\t' read -r kind name reference; do + [[ $kind == Deployment ]] || fail "unexpected image outside a Deployment" + case "$name" in + tekton-kueue-controller-manager) controller=$((controller + 1)) ;; + tekton-kueue-webhook) webhook=$((webhook + 1)) ;; + *) fail "unexpected image deployment" ;; + esac + # Both variants must have the same digest; only release.yaml may have the version tag. + prefix=$image_repository + if [[ $file == release.yaml ]]; then + prefix="$prefix:$version" + fi + [[ $reference == "$prefix"@sha256:* ]] || fail "unexpected image repository or tag" + candidate=${reference##*@} + [[ $candidate =~ ^sha256:[0-9a-f]{64}$ ]] || fail "invalid image digest" + [[ ${digest:-$candidate} == "$candidate" ]] || fail "inconsistent manifest image digests" + digest=$candidate + count=$((count + 1)) + done < "$work/images" + [[ $count == 2 && $controller == 1 && $webhook == 1 ]] || + fail "both controller and webhook images are required" +done +image="$image_repository@$digest" + +crane manifest "$image" > "$work/index.json" +jq -e ' + [.manifests[].platform | .os + "/" + .architecture] | sort == + ["linux/amd64", "linux/arm64", "linux/ppc64le", "linux/s390x"] +' "$work/index.json" >/dev/null || fail "unexpected image platforms" +for platform in linux/amd64 linux/arm64 linux/ppc64le linux/s390x; do + crane config --platform "$platform" "$image" > "$work/config.json" + jq -e --arg source "$source_url" --arg platform "$platform" ' + (.os + "/" + .architecture) == $platform + and .config.Labels["org.opencontainers.image.source"] == $source + and .config.Labels["org.opencontainers.image.url"] == $source + ' "$work/config.json" >/dev/null || fail "incorrect OCI labels or platform: $platform" +done +echo "PASS manifest images, four platforms and OCI labels" + +cosign verify --key "$key" "$image" > "$work/signatures.json" +jq -e -s --arg digest "$digest" ' + [ .[] | if type == "array" then .[] else . end + | select(.critical.image["docker-manifest-digest"] == $digest) ] | length > 0 +' \ + "$work/signatures.json" >/dev/null || fail "no verified image signature" +cosign verify-attestation --key "$key" --type https://slsa.dev/provenance/v1 \ + "$image" > "$work/attestations.json" +# cosign emits verified DSSE envelopes as JSON objects (or an array). +jq -e -s --arg digest "${digest#sha256:}" --arg commit "$commit" ' + def repository: + # Chains uses git+https URIs with an optional ref suffix. + test("^(git\\+)?https://github\\.com/tektoncd/tekton-kueue(\\.git)?(@[^[:space:]]+)?$"); + def source: + (.uri // "" | repository) + and ((.digest.sha1 // .digest.gitCommit) == $commit); + [ .[] | if type == "array" then .[] else . end + | .payload as $payload | $payload | @base64d | fromjson + | select(._type == "https://in-toto.io/Statement/v1" + and .predicateType == "https://slsa.dev/provenance/v1") + | select(any(.subject[]; .digest.sha256 == $digest)) + | select( + any(.predicate.buildDefinition.resolvedDependencies[]?; source) + or (.predicate.buildDefinition.externalParameters.buildConfigSource + | (.repository // "" | repository) and .ref == ("sha1:" + $commit))) + | $payload + ] | if length > 0 then . else error("no matching verified provenance") end +' "$work/attestations.json" > "$work/verified-payloads.json" || + fail "no verified provenance matching source and digest" +echo "PASS configured-key signature and SLSA provenance" + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +bash "$root/hack/verify-release-tag.sh" "$version" "$commit" "$(jq -r '.draft != false' "$work/release.json")" + +jq -n --argjson release_id "$receipt_id" --arg version "$version" --arg commit "$commit" \ + --arg image "$image" --arg digest "$digest" --slurpfile release "$work/release.json" \ + '{release_id: $release_id, version: $version, commit: $commit, image: $image, + digest: $digest, draft: $release[0].draft, + assets: [$release[0].assets[] + | select(.name == "release.yaml" or .name == "release.notags.yaml" or .name == "SHA256SUMS") + | {name, id}] | sort_by(.name)}' > "$work/release-info.json" +mkdir -p "$output" +for file in release.yaml release.notags.yaml SHA256SUMS release-info.json verified-payloads.json; do + # Refuse to overwrite even if another process populated the output during verification. + (set -o noclobber; cat "$work/$file" > "$output/$file") +done +echo "PASS $version artifacts saved to $output" diff --git a/hack/verify-release-artifacts_test.sh b/hack/verify-release-artifacts_test.sh new file mode 100644 index 000000000..c1f38f3c9 --- /dev/null +++ b/hack/verify-release-artifacts_test.sh @@ -0,0 +1,313 @@ +#!/usr/bin/env bash + +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +export FIXTURES="$work/fixtures" +mkdir -p "$work/bin" "$FIXTURES" +export PATH="$work/bin:$PATH" +export GH_TOKEN=test-token +unset GITHUB_REPOSITORY +export VERSION=v0.4.3 +export COMMIT=0123456789abcdef0123456789abcdef01234567 +export DIGEST=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +export IMAGE="ghcr.io/tektoncd/tekton-kueue@sha256:$DIGEST" + +# A fixed Ed25519 public key; openssl (not a fake) validates this boundary. +cat > "$work/key.pub" <<'EOF' +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEA11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo= +-----END PUBLIC KEY----- +EOF +write_artifacts() { +for variant in release.yaml release.notags.yaml; do + reference=$IMAGE + [[ $variant != release.yaml ]] || reference="ghcr.io/tektoncd/tekton-kueue:$VERSION@sha256:$DIGEST" + for deployment in controller-manager webhook; do + name=$deployment + candidate=$reference + case "${SCENARIO:-success}" in + wrong-image-repository) candidate=${candidate/tektoncd\/tekton-kueue/other\/tekton-kueue} ;; + inconsistent-image) [[ $deployment != webhook ]] || candidate=${candidate/$DIGEST/${DIGEST//a/b}} ;; + missing-webhook) [[ $deployment != webhook ]] || continue ;; + wrong-deployment) [[ $deployment != webhook ]] || name=other ;; + tag-in-notags) [[ $variant != release.notags.yaml ]] || candidate="ghcr.io/tektoncd/tekton-kueue:$VERSION@sha256:$DIGEST" ;; + esac + cat < "$FIXTURES/$variant" +done +( + cd "$FIXTURES" + if command -v sha256sum >/dev/null; then + sha256sum release.yaml release.notags.yaml + else + shasum -a 256 release.yaml release.notags.yaml + fi +) > "$FIXTURES/SHA256SUMS" +} + +# Only network/registry/signing executables are replaced. JSON parsing, hashing, +# key parsing, manifest extraction and byte comparisons use the real commands. +cat > "$work/bin/fake" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +case "${0##*/}" in + gh) + if [[ $* == "api --paginate --slurp repos/tektoncd/tekton-kueue/git/matching-refs/tags/$VERSION" ]]; then + [[ $SCENARIO != tag-api-failure ]] || exit 42 + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + if $scenario == "missing-tag" or $scenario == "published-missing-tag" then [[]] + else [[{ref:("refs/tags/" + $version),object:{ + type:(if $scenario == "annotated-tag" or $scenario == "wrong-annotated-tag" then "tag" else "commit" end), + sha:(if $scenario == "wrong-tag" then "f"*40 else $commit end)}}]] end' + exit + elif [[ $* == "api repos/tektoncd/tekton-kueue/git/tags/$COMMIT" ]]; then + jq -n --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {object:{type:"commit",sha:(if $scenario == "wrong-annotated-tag" then "f"*40 else $commit end)}}' + exit + fi + [[ ${VERIFY_MODE:-123} != bucket ]] || exit 90 + [[ $1 == api && $2 == --method && $3 == GET ]] || exit 90 + if [[ $4 == repos/tektoncd/tekton-kueue/releases/123 ]]; then + if [[ $SCENARIO == malformed-metadata ]]; then printf '{\n'; exit; fi + jq -n --arg version "$VERSION" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {id:123,tag_name:$version,target_commitish:$commit,draft:true,assets:[ + {id:11,name:"release.yaml"},{id:12,name:"release.notags.yaml"},{id:13,name:"SHA256SUMS"}]} + | if $scenario == "published" or $scenario == "published-missing-tag" then .draft = false + elif $scenario == "metadata-id" then .id = 124 + elif $scenario == "metadata-tag" then .tag_name = "v0.4.2" + elif $scenario == "metadata-commit" then .target_commitish = "main" + elif $scenario == "metadata-draft" then .draft = "true" + elif $scenario == "metadata-assets" then .assets = null + elif $scenario == "duplicate-asset" then .assets += [.assets[0]] + elif $scenario == "missing-asset" then .assets |= .[0:2] + elif $scenario == "unsafe-asset-id" then .assets[0].id = "../evil" + elif $scenario == "duplicate-asset-id" then .assets[1].id = 11 + else . end' + else + [[ $4 == -H && $5 == "Accept: application/octet-stream" ]] || exit 90 + case "$6" in + repos/tektoncd/tekton-kueue/releases/assets/11) file=release.yaml ;; + repos/tektoncd/tekton-kueue/releases/assets/12) file=release.notags.yaml ;; + repos/tektoncd/tekton-kueue/releases/assets/13) file=SHA256SUMS ;; + *) exit 90 ;; + esac + if [[ $file == SHA256SUMS ]]; then + case "$SCENARIO" in + checksum) printf '%064d release.yaml\n' 0; exit ;; + traversal) printf '%s ../release.yaml\n' "$DIGEST"; exit ;; + absolute) printf '%s /tmp/release.yaml\n' "$DIGEST"; exit ;; + extra-checksum) cat "$FIXTURES/$file"; printf '%s extra.yaml\n' "$DIGEST"; exit ;; + duplicate-checksum) cat "$FIXTURES/$file" "$FIXTURES/$file"; exit ;; + missing-checksum) awk 'NR == 1' "$FIXTURES/$file"; exit ;; + empty-checksum) exit ;; + esac + fi + cat "$FIXTURES/$file" + fi + ;; + curl) + [[ $# == 7 && $1 == --fail && $2 == --silent && $3 == --show-error && + $4 == --location && $6 == --output ]] || exit 90 + prefix="https://infra.tekton.dev/tekton-releases/tekton-kueue/previous/$VERSION/" + [[ $5 == "$prefix"* ]] || exit 90 + file=${5#"$prefix"} + case "$file" in release.yaml|release.notags.yaml|SHA256SUMS) ;; *) exit 90 ;; esac + [[ $SCENARIO != "missing-bucket-$file" ]] || exit 22 + cat "$FIXTURES/$file" > "$7" + [[ $SCENARIO != "empty-bucket-$file" ]] || : > "$7" + [[ $SCENARIO != "bucket-$file" ]] || printf '\nchanged\n' >> "$7" + ;; + crane) + if [[ $1 == manifest && $2 == "$IMAGE" && $# == 2 ]]; then + jq -n --arg scenario "$SCENARIO" ' + ["amd64","arm64","ppc64le","s390x"] + | if $scenario == "missing-platform" then .[0:3] + elif $scenario == "duplicate-platform" then . + ["amd64"] + elif $scenario == "wrong-platform" then .[3] = "riscv64" + else . end + | {schemaVersion:2, manifests:map({digest:("sha256:" + ("b" * 64)), + platform:{os:"linux",architecture:.}})}' + elif [[ $1 == config && $2 == --platform && $4 == "$IMAGE" && $# == 4 ]]; then + jq -n --arg platform "$3" --arg scenario "$SCENARIO" ' + {os:"linux",architecture:($platform | split("/")[1]),config:{Labels:{ + "org.opencontainers.image.source":"https://github.com/tektoncd/tekton-kueue", + "org.opencontainers.image.url":"https://github.com/tektoncd/tekton-kueue"}}} + | if $platform == "linux/s390x" then + if $scenario == "bad-source-label" then .config.Labels["org.opencontainers.image.source"] = "https://example.com" + elif $scenario == "bad-url-label" then del(.config.Labels["org.opencontainers.image.url"]) + elif $scenario == "bad-config-platform" then .os = "windows" + else . end + else . end' + else exit 90; fi + ;; + cosign) + [[ $2 == --key && -s $3 ]] || exit 90 + case "$1" in + verify) + [[ $# == 4 && $4 == "$IMAGE" ]] || exit 90 + [[ $SCENARIO != signature-failure ]] || exit 1 + [[ $SCENARIO != empty-signature ]] || exit + printf '[{"critical":{"image":{"docker-manifest-digest":"sha256:%s"}}}]\n' "$DIGEST" + ;; + verify-attestation) + [[ $# == 6 && $4 == --type && $5 == https://slsa.dev/provenance/v1 && $6 == "$IMAGE" ]] || exit 90 + [[ $SCENARIO != attestation-failure ]] || exit 1 + [[ $SCENARIO != empty-attestation ]] || exit + if [[ $SCENARIO == malformed-attestation ]]; then + printf '{"payload":"invalid base64"}\n' + exit + fi + jq -n --arg digest "$DIGEST" --arg commit "$COMMIT" --arg scenario "$SCENARIO" ' + {_type:"https://in-toto.io/Statement/v1",predicateType:"https://slsa.dev/provenance/v1", + subject:[{name:"ghcr.io/tektoncd/tekton-kueue",digest:{sha256:$digest}}], + predicate:{buildDefinition:{resolvedDependencies:[ + {uri:"git+https://github.com/tektoncd/tekton-kueue.git@refs/tags/v0.4.3",digest:{sha1:$commit}}]}}} + | if $scenario == "bad-source" then .predicate.buildDefinition.resolvedDependencies[0].uri = "https://github.com/tektoncd/tekton-kueue-evil" + elif $scenario == "bad-commit" then .predicate.buildDefinition.resolvedDependencies[0].digest.sha1 = ("f" * 40) + elif $scenario == "bad-digest" then .subject[0].digest.sha256 = ("b" * 64) + elif $scenario == "bad-predicate" then .predicateType = "https://slsa.dev/provenance/v0.2" + elif $scenario == "build-config-source" then + .predicate.buildDefinition.externalParameters.buildConfigSource = + {repository:"git+https://github.com/tektoncd/tekton-kueue",ref:("sha1:" + $commit),path:"tekton/release-pipeline.yaml"} + | del(.predicate.buildDefinition.resolvedDependencies) + else . end + | if $scenario == "multiple-attestations" then + (.subject[0].digest.sha256 = ("b" * 64)), . + else . end + | {payloadType:"application/vnd.in-toto+json",payload:(tojson | @base64),signatures:[{sig:"verified-by-fake-cosign"}]} + | if $scenario == "array-attestations" then [.] else . end' + ;; + *) exit 90 ;; + esac + ;; + *) exit 90 ;; +esac +EOF +chmod +x "$work/bin/fake" +for executable in gh curl crane cosign; do + ln -s fake "$work/bin/$executable" +done + +run_verifier() { + bash "$root/hack/verify-release-artifacts.sh" \ + "$VERSION" "$COMMIT" "${VERIFY_MODE:-123}" "$work/key.pub" "$work/output" +} + +passed=0 +for SCENARIO in success published build-config-source multiple-attestations array-attestations \ + annotated-tag missing-tag wrong-tag wrong-annotated-tag published-missing-tag tag-api-failure \ + checksum traversal absolute extra-checksum duplicate-checksum missing-checksum empty-checksum \ + bucket-release.yaml bucket-release.notags.yaml bucket-SHA256SUMS \ + malformed-metadata metadata-id metadata-tag metadata-commit metadata-draft metadata-assets \ + duplicate-asset missing-asset unsafe-asset-id duplicate-asset-id \ + wrong-image-repository inconsistent-image missing-webhook wrong-deployment tag-in-notags \ + missing-platform duplicate-platform wrong-platform bad-source-label bad-url-label bad-config-platform \ + signature-failure empty-signature attestation-failure empty-attestation malformed-attestation \ + bad-source bad-commit bad-digest bad-predicate; do + export SCENARIO + rm -rf "$work/output" + write_artifacts + status=0 + run_verifier > "$work/log" 2>&1 || status=$? + case "$SCENARIO" in + success|published|build-config-source|multiple-attestations|array-attestations|annotated-tag|missing-tag) + if [[ $status != 0 ]]; then cat "$work/log" >&2; exit 1; fi + for file in release.yaml release.notags.yaml SHA256SUMS; do + cmp "$FIXTURES/$file" "$work/output/$file" + done + jq -e --arg commit "$COMMIT" --arg image "$IMAGE" --arg scenario "$SCENARIO" \ + --arg version "$VERSION" --arg digest "sha256:$DIGEST" ' + . == {release_id:123,version:$version,commit:$commit,image:$image,digest:$digest, + draft:($scenario != "published"),assets:[ + {name:"SHA256SUMS",id:13},{name:"release.notags.yaml",id:12}, + {name:"release.yaml",id:11}]}' "$work/output/release-info.json" >/dev/null + jq -e --arg digest "$DIGEST" ' + [.[] | @base64d | fromjson + | select(any(.subject[]; .digest.sha256 == $digest))] + | length > 0' "$work/output/verified-payloads.json" >/dev/null + ;; + *) + [[ $status != 0 && ! -e $work/output/release-info.json ]] || { + echo "unexpected verification success: $SCENARIO" >&2; exit 1; + } + ;; + esac + passed=$((passed + 1)) +done + +export VERIFY_MODE=bucket +for SCENARIO in success build-config-source signature-failure attestation-failure \ + bad-source bad-commit bad-digest missing-platform bad-source-label \ + wrong-image-repository inconsistent-image missing-webhook \ + bucket-release.yaml bucket-release.notags.yaml bucket-SHA256SUMS \ + missing-bucket-release.yaml missing-bucket-release.notags.yaml missing-bucket-SHA256SUMS \ + empty-bucket-release.yaml empty-bucket-release.notags.yaml empty-bucket-SHA256SUMS; do + export SCENARIO + rm -rf "$work/output" + write_artifacts + status=0 + run_verifier > "$work/log" 2>&1 || status=$? + case "$SCENARIO" in + success|build-config-source) + [[ $status == 0 ]] || { cat "$work/log" >&2; exit 1; } + jq -e --arg commit "$COMMIT" --arg image "$IMAGE" --arg version "$VERSION" \ + --arg digest "sha256:$DIGEST" ' + . == {release_id:null,version:$version,commit:$commit,image:$image, + digest:$digest,draft:null,assets:[]}' "$work/output/release-info.json" >/dev/null + for file in release.yaml release.notags.yaml SHA256SUMS; do + cmp "$FIXTURES/$file" "$work/output/$file" + done + ;; + *) + [[ $status != 0 && ! -e $work/output/release-info.json ]] || { + echo "unexpected bucket verification success: $SCENARIO" >&2; exit 1; + } + ;; + esac + passed=$((passed + 1)) +done +unset VERIFY_MODE +export SCENARIO=success +expect_failure() { + if "$@" > "$work/log" 2>&1; then + echo "unexpected boundary acceptance" >&2 + exit 1 + fi + passed=$((passed + 1)) +} +verifier="$root/hack/verify-release-artifacts.sh" +rm -rf "$work/output" +expect_failure bash "$verifier" +expect_failure bash "$verifier" ../v0.4.3 "$COMMIT" 123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "${COMMIT^^}" 123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" ../123 "$work/key.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/missing.pub" "$work/output" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "" +expect_failure env GH_TOKEN= bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +expect_failure env GITHUB_REPOSITORY=../repo bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/key.pub" "$work/output" +touch "$work/empty.pub" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/empty.pub" "$work/output" +printf 'not a public key\n' > "$work/invalid.pub" +expect_failure bash "$verifier" "$VERSION" "$COMMIT" 123 "$work/invalid.pub" "$work/output" +mkdir "$work/output" +printf 'existing artifact\n' > "$work/output/release.yaml" +expect_failure run_verifier +[[ $(< "$work/output/release.yaml") == "existing artifact" ]] +echo "PASS $passed release artifact verification cases" diff --git a/hack/verify-release-config.sh b/hack/verify-release-config.sh index 9513a9676..0aad70ffa 100644 --- a/hack/verify-release-config.sh +++ b/hack/verify-release-config.sh @@ -25,12 +25,33 @@ test -n "$go_image" grep -Fq "apk add --no-cache bash make" "$root/tekton/publish.yaml" grep -Fq 'org.opencontainers.image.source=https://$PACKAGE' "$root/tekton/publish.yaml" grep -Fq 'org.opencontainers.image.url=https://$PACKAGE' "$root/tekton/publish.yaml" +if grep -Fq '/latest/' "$root/tekton/release-pipeline.yaml" || + grep -Eq 'crane (cp|copy|tag).*latest' "$root/tekton/publish.yaml" || + grep -Fq '$(params.releaseAsLatest)' "$root/tekton/release-pipeline.yaml"; then + echo "release staging must not promote latest" >&2 + exit 1 +fi +for caller in "$root/.tekton/release.yaml" "$root/.tekton/release-patch.yaml"; do + if grep -Fq 'name: releaseAsLatest' "$caller"; then + echo "release entry points must not accept latest promotion" >&2 + exit 1 + fi +done "$container_tool" run --rm --platform "$platform" \ -v "$root:/workspace/source" \ "$publish_image" sh -ec ' apk add --no-cache bash make >/dev/null cd /workspace/source + # Exercise the shipped guard, not a copy of its condition. + awk '\'' + $0 == " - name: publish-image-and-manifests" { step = 1 } + step && $0 == " script: |" { script = 1; next } + script { sub(/^ /, ""); print } + '\'' tekton/publish.yaml > /tmp/publish.sh + status=0 + VERSION_TAG=v0.0.0 RELEASE_AS_LATEST=true sh /tmp/publish.sh || status=$? + test "$status" -eq 64 make release \ KUSTOMIZE=/go/bin/kustomize \ RELEASE_DIR=/tmp/release \ diff --git a/hack/verify-release-tag.sh b/hack/verify-release-tag.sh new file mode 100644 index 000000000..1aa05b686 --- /dev/null +++ b/hack/verify-release-tag.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +fail() { echo "release tag verification failed: $*" >&2; exit 1; } +[[ $# == 3 ]] || fail "usage: $0 VERSION COMMIT ALLOW_MISSING" +version=$1 +commit=$2 +allow_missing=$3 +repository=${GITHUB_REPOSITORY:-tektoncd/tekton-kueue} +[[ $version =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ && $commit =~ ^[0-9a-f]{40}$ && + $repository =~ ^[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+$ && + ( $allow_missing == true || $allow_missing == false ) ]] || fail "invalid candidate" +object=$(gh api --paginate --slurp "repos/$repository/git/matching-refs/tags/$version" | + jq -c --arg ref "refs/tags/$version" ' + [.[][] | select(.ref == $ref)] + | if length <= 1 then .[0].object // null else error("ambiguous tag") end') +if [[ $object == null ]]; then + [[ $allow_missing == true ]] || fail "published release tag is missing" + exit 0 +fi +while :; do + sha=$(jq -r .sha <<< "$object") + [[ $sha =~ ^[0-9a-f]{40}$ ]] || fail "invalid tag object" + case $(jq -r .type <<< "$object") in + commit) + [[ $sha == "$commit" ]] || fail "tag points to another source" + exit 0 + ;; + tag) + object=$(gh api "repos/$repository/git/tags/$sha" | jq -ce '.object') + ;; + *) fail "tag does not resolve to a commit" ;; + esac +done diff --git a/tekton/publish.yaml b/tekton/publish.yaml index bc4413fae..e339d0644 100644 --- a/tekton/publish.yaml +++ b/tekton/publish.yaml @@ -31,7 +31,8 @@ spec: - name: imageRegistryUser default: tekton-robot - name: releaseAsLatest - default: "true" + description: Must be false; latest promotion belongs to release finalization. + default: "false" - name: platforms default: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le - name: serviceAccountPath @@ -90,17 +91,17 @@ spec: #!/usr/bin/env sh set -eu - apk add --no-cache bash make - printf '%s\n' "$VERSION_TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' || { echo "invalid release version: $VERSION_TAG" >&2 exit 1 } case "$RELEASE_AS_LATEST" in - true|false) ;; - *) echo "releaseAsLatest must be true or false" >&2; exit 1 ;; + false) ;; + *) echo "latest promotion requires the release finalization workflow" >&2; exit 64 ;; esac + apk add --no-cache bash make + OUTPUT_RELEASE_DIR="$OUTPUT_ROOT/$VERSION_TAG" git config --global --add safe.directory "$PROJECT_ROOT" mkdir -p "$HOME/.docker" "$OUTPUT_RELEASE_DIR/template" @@ -139,15 +140,3 @@ spec: *) echo "unexpected release image: $image" >&2; exit 1 ;; esac printf '%s,' "$image" > "$(results.IMAGES.path)" - - - name: tag-latest - image: cgr.dev/chainguard/crane:latest-dev@sha256:dd9a345764e47517b80391648020b78c0ae6d4db0e392c7013b71e3817ed544c - script: | - #!/bin/sh - set -eu - - [ "$RELEASE_AS_LATEST" = "true" ] || exit 0 - mkdir -p "$HOME/.docker" - cp /workspace/docker-config.json "$HOME/.docker/config.json" - image=$(cat /workspace/built-images) - crane cp "$image" "${image%@*}:latest" diff --git a/tekton/release-cheat-sheet.md b/tekton/release-cheat-sheet.md index b6f6e8322..da31d8e9d 100644 --- a/tekton/release-cheat-sheet.md +++ b/tekton/release-cheat-sheet.md @@ -9,7 +9,7 @@ The automated steps below become operational only after the release Pipeline tra 1. Confirm the target version, LTS designation, end-of-life date, and compatibility ranges with the maintainers. 2. Confirm all intended changes are merged and their release-note blocks are complete. 3. From a clean checkout of the candidate commit, run generation, lint, unit, and end-to-end checks. -4. Run a release candidate with `releaseAsLatest=false` through the Tekton-controlled release Pipeline so it cannot update the `latest` image tag or artifacts. +4. Stage the release candidate through the Tekton-controlled release Pipeline. It publishes only versioned images and artifacts; it cannot update `latest`. 5. Verify both generated manifests reference only `ghcr.io/tektoncd/tekton-kueue` images by immutable digest. 6. Install and upgrade the candidate on the supported Kubernetes, Tekton Pipelines, and Kueue versions. @@ -23,7 +23,7 @@ The automated steps below become operational only after the release Pipeline tra ## Publish a patch release 1. Cherry-pick an approved fix to the supported release branch. -2. After the cherry-pick passes CI on the release branch, run the **Patch Release** GitHub workflow with the branch, next `vX.Y.Z` version, and whether it should update `latest`. +2. After the cherry-pick passes CI on the release branch, run the **Patch Release** GitHub workflow with the branch and next `vX.Y.Z` version. This stages a draft and does not update `latest`. 3. Monitor and verify the release as for an initial release. The scheduled patch workflow scans `release-v0.3.x` and newer for unreleased commits. Older release lines require manual dispatch. A release operator must still verify every resulting release before announcing it. @@ -41,6 +41,11 @@ Verify and record: Do not announce the release until every promised artifact is downloadable and independently verifiable. +Staging does not authorize publication or promotion. Those actions require +maintainer approval and verification of the finished release. Automated +approval-gated finalization is a separate rollout; staging alone leaves the +GitHub release as a draft and preserves the existing `latest` artifacts. + ## After the release 1. Update [`releases.md`](../releases.md) with the release date, exact end-of-life date, compatibility ranges, and release links. diff --git a/tekton/release-pipeline.yaml b/tekton/release-pipeline.yaml index 9518f7034..c54507356 100644 --- a/tekton/release-pipeline.yaml +++ b/tekton/release-pipeline.yaml @@ -27,9 +27,6 @@ spec: - name: releaseBucket description: bucket where the release is stored default: tekton-releases - - name: releaseAsLatest - description: Whether to tag and publish this release as latest - default: "false" - name: platforms description: Linux platforms to publish default: linux/amd64,linux/arm64,linux/s390x,linux/ppc64le @@ -97,7 +94,7 @@ spec: - name: url value: https://github.com/tektoncd/plumbing - name: revision - value: 0dc3174db86dcfa70724eced275ba4c1b2e25e76 + value: 39fbe8fe22b3336190d3f37fdb11c76eb2660419 - name: pathInRepo value: tekton/resources/release/base/prerelease_checks_oci.yaml params: @@ -107,6 +104,8 @@ spec: value: $(params.versionTag) - name: releaseBucket value: $(params.releaseBucket)/$(params.repoName) + - name: imageRepository + value: $(params.imageRegistry)/$(params.imageRegistryPath) workspaces: - name: source-to-release workspace: workarea @@ -163,7 +162,7 @@ spec: - name: imageRegistryUser value: $(params.imageRegistryUser) - name: releaseAsLatest - value: $(params.releaseAsLatest) + value: "false" - name: serviceAccountPath value: $(params.serviceAccountImagesPath) - name: platforms @@ -208,41 +207,6 @@ spec: - name: recursive value: "true" - - name: publish-to-bucket-latest - runAfter: [publish-images] - when: - - input: "$(params.releaseAsLatest)" - operator: in - values: ["true"] - taskRef: - resolver: bundles - params: - - name: bundle - value: ghcr.io/tektoncd/catalog/upstream/tasks/oracle-cloud-storage-upload@sha256:f1dd28da10dfb72ebbb3a5258e9f1ba6bfe82ee587311d708a7be3207f69e5e6 - - name: name - value: oracle-cloud-storage-upload - - name: kind - value: task - workspaces: - - name: credentials - workspace: release-secret - - name: source - workspace: workarea - subPath: bucket - params: - - name: path - value: $(params.versionTag) - - name: bucketName - value: $(params.releaseBucket) - - name: objectPrefix - value: $(params.repoName)/latest/ - - name: replaceExistingFiles - value: "true" - - name: recursive - value: "true" - - name: deleteExtraFiles - value: "true" - - name: report-bucket runAfter: [publish-to-bucket] params: @@ -365,7 +329,7 @@ spec: for uuid in ${CANDIDATES}; do KIND=$(curl -fsS "${REKOR_HOST}/api/v1/log/entries/${uuid}" \ | jq -r '.[].body' | base64 -d 2>/dev/null | jq -r '.kind' 2>/dev/null || true) - if [ "${KIND}" = "intoto" ]; then + if [ "${KIND}" = "intoto" ] || [ "${KIND}" = "dsse" ]; then REKOR_UUID="${uuid}" break fi @@ -373,19 +337,8 @@ spec: fi if [ -z "${REKOR_UUID}" ]; then - echo "WARNING: Could not find an in-toto attestation entry; falling back to the transparency annotation" - TRANSPARENCY=$(kubectl get taskrun "${TASKRUN_NAME}" -n "${NAMESPACE}" \ - -o jsonpath='{.metadata.annotations.chains\.tekton\.dev/transparency}' 2>/dev/null || echo "") - if [ -n "${TRANSPARENCY}" ]; then - REKOR_UUID=$(curl -fsS "${TRANSPARENCY}" | jq -r 'keys[0] // empty' || true) - if [ -z "${REKOR_UUID}" ]; then - REKOR_UUID=$(echo "${TRANSPARENCY}" | grep -oE 'logIndex=[0-9]+' | cut -d= -f2) - fi - fi - fi - - if [ -z "${REKOR_UUID}" ]; then - REKOR_UUID="unknown" + echo "ERROR: Could not find an intoto or dsse attestation entry for ${IMAGE_DIGEST}" + exit 1 fi echo "Rekor UUID: ${REKOR_UUID}" printf '%s' "${REKOR_UUID}" > "$(results.rekor-uuid.path)" @@ -491,7 +444,7 @@ spec: - name: url value: https://github.com/tektoncd/plumbing - name: revision - value: 894710e292fcb6b17276af01a2bc318107eef8a7 + value: df071e78c5b1a58867435efc7a9b295688d58969 - name: pathInRepo value: tekton/resources/release/base/github_release_oci.yaml params: diff --git a/test/release-smoke/README.md b/test/release-smoke/README.md new file mode 100644 index 000000000..13d5a9b6b --- /dev/null +++ b/test/release-smoke/README.md @@ -0,0 +1,23 @@ +# Finished-release smoke test + +Run the published artifact verifier first, using an independently configured +signing public key. Pass its saved manifest and immutable image reference to: + +```sh +bash hack/smoke-release.sh VERIFIED_MANIFEST EXPECTED_IMAGE ARTIFACT_DIR +``` + +The script requires Bash, Kind, kubectl, jq, OpenSSL and a working container +provider. It creates its own cluster and kubeconfig, installs the manifest +without rebuilding its image, checks webhook mutation and Kueue admission, +and waits for a PipelineRun to succeed. It saves diagnostics and deletes only +the cluster it created, including on failure. The artifact directory must be +empty. + +The pull-request workflow exercises the existing public v0.4.3 manifest, +pinned by SHA-256 and image digest, on an isolated runner. This installation +check is not a signing-key trust bootstrap. The finalization workflow will +call the same script with the candidate that passed signature verification. + +`release.yaml` in this directory is an offline parsing fixture for +`bash hack/smoke-release_test.sh`; it is not an installable release. diff --git a/test/release-smoke/release.yaml b/test/release-smoke/release.yaml new file mode 100644 index 000000000..f45caa1db --- /dev/null +++ b/test/release-smoke/release.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: tekton-kueue +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tekton-kueue-controller-manager + namespace: tekton-kueue +spec: + selector: + matchLabels: + app: controller + template: + metadata: + labels: + app: controller + spec: + containers: + - name: manager + image: ghcr.io/tektoncd/tekton-kueue:v0.4.3@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tekton-kueue-webhook + namespace: tekton-kueue +spec: + selector: + matchLabels: + app: webhook + template: + metadata: + labels: + app: webhook + spec: + containers: + - name: webhook + image: ghcr.io/tektoncd/tekton-kueue@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa