ci: validate Helm charts integrity nightly (#4052)

aabouzaid · web-flow · commit 426afda8cd6f · 2025-09-01T18:16:46.000+02:00
diff --git a/.github/workflows/chart-release-artifact-verify.yaml b/.github/workflows/chart-release-artifact-verify.yaml
@@ -0,0 +1,150 @@
+name: "Test - Chart Released Artifact Verification"
+
+on:
+  schedule:
+    - cron: "0 8 * * *"
+  workflow_dispatch: { }
+
+permissions:
+  contents: read
+
+jobs:
+  verify:
+    name: Verify released chart artifacts
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # v3.9.2
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+        with:
+          fetch-depth: 0
+      - name: Get chart versions
+        id: get-chart-versions
+        uses: ./.github/actions/get-chart-versions
+      - name: ⭐ Verify artifacts ⭐
+        id: verify-artifacts
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        # NOTE: Once we move to Helm OCI repo, the logic here will be simpler
+        #       as Cosign can work directly with the OCI registry.
+        run: |
+          # Track verification results.
+          skipped_tags=()
+          skipped_verifications=()
+          failed_verifications=()
+          successful_verifications=()
+
+          # Loop over the chart versions to check the artifact integrity.
+          for camunda_version in ${{ steps.get-chart-versions.outputs.active }}; do
+            echo ""
+            echo "##################################################"
+            echo "# 🏢 Camunda ${camunda_version}"
+            echo "##################################################"
+
+            chart_version_prefix="camunda-platform-${camunda_version}"
+            chart_git_tags=$(git tag -l | grep "${chart_version_prefix}-")
+            for chart_tag_name in ${chart_git_tags}; do
+              chart_version="$(echo "${chart_tag_name}" | sed "s/${chart_version_prefix}-//")"
+              chart_cosign_verify_file="camunda-platform-${chart_version}-cosign-verify.sh"
+
+              # Print chart details.
+              echo ""
+              echo "📜 Chart ${chart_version}:"
+              echo "=========================="
+
+              echo "- 🏷️ Release tag \"${chart_tag_name}\"."
+
+              # Early return if the chart release doesn't exist.
+              if ! gh release view "${chart_tag_name}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then
+                echo "- ⚠️ Chart GitHub release does not exist."
+                skipped_tags+=("${chart_tag_name}")
+                continue
+              fi
+
+              echo "- 📥 Download release artifacts."
+              gh release download "${chart_tag_name}" --dir chart-release-artifacts
+
+              # Run only if the release has a verify file.
+              cd chart-release-artifacts
+
+              # Early return if the verify file is not present.
+              if [ ! -f "${chart_cosign_verify_file}" ]; then
+                echo "- ⛔ The chart release doesn't have a Cosign verification script."
+                skipped_verifications+=("${chart_tag_name}")
+                continue
+              fi
+
+              # Run the Cosign verification script.
+              echo "- 🔐 Running Cosign verification"
+              if bash "${chart_cosign_verify_file}"; then
+                echo "- ✅ Cosign verification successful."
+                successful_verifications+=("${chart_tag_name}")
+              else
+                echo "- ❌ Cosign verification failed."
+                cat "${chart_cosign_verify_file}"
+                failed_verifications+=("${chart_tag_name}")
+              fi
+
+              # Clean up.
+              cd ..
+              rm -rf chart-release-artifacts
+
+            done # End of chart version loop.
+          done # End of Camunda minor version loop.
+
+          echo "##################################################"
+          echo "# Export outputs."
+          echo "##################################################"
+          echo "skipped-tags-count=${#skipped_tags[@]}" | tee >> $GITHUB_OUTPUT
+          echo "skipped-verifications-count=${#skipped_verifications[@]}" | tee >> $GITHUB_OUTPUT
+          echo "failed-verifications-count=${#failed_verifications[@]}" | tee >> $GITHUB_OUTPUT
+          echo "successful-verifications-count=${#successful_verifications[@]}" | tee >> $GITHUB_OUTPUT
+
+          echo ""
+          echo "##################################################"
+          echo "# 📊 Verification Summary"
+          echo "##################################################"
+
+          echo -e "\n⚠️ Skipped tags (${#skipped_tags[@]}):"
+          printf -- "- %s\n" "${skipped_tags[@]:-🎉 No skipped tags}"
+
+          echo -e "\n⛔ Skipped verifications (${#skipped_verifications[@]}):"
+          printf -- "- %s\n" "${skipped_verifications[@]:-🎉 No skipped verifications}"
+
+          echo -e "\n❌ Failed verifications (${#failed_verifications[@]}):"
+          printf -- "- %s\n" "${failed_verifications[@]:-🎉 No failed verifications}"
+
+          echo -e "\n✅ Successful verifications (${#successful_verifications[@]}):"
+          printf -- "- %s\n" "${successful_verifications[@]:-⚠️ No successful verifications ⚠️}"
+
+          if [[ ${#failed_verifications[@]} -gt 0 || ${#successful_verifications[@]} -eq 0 ]]; then
+            exit 1
+          fi
+      # NOTE: If needed, we can limit the scope of the Slack notification to failed verifications only.
+      - name: 🚨 Notify the team via Slack 🚨
+        if: failure()
+        uses: slackapi/slack-github-action@91efab103c0de0a537f72a35f6b8cda0ee76bf0a # v2.1.1
+        with:
+          webhook: ${{ secrets.SLACK_DISTRO_TEAM_WEBHOOK }}
+          webhook-type: incoming-webhook
+          payload: |
+            blocks:
+              - type: header
+                text:
+                  type: plain_text
+                  text: Chart Released Artifact Verification Failed
+                  emoji: true
+              - type: section
+                text:
+                  type: mrkdwn
+                  text: |
+                    🚨 *Action Required* 🚨
+                    There could be a compromised Helm chart artifact! Please investigate the issue immediately.
+
+                    *Summary:*
+                      • Failed verifications: ${{ steps.verify-artifacts.outputs.failed-verifications-count }}
+                      • Successful verifications: ${{ steps.verify-artifacts.outputs.successful-verifications-count }}
+                      • Skipped verifications: ${{ steps.verify-artifacts.outputs.skipped-verifications-count }}
+                      • Skipped tags: ${{ steps.verify-artifacts.outputs.skipped-tags-count }}
+
+                    <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Check failed workflow>
diff --git a/.github/workflows/chart-release.yaml b/.github/workflows/chart-release.yaml
@@ -100,11 +100,15 @@ jobs:
       contents: write
       id-token: write
     env:
-      CHART_RELEASER_CONFIG: ".github/config/chart-releaser.yaml"
+      CHART_RELEASER_CONFIG_FILE: ".github/config/chart-releaser.yaml"
       CHART_DIR: "charts/camunda-platform-${{ matrix.chart.dirID }}"
-      CHART_VERSION: "${{ matrix.chart.version }}"
-      CHART_PACKAGE_NAME: "camunda-platform-${{ matrix.chart.version }}"
-      CHART_TAG_NAME: "camunda-platform-${{ matrix.chart.appVersion }}-${{ matrix.chart.version }}"
+      CHART_RELEASE_VERSION: "${{ matrix.chart.version }}"
+      CHART_RELEASE_TAG_NAME: "camunda-platform-${{ matrix.chart.appVersion }}-${{ matrix.chart.version }}"
+      CHART_RELEASE_PACKAGE_FILE: "camunda-platform-${{ matrix.chart.version }}.tgz"
+      CHART_RELEASE_COSIGN_BUNDLE_FILE: "camunda-platform-${{ matrix.chart.version }}-cosign-bundle.json"
+      CHART_RELEASE_COSIGN_VERIFY_FILE: "camunda-platform-${{ matrix.chart.version }}-cosign-verify.sh"
+      CHART_RELEASE_COSIGN_CERTIFICATE_IDENTITY: "https://github.com/${{ github.workflow_ref }}"
+      CHART_RELEASE_COSIGN_CERTIFICATE_OIDC_ISSUER: "https://token.actions.githubusercontent.com"
     steps:
       # Init.
       - name: Checkout
@@ -165,65 +169,93 @@ jobs:
       # Using the chart-releaser CLI provides more flexibility and control over the release process.
       - name: Run Chart Releaser - Packaging
         run: |
-          helm-cr package ${{ env.CHART_DIR }} --config ${{ env.CHART_RELEASER_CONFIG }}
+          helm-cr package ${{ env.CHART_DIR }} --config ${{ env.CHART_RELEASER_CONFIG_FILE }}
       # Only keep the chart in the release process to avoid releasing untargeted charts.
       - name: Clean up packages
         run: |
           ls -lsa .cr-release-packages/*
           find .cr-release-packages/* \
-            -not -name "camunda-platform-${{ env.CHART_VERSION }}.tgz" \
+            -not -name "${{ env.CHART_RELEASE_PACKAGE_FILE }}" \
             -delete
       - name: Run Chart Releaser - Tagging/Uploading
         env:
           CR_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
-          helm-cr upload --config ${{ env.CHART_RELEASER_CONFIG }} \
+          helm-cr upload --config ${{ env.CHART_RELEASER_CONFIG_FILE }} \
             --push \
             --owner "${{ github.repository_owner }}" \
             --git-repo "$(basename ${{ github.repository }})" \
-            --release-name-template "${{ env.CHART_TAG_NAME }}"
+            --release-name-template "${{ env.CHART_RELEASE_TAG_NAME }}"
       - name: Run Chart Releaser - Indexing
         env:
           CR_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
-          helm-cr index --config ${{ env.CHART_RELEASER_CONFIG }} \
+          helm-cr index --config ${{ env.CHART_RELEASER_CONFIG_FILE }} \
             --push \
             --owner "${{ github.repository_owner }}" \
             --git-repo "$(basename ${{ github.repository }})" \
-            --release-name-template "${{ env.CHART_TAG_NAME }}"
+            --release-name-template "${{ env.CHART_RELEASE_TAG_NAME }}"
       - name: Set GitHub release type
         if: ${{ matrix.chart.prerelease }}
         env:
           GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
-          gh release edit "${{ env.CHART_TAG_NAME }}" \
+          gh release edit "${{ env.CHART_RELEASE_TAG_NAME }}" \
             --repo "${GITHUB_REPOSITORY}" \
             --prerelease
 
       # Sign and upload the signature.
       - name: Sign Helm chart with Cosign
+        working-directory: .cr-release-packages
         run: |
-          cosign sign-blob -y .cr-release-packages/${{ env.CHART_PACKAGE_NAME }}.tgz \
-            --bundle "${{ env.CHART_PACKAGE_NAME }}.cosign.bundle"
+          cosign sign-blob -y ${{ env.CHART_RELEASE_PACKAGE_FILE }} \
+            --bundle "${{ env.CHART_RELEASE_COSIGN_BUNDLE_FILE }}"
+      - name: Get Helm chart Cosign Rekor log index
+        working-directory: .cr-release-packages
+        run: |
+          rekor_log_index="$(cat ${{ env.CHART_RELEASE_COSIGN_BUNDLE_FILE }} | jq '.rekorBundle.Payload.logIndex')"
+          echo "CHART_RELEASE_COSIGN_REKOR_LOG_INDEX=${rekor_log_index}" >> $GITHUB_ENV
+      - name: Create the script to verify signed Helm chart with Cosign
+        working-directory: .cr-release-packages
+        run: |
+          cat << EOF > ${{ env.CHART_RELEASE_NAME }}-cosign-verify.sh
+          # Rekor.
+          echo "Rekor record:"
+          echo "https://search.sigstore.dev/?logIndex=${{ env.CHART_RELEASE_COSIGN_REKOR_LOG_INDEX }}"
+
+          # Cosign.
+          cosign verify-blob ${{ env.CHART_RELEASE_PACKAGE_FILE }} \\
+            --bundle "${{ env.CHART_RELEASE_COSIGN_BUNDLE_FILE }}" \\
+            --certificate-identity "${{ env.CHART_RELEASE_COSIGN_CERTIFICATE_IDENTITY }}" \\
+            --certificate-oidc-issuer "${{ env.CHART_RELEASE_COSIGN_CERTIFICATE_OIDC_ISSUER }}"
+          EOF
       - name: Verify signed Helm chart with Cosign
+        working-directory: .cr-release-packages
         run: |
-          cosign verify-blob .cr-release-packages/${{ env.CHART_PACKAGE_NAME }}.tgz \
-            --bundle "${{ env.CHART_PACKAGE_NAME }}.cosign.bundle" \
-            --certificate-identity "https://github.com/${GITHUB_WORKFLOW_REF}" \
-            --certificate-oidc-issuer "https://token.actions.githubusercontent.com"
-      - name: Upload Helm chart signature bundle
+          bash ${{ env.CHART_RELEASE_COSIGN_VERIFY_FILE }}
+      - name: Upload Helm chart Cosign bundle file
+        working-directory: .cr-release-packages
+        env:
+          GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
+        run: |
+          gh release upload "${{ env.CHART_RELEASE_TAG_NAME }}" \
+            "${{ env.CHART_RELEASE_COSIGN_BUNDLE_FILE }}" \
+            --repo "${GITHUB_REPOSITORY}"
+      - name: Upload Helm chart Cosign verify file
+        working-directory: .cr-release-packages
         env:
           GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
         run: |
-          gh release upload "${{ env.CHART_TAG_NAME }}" \
-            "${{ env.CHART_PACKAGE_NAME }}.cosign.bundle" \
+          gh release upload "${{ env.CHART_RELEASE_TAG_NAME }}" \
+            "${{ env.CHART_RELEASE_COSIGN_VERIFY_FILE }}" \
             --repo "${GITHUB_REPOSITORY}"
       - name: Add release info to workflow summary
         run: |
           echo "ℹ️ Release Published ℹ️" >> $GITHUB_STEP_SUMMARY
           cat << EOF >> $GITHUB_STEP_SUMMARY
-          - GitHub: https://github.com/${GITHUB_REPOSITORY}/releases/tag/${{ env.CHART_TAG_NAME }}
-          - Artifact Hub: https://artifacthub.io/packages/helm/camunda/camunda-platform/${{ env.CHART_VERSION }}
+          - GitHub: https://github.com/${GITHUB_REPOSITORY}/releases/tag/${{ env.CHART_RELEASE_TAG_NAME }}
+          - Artifact Hub: https://artifacthub.io/packages/helm/camunda/camunda-platform/${{ env.CHART_RELEASE_VERSION }}
+          - Rekor record: https://rekor.sigstore.dev/?logIndex=${{ env.CHART_RELEASE_COSIGN_REKOR_LOG_INDEX }}
           Note: Artifact Hub link needs some time till it's AH scraps the Helm repo index.
           EOF
 
@@ -261,13 +293,6 @@ jobs:
             helm
             helm-ct
             yq
-      # - name: Simple smoke test
-      #   uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3
-      #   with:
-      #     max_attempts: 3
-      #     timeout_minutes: 5
-      #     retry_wait_seconds: 10
-      #     command: make release.verify-components-version
       - name: Label PRs with app and chart version
         env:
           GH_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
diff --git a/scripts/generate-release-notes.sh b/scripts/generate-release-notes.sh
@@ -90,7 +90,7 @@ main () {
 
 release_notes_footer () {
     chart_dir="${1}"
-    export CHART_NAME_WITH_VERSION="$(yq '[.name, .version] | join("-")' "${chart_dir}/Chart.yaml")"
+    export CHART_RELEASE_NAME="$(yq '[.name, .version] | join("-")' "${chart_dir}/Chart.yaml")"
     export VERSION_MATRIX_RELEASE_HEADER="false"
     export VERSION_MATRIX_RELEASE_INFO="$(CHART_DIR=${chart_dir} make release.generate-version-matrix-unreleased)"
     echo "\nChart dir: $${chart_dir}";\
diff --git a/scripts/templates/release-notes/RELEASE-NOTES-FOOTER.md.tpl b/scripts/templates/release-notes/RELEASE-NOTES-FOOTER.md.tpl
@@ -4,11 +4,13 @@
 
 ### Verification
 
-To verify the integrity of the Helm chart using [Cosign](https://docs.sigstore.dev/signing/quickstart/):
+For quick verification of the Helm chart integrity using [Cosign](https://docs.sigstore.dev/signing/quickstart/):
 
 ```shell
-cosign verify-blob {{ getenv "CHART_NAME_WITH_VERSION" }}.tgz \
-  --bundle {{ getenv "CHART_NAME_WITH_VERSION" }}.cosign.bundle \
-  --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \
-  --certificate-identity "https://github.com/{{ getenv "GITHUB_WORKFLOW_REF" }}"
+cosign verify-blob {{ getenv "CHART_RELEASE_NAME" }}.tgz \
+  --bundle "{{ getenv "CHART_RELEASE_NAME" }}-cosign-bundle.json" \
+  --certificate-identity-regex "https://github.com/{{ getenv "GITHUB_REPOSITORY" }}" \
+  --certificate-oidc-issuer "https://token.actions.githubusercontent.com"
 ```
+
+For detailed verification instructions, check the steps in the `{{ getenv "CHART_RELEASE_NAME" }}-cosign-verify.sh` file.
