test-integration-runner.yaml

# Docs: https://github.com/camunda/camunda-platform-helm/blob/main/docs/gha-workflows.md
name: "Test - Integration - Runner"
on:
  workflow_call:
    inputs:
      helmChartVersion:
        description: Deploy from OCI registry using this version (overrides git-based chart packaging for install flow)
        required: false
        default: ""
        type: string
      values-config:
        type: string
        description: Some of the values files will allow subsitutions. For example, on the image.tags. This config supplies those values
        default: '{}'
        required: false
      auth:
        description: The authentication type to use for the tests.
        required: true
        default: keycloak
        type: string
      exclude:
        description: The test suites to exclude.
        required: false
        default: ""
        type: string
      identifier:
        description: The unique identifier of used in the deployment hostname.
        required: true
        type: string
      camunda-helm-repo:
        description: The Helm repo which is used during the upgrade flow.
        required: false
        default: camunda/camunda-platform
        type: string
      camunda-helm-dir:
        required: false
        default: camunda-platform-8.8
        type: string
      camunda-helm-git-ref:
        required: false
        default: main
        type: string
      camunda-helm-upgrade-version:
        description: The Helm chart version used in the upgrade-patch flow.
        required: false
        default: ""
        type: string
      camunda-version-previous:
        description: The Camunda version used in the upgrade-minor flow.
        required: false
        default: ""
        type: string
      caller-git-ref:
        required: false
        default: main
        type: string
      deployment-ttl:
        description: Define a TTL for the lifespan of the deployment.
        required: false
        default: "12h"
        type: string
      distro-platform:
        default: gke
        type: string
      distro-type:
        default: kubernetes
        type: string
      infra-type:
        description: Define the infrastructure that will be used to run the deployment.
        default: "preemptible"
        type: string
      flow:
        required: false
        default: install
        type: string
      test-enabled:
        required: false
        default: true
        type: boolean
      e2e-enabled:
        required: false
        default: false
        type: boolean
      extra-values:
        description: Pass extra values to the Helm chart.
        required: false
        type: string
      vault-secret-mapping:
        description: (optional) defines how to map Vault secrets to distro CI environment variables
        required: false
        type: string
      auth-data:
        required: false
        type: string
      cluster-type:
        required: false
        type: string
      test-case:
        required: false
        default: "pr"
        type: string
      scenario:
        required: false
        default: "elasticsearch"
        type: string
      # =============================================================================
      # Selection + Composition Model Inputs (New - Preferred)
      # These inputs allow explicit configuration of identity, persistence, platform,
      # and features instead of relying on scenario name parsing.
      # =============================================================================
      test-identity:
        description: "Identity selection: keycloak, keycloak-external, oidc, basic, hybrid"
        required: false
        type: string
      test-persistence:
        description: "Persistence selection: elasticsearch, opensearch, rdbms, rdbms-oracle"
        required: false
        type: string
      test-platform:
        description: "Platform selection: gke, eks, openshift"
        required: false
        type: string
      test-features:
        description: "Features (comma-separated): multitenancy, rba, documentstore"
        required: false
        type: string
      test-qa:
        description: "Enable QA configuration"
        required: false
        default: false
        type: boolean
      test-upgrade:
        description: "Enable upgrade flow configuration"
        required: false
        default: false
        type: boolean
      skip-e2e:
        description: "Skip E2E (Playwright smoke) tests for this scenario (declared in ci-test-config.yaml)"
        required: false
        default: false
        type: boolean
      skip-it:
        description: "Skip integration (Playwright IT) tests for this scenario (declared in ci-test-config.yaml)"
        required: false
        default: false
        type: boolean
      pr-head-sha:
        description: "PR HEAD commit SHA for recording cache results"
        required: false
        default: ""
        type: string
      camunda-version:
        description: "Camunda version (e.g., 8.9) for cache result recording"
        required: false
        default: ""
        type: string
      camunda-helm-post-render:
        description: Set to true if you would like to run the post-renderer script for OpenShift
        required: false
        default: true
        type: boolean
      camunda-helm-credentials-source:
        description: |
          Auto-generate credentials or copy them from external secret.
          Valid options: auto-generated or external-secret-only.
          New optional auto-generated secrets are only supported in Camunda 8.4 chart and above.
        required: false
        default: auto-generated
        type: string
      cluster-name:
        required: false
        type: string
      cluster-location:
        required: false
        type: string
      workload-identity-provider:
        required: false
        type: string
      service-account:
        required: false
        type: string
      server-url:
        required: false
        type: string
      username:
        required: false
        type: string
      password:
        required: false
        type: string
      e2e-test-pattern:
        required: false
        default: "**/smoke-tests.spec.{ts,js}"
        type: string
      run-all-e2e-tests:
        description: "Run all E2E tests (playwright)"
        required: false
        default: false
        type: boolean
      teleport-token:
        description: The Teleport token to use for authentication.
        required: false
        default: infra-ci-prod-github-action-distribution
        type: string
      shortname:
        description: The shortname of the scenario.
        required: true
        default: "eske"
        type: string
      namespace-prefix:
        description: The prefix for the namespace. This is necessary because we only have permissions to create namespaces with a specific prefix.
        required: false
        default: ""
        type: string
      values-enterprise:
        required: false
        default: "false"
        type: string
        description: Enable enterprise values
      values-digest:
        required: false
        default: "true"
        type: string
        description: Enable digest values
      always-delete-namespace:
        description: Always delete the namespace after the test run
        default: false
        type: boolean
        required: false
      initialClaimValue:
        description: This is only used for testing Entra. This is here so we can do exploratory testing with Entra.
        required: false
        default: ""
        type: string
      include-image-tags:
        description: Include the image tags file for substitution from values-config
        required: false
        default: false
        type: boolean
      helm-version:
        description: "Helm version to install. Empty means use the pre-baked Helm from the CI runner image."
        required: false
        default: ""
        type: string
    outputs:
      deployment-url:
        description: The URL of the deployed application.
        value: ${{ jobs.install.outputs.vars-ingress-host }}
      namespace:
        description: The namespace where the deployment is running.
        value: ${{ jobs.install.outputs.vars-namespace }}

permissions:
  contents: read
  id-token: write
  deployments: write
  packages: read
  statuses: write

# No concurrency block here — this is the innermost layer containing the cleanup job.
# cancel-in-progress at this level kills ALL jobs including cleanup, causing orphaned
# cloud resources. Cancellation is handled by outer caller workflows. See #5307.

env:
  # Vars with "CI_" prefix are used in the CI workflow only.
  # Vars with "TEST_" prefix are used in the test runner tool (Task).
  CI_DEPLOYMENT_TTL: ${{ inputs.deployment-ttl }}
  # The Helm repo which is used during the upgrade flow.
  TEST_CHART_REPO: ${{ inputs.camunda-helm-repo }}
  # Docker Hub auth to avoid image pull rate limit.
  TEST_CREATE_DOCKER_LOGIN_SECRET: "TRUE"
  # OIDC.
  INITIAL_CLAIM_VALUE: ${{ inputs.initialClaimValue }}
  # Mapping rule client IDs - defaults for Keycloak; overridden by entra.ensure-venom-app for OIDC.
  VENOM_CLIENT_ID: venom
  CONNECTORS_CLIENT_ID: connectors
  # Misc.
  # NOTE: Infrastructure values (CLUSTER_NAME, AWS_PROFILE, OPENSEARCH_HOST/PROTOCOL/PORT,
  # POSTGRESQL_JDBC_URL) are loaded from .github/config/infra.yaml by the workflow-vars action
  # (invoked via integration-test-setup) and exported to $GITHUB_ENV per-job.
  TELEPORT_TOKEN: ${{ inputs.teleport-token }}
  FLOW: ${{ inputs.flow }}

jobs:
  install:
    if: github.event.action != 'closed' && inputs.flow != 'modular-upgrade-minor'
    name: install for ${{ inputs.flow }} on ${{ inputs.distro-platform }} - ${{ inputs.shortname }}
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/camunda/team-distribution/ci-runner:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --user root
    defaults:
      run:
        shell: bash
    outputs:
      vars-identifier: ${{ steps.setup.outputs.identifier }}
      vars-ingress-host: ${{ steps.setup.outputs.ingress-host }}
      vars-namespace: ${{ steps.setup.outputs.namespace }}
      SHARD_TOTAL: ${{ env.SHARD_TOTAL }}
      SHARD_INDEX: ${{ env.SHARD_INDEX }}
    permissions:
      contents: read
      id-token: write
      deployments: write
      packages: read
    env:
      TEST_CLUSTER_TYPE: ${{ inputs.distro-type || inputs.cluster-type }}

    steps:
      - name: Info - ℹ️ Print workflow inputs ℹ️
        env:
          GITHUB_CONTEXT: ${{ toJson(inputs) }}
        run: |
          if [[ -n "${INITIAL_CLAIM_VALUE}" ]]; then
            echo "::add-mask::${INITIAL_CLAIM_VALUE}"
          fi
          echo "Workflow Inputs:"
          echo "${GITHUB_CONTEXT}" | jq '."extra-values" = "<Check below>"'
          echo "Workflow Inputs - Extra Values:"
          echo "${GITHUB_CONTEXT}" | jq -r '."extra-values"'

      - name: CI Setup - Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          # This is needed to load repo GH composite actions if the workflow triggered by workflow_call.
          repository: camunda/camunda-platform-helm
          ref: ${{ inputs.camunda-helm-git-ref }}

      - name: CI Setup - Configure Git safe directory
        run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"

      - name: Configure curl and wget
        uses: ./.github/actions/setup-curl

      # Tools (golang, helm, kubectl, oc, task, yq, zbctl) are pre-installed in the CI runner container image
      # See: .github/docker/ci-runner/Dockerfile
      # The per-scenario helm-version input below can override the pre-baked helm when set in ci-test-config.yaml.
      # When no override is provided, fall back to the helm version pinned in .tool-versions.

      - name: CI Setup - Resolve Helm version
        id: helm-resolve
        run: |
          version="${{ inputs.helm-version }}"
          if [[ -z "$version" ]]; then
            version="$(awk '$1=="helm"{print $2}' .tool-versions)"
          fi
          prebaked="$(helm version --short --template '{{.Version}}' 2>/dev/null | sed 's/^v//' | cut -d'+' -f1)"
          echo "version=$version" >> "$GITHUB_OUTPUT"
          if [[ "$version" == "$prebaked" ]]; then
            echo "needs-install=false" >> "$GITHUB_OUTPUT"
            echo "Pre-baked helm $prebaked matches requested $version — skipping override"
          else
            echo "needs-install=true" >> "$GITHUB_OUTPUT"
            echo "Pre-baked helm $prebaked != requested $version — installing override"
          fi

      - name: CI Setup - Override Helm version
        if: steps.helm-resolve.outputs.needs-install == 'true'
        env:
          HELM_VERSION: ${{ steps.helm-resolve.outputs.version }}
        run: |
          tarball="helm-v${HELM_VERSION}-linux-amd64.tar.gz"
          curl -fsSL "https://get.helm.sh/${tarball}" -o "/tmp/${tarball}"
          curl -fsSL "https://get.helm.sh/${tarball}.sha256sum" -o "/tmp/${tarball}.sha256sum"
          (cd /tmp && sha256sum --check "${tarball}.sha256sum")
          tar -xzf "/tmp/${tarball}" -C /tmp
          mv /tmp/linux-amd64/helm /usr/local/bin/helm
          rm -rf "/tmp/${tarball}" "/tmp/${tarball}.sha256sum" /tmp/linux-amd64
          helm version --short

      - name: CI Setup - Shared integration test setup
        id: setup
        uses: ./.github/actions/integration-test-setup
        with:
          vault-secret-mapping:        ${{ inputs.vault-secret-mapping }}
          distro-platform:             ${{ inputs.distro-platform }}
          auth-data:                   ${{ inputs.auth-data }}
          deployment-ttl:              ${{ env.CI_DEPLOYMENT_TTL }}
          flow:                        ${{ inputs.flow }}
          identifier:                  ${{ inputs.identifier }}
          camunda-helm-dir:            ${{ inputs.camunda-helm-dir }}
          camunda-helm-upgrade-version: ${{ inputs.camunda-helm-upgrade-version }}
          namespace-prefix:            ${{ inputs.namespace-prefix }}
          infra-type:                  ${{ inputs.infra-type }}
          values-enterprise:           ${{ inputs.values-enterprise }}
          values-digest:               ${{ inputs.values-digest }}
          camunda-version-previous:    ${{ inputs.camunda-version-previous }}
        env:
          VAULT_ADDR:       ${{ secrets.VAULT_ADDR }}
          VAULT_ROLE_ID:    ${{ secrets.VAULT_ROLE_ID }}
          VAULT_SECRET_ID:  ${{ secrets.VAULT_SECRET_ID }}
          GH_APP_ID:        ${{ secrets.GH_APP_ID_DISTRO_CI_MANAGE_GH_ENVS }}
          GH_APP_KEY:       ${{ secrets.GH_APP_PRIVATE_KEY_DISTRO_CI_MANAGE_GH_ENVS }}
          ROSA_URL:         ${{ secrets[inputs.server-url] }}
          ROSA_USER:        ${{ secrets[inputs.username] }}
          ROSA_PASS:        ${{ secrets[inputs.password] }}
          DISTRO_CI_DOCKER_USERNAME_DOCKERHUB: ${{ secrets.DISTRO_CI_DOCKER_USERNAME_DOCKERHUB }}
          DISTRO_CI_DOCKER_PASSWORD_DOCKERHUB: ${{ secrets.DISTRO_CI_DOCKER_PASSWORD_DOCKERHUB }}

      - name: CI Setup - Start GitHub deployment
        uses: bobheadxi/deployments@648679e8e4915b27893bd7dbc35cb504dc915bc8 # v1
        id: deployment
        continue-on-error: true # Deployment status is cosmetic; don't fail the job if the API 404s (e.g. merge queue refs).
        with:
          step: start
          token: ${{ steps.setup.outputs.cluster-auth-token }}
          env: ${{ steps.setup.outputs.identifier }}
          ref: ${{ inputs.caller-git-ref }}

      - name: Cluster Setup - Configure the namespace
        run: |
          echo $TEST_NAMESPACE
          # Check if namespace exists and if we own it (created by same run)
          if kubectl get ns $TEST_NAMESPACE &>/dev/null; then
            EXISTING_RUN_ID=$(kubectl get ns $TEST_NAMESPACE -o jsonpath='{.metadata.labels.github-run-id}' 2>/dev/null || echo "")
            if [[ "$EXISTING_RUN_ID" == "$GITHUB_WORKFLOW_RUN_ID" ]]; then
              echo "Namespace exists and was created by this run, deleting and recreating..."
              kubectl delete ns $TEST_NAMESPACE
            else
              echo "Namespace exists but was created by a different run (run-id: $EXISTING_RUN_ID), cleaning up stale release..."
              # Remove any stale Helm release left behind by a cancelled/completed previous run.
              # Without this, 'helm install' fails with "cannot re-use a name that is still in use".
              helm uninstall integration --namespace $TEST_NAMESPACE --wait --timeout 2m0s 2>/dev/null || true
            fi
          fi
          kubectl create ns $TEST_NAMESPACE --dry-run=client -o yaml | kubectl apply -f -
          kubectl label ns $TEST_NAMESPACE github-id=${{ steps.setup.outputs.identifier }} --overwrite
          kubectl label ns $TEST_NAMESPACE test-flow=${{ inputs.flow }} --overwrite
          kubectl label ns $TEST_NAMESPACE github-run-id=$GITHUB_WORKFLOW_RUN_ID --overwrite
          kubectl label ns $TEST_NAMESPACE github-job-id=$GITHUB_WORKFLOW_JOB_ID --overwrite
          kubectl label ns $TEST_NAMESPACE github-org=$(dirname $GITHUB_REPOSITORY) --overwrite
          kubectl label ns $TEST_NAMESPACE github-repo=$(basename $GITHUB_REPOSITORY) --overwrite
          kubectl annotate ns $TEST_NAMESPACE cleaner/ttl=${{ inputs.deployment-ttl || '1h' }} --overwrite
          kubectl annotate ns $TEST_NAMESPACE janitor/ttl=${{ inputs.deployment-ttl || '1h' }} --overwrite
          kubectl annotate ns $TEST_NAMESPACE camunda.cloud/ephemeral=true --overwrite
          kubectl annotate ns $TEST_NAMESPACE github-workflow-run-url=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID --overwrite
          kubectl get ns $TEST_NAMESPACE -o yaml

      - name: Apply Vault mapped Secret
        if: inputs.vault-secret-mapping != ''
        run: |
          kubectl -n "$TEST_NAMESPACE" apply -f /tmp/vault-mapped-secrets.yaml

      - name: Cluster Setup - Install docker registry on cluster
        run: |
          kubectl create secret docker-registry registry-camunda-cloud \
          --namespace $TEST_NAMESPACE \
          --docker-server "registry.camunda.cloud" \
          --docker-username $TEST_DOCKER_USERNAME_CAMUNDA_CLOUD \
          --docker-password $TEST_DOCKER_PASSWORD_CAMUNDA_CLOUD \
          --dry-run=client -o yaml | kubectl apply -f -

          kubectl create secret docker-registry ghcr-io \
          --namespace $TEST_NAMESPACE \
          --docker-server "ghcr.io" \
          --docker-username $GITHUB_ACTOR \
          --docker-password $GITHUB_TOKEN \
          --dry-run=client -o yaml | kubectl apply -f -

          # Docker Hub pull secret (replaces the legacy `task init.seed` lib taskfile target).
          # TEST_DOCKER_USERNAME/PASSWORD are exported by integration-test-setup from Vault.
          if [[ "${TEST_CREATE_DOCKER_LOGIN_SECRET:-FALSE}" == "TRUE" && -n "${TEST_DOCKER_USERNAME:-}" && -n "${TEST_DOCKER_PASSWORD:-}" ]]; then
            kubectl create secret docker-registry index-docker-io \
            --namespace $TEST_NAMESPACE \
            --docker-server "https://index.docker.io/v1/" \
            --docker-username "$TEST_DOCKER_USERNAME" \
            --docker-password "$TEST_DOCKER_PASSWORD" \
            --dry-run=client -o yaml | kubectl apply -f -
          fi
        env:
          TEST_DOCKER_USERNAME_CAMUNDA_CLOUD: ${{ env.TEST_DOCKER_USERNAME_CAMUNDA_CLOUD || env.NEXUS_USERNAME }}
          TEST_DOCKER_PASSWORD_CAMUNDA_CLOUD: ${{ env.TEST_DOCKER_PASSWORD_CAMUNDA_CLOUD || env.NEXUS_PASSWORD }}
          GITHUB_ACTOR: ${{ github.actor }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      ##########################################################################################################
      # Install
      ##########################################################################################################

      - name: Cluster Setup - Configure TLS Certificates/Secrets
        uses: ./.github/actions/cluster-setup-secrets
        with:
          distro-platform: ${{ inputs.distro-platform }}
          chart-path: ${{ env.CHART_PATH }}
          namespace: ${{ env.TEST_NAMESPACE }}

      - name: Helm - Install - Configure Helm with Entra Values
        if: (inputs.auth == 'oidc' || inputs.scenario == 'oidc' || inputs.test-identity == 'oidc') && inputs.values-config == '{}'
        timeout-minutes: 5
        env:
          TEST_CHART_FLOW: ${{ inputs.flow }}
          TEST_INGRESS_HOST: ${{ steps.setup.outputs.ingress-host }}
          TEST_VALUES_SCENARIO: "${{ inputs.scenario }}"
          TEST_AUTH_TYPE: ${{ inputs.auth }}
          INFRA_TYPE: ${{ inputs.infra-type }}
          EXTRA_VALUES: ${{ inputs.extra-values }}
          # Selection + composition model environment variables (new - preferred)
          TEST_IDENTITY: ${{ inputs.test-identity }}
          TEST_PERSISTENCE: ${{ inputs.test-persistence }}
          TEST_PLATFORM: ${{ inputs.test-platform || inputs.distro-platform }}
          TEST_FEATURES: ${{ inputs.test-features }}
          TEST_HELM_EXTRA_ARGS: >-
            ${{ env.TEST_HELM_EXTRA_ARGS }}
            ${{ steps.setup.outputs.legacy-ingress-arg }}
            --set global.host=${{ steps.setup.outputs.ingress-host }}
        run: |
          echo "Extra values from workflow:"
          echo "$EXTRA_VALUES" | tee /tmp/extra-values-file.yaml
          cd ./scripts/deploy-camunda
          go run . entra update-redirect-uris \
            --ingress-host "$TEST_INGRESS_HOST" \
            --log-level info
          go run . entra ensure-venom-app \
            --namespace "$TEST_NAMESPACE" \
            --log-level info | tee /tmp/entra-output.txt
          # Capture VENOM_CLIENT_ID and CONNECTORS_CLIENT_ID from Go CLI stdout.
          grep '^VENOM_CLIENT_ID=' /tmp/entra-output.txt >> "$GITHUB_ENV"
          grep '^CONNECTORS_CLIENT_ID=' /tmp/entra-output.txt >> "$GITHUB_ENV"
          # Also export for the current step.
          eval "$(grep '^VENOM_CLIENT_ID=' /tmp/entra-output.txt)"
          eval "$(grep '^CONNECTORS_CLIENT_ID=' /tmp/entra-output.txt)"

      - name: Helm - Install - Pre-install scenario tweaks
        timeout-minutes: 5
        env:
          EXTRA_VALUES: ${{ inputs.extra-values }}
        run: |
          # Persist user-supplied extra values for `deploy-camunda matrix run` to consume.
          # The runner uses a fixed location (/tmp/extra-values-file.yaml) which is added to
          # the helm values chain by deploy.Execute via flags.Deployment.ExtraValues.
          echo "Extra values from workflow:"
          echo "$EXTRA_VALUES" | tee /tmp/extra-values-file.yaml

          # Configure OpenSearch values when running the opensearch scenario
          # (legacy substitution; only applies to the opensearch scenario name).
          if [[ "${{ inputs.scenario }}" == "opensearch" ]] && [[ -n "${OPENSEARCH_USERNAME:-}" ]] && [[ -n "${OPENSEARCH_PASSWORD:-}" ]]; then
            values_file="${{ env.CHART_PATH }}/test/integration/scenarios/chart-full-setup/values-integration-test-ingress-${{ inputs.scenario }}.yaml"
            if [[ -f "$values_file" ]]; then
              envsubst < "$values_file" > /tmp/values-substituted.yaml && mv /tmp/values-substituted.yaml "$values_file"
            fi
          fi

      - name: Helm - OCI Registry Login
        if: inputs.helmChartVersion != ''
        run: |
          printf '%s' "${TEST_DOCKER_PASSWORD_CAMUNDA_CLOUD}" | helm registry login registry.camunda.cloud \
            --username "${TEST_DOCKER_USERNAME_CAMUNDA_CLOUD}" \
            --password-stdin
          echo "TEST_CHART_VERSION=${{ inputs.helmChartVersion }}" >> "$GITHUB_ENV"

      - name: Helm - Install - 🌟 Install Camunda chart 🌟
        # For upgrade flows, the upgrade job runs the full two-step orchestration
        # (install previous version + upgrade to current) via deploy-camunda matrix run.
        # The install job only deploys for plain `install` flow; for upgrade-patch /
        # upgrade-minor it stops after pre-install setup so the upgrade job can take
        # over a clean (but pre-secret-configured) namespace.
        if: inputs.flow == 'install'
        env:
          TEST_CHART_FLOW: ${{ inputs.flow }}
          # CAMUNDA_HOSTNAME is required by layered values (e.g. base.yaml contains $CAMUNDA_HOSTNAME)
          CAMUNDA_HOSTNAME: ${{ steps.setup.outputs.ingress-host }}
          # License key for Camunda Platform. Persisted to a file and injected via
          # --set-file (avoids shell word-splitting on spaces/semicolons).
          E2E_TESTS_LICENSE_KEY: ${{ secrets.E2E_TESTS_LICENSE_KEY || env.E2E_TESTS_LICENSE_KEY }}
          CAMUNDA_HELM_DIR: ${{ inputs.camunda-helm-dir }}
          INFRA_TYPE: ${{ inputs.infra-type }}
          # Selection + composition model environment variables (consumed by
          # deploy-camunda's prepare-values via the matrix run scenario resolver
          # — no flag wiring required, the runner reads $TEST_* env vars).
          TEST_IDENTITY: ${{ inputs.test-identity }}
          TEST_PERSISTENCE: ${{ inputs.test-persistence }}
          TEST_PLATFORM: ${{ inputs.test-platform || inputs.distro-platform }}
          TEST_FEATURES: ${{ inputs.test-features }}
          TEST_QA: ${{ inputs.test-qa }}
          TEST_UPGRADE: ${{ inputs.test-upgrade }}
          TEST_IMAGE_TAGS: ${{ inputs.include-image-tags }}
          TEST_VALUES_SCENARIO: "${{ inputs.scenario }}"
          VALUES_CONFIG: ${{ inputs.values-config }}
          HELM_CHART_VERSION: ${{ inputs.helmChartVersion }}
        run: |
          set -euo pipefail
          CHART_VERSION="${CAMUNDA_HELM_DIR#camunda-platform-}"

          # License key: pre-8.9 charts use global.license.key, >=8.9 use
          # global.license.secret.inlineSecret. Detect by grepping the chart's
          # values.yaml @param annotations.
          EXTRA_HELM_ARGS=()
          if [[ -n "${E2E_TESTS_LICENSE_KEY:-}" ]]; then
            LICENSE_FILE=/tmp/license-key.txt
            printf '%s' "${E2E_TESTS_LICENSE_KEY}" > "$LICENSE_FILE"
            if grep -q '@param global.license.key' "${{ env.CHART_PATH }}/values.yaml" 2>/dev/null; then
              EXTRA_HELM_ARGS+=(--extra-helm-arg=--set-file=global.license.key="$LICENSE_FILE")
            else
              EXTRA_HELM_ARGS+=(--extra-helm-arg=--set-file=global.license.secret.inlineSecret="$LICENSE_FILE")
            fi
          fi

          # Pass the legacy ingress arg + global.host as helm extra args.
          LEGACY_INGRESS_ARG="${{ steps.setup.outputs.legacy-ingress-arg }}"
          if [[ -n "$LEGACY_INGRESS_ARG" ]]; then
            # legacy-ingress-arg is space-delimited (e.g. "--set foo=bar"); split it.
            # shellcheck disable=SC2206
            LEGACY_TOKENS=($LEGACY_INGRESS_ARG)
            for tok in "${LEGACY_TOKENS[@]}"; do
              EXTRA_HELM_ARGS+=(--extra-helm-arg="$tok")
            done
          fi

          # OCI chart override: when helmChartVersion is set, deploy from the published
          # OCI artifact instead of the local git chart directory.
          CHART_REF_ARGS=()
          if [[ -n "${HELM_CHART_VERSION:-}" ]]; then
            CHART_REF_ARGS+=(--chart-ref "oci://registry.camunda.cloud/team-distribution/camunda-platform")
            CHART_REF_ARGS+=(--chart-version "${HELM_CHART_VERSION}")
          fi

          # Image version source selection:
          #   TEST_IMAGE_TAGS=false → --disable-image-tags; when OCI chart is set the runner
          #     applies no version overlay (chart's values.yaml is authoritative); for local
          #     chart deploys the runner falls back to values-digest.yaml.
          #   TEST_IMAGE_TAGS=true  → load VALUES_CONFIG as .env file so base-image-tags.yaml
          #     substitution resolves $E2E_TESTS_*_IMAGE_TAG placeholders (SNAPSHOT builds).
          IMAGE_TAGS_ARGS=()
          ENV_FILE_ARGS=()
          if [[ "${TEST_IMAGE_TAGS:-true}" == "false" ]]; then
            IMAGE_TAGS_ARGS=(--disable-image-tags)
          elif [[ -n "${VALUES_CONFIG:-}" && "${VALUES_CONFIG}" != "{}" ]]; then
            echo "${VALUES_CONFIG}" | jq -r 'to_entries[] | "\(.key)=\(.value)"' > /tmp/values-config.env
            ENV_FILE_ARGS=(--env-file /tmp/values-config.env)
          fi

          cd ./scripts/deploy-camunda
          go run . matrix run \
            --repo-root "${GITHUB_WORKSPACE}" \
            --include-disabled \
            --scenario-filter "${{ inputs.scenario }}" \
            --shortname-filter "${{ inputs.shortname }}" \
            --shortname-exact \
            --flow-filter "${{ inputs.flow }}" \
            --versions "${CHART_VERSION}" \
            --platform "${{ inputs.distro-platform }}" \
            --namespace-override "${TEST_NAMESPACE}" \
            --skip-dependency-update=false \
            --timeout 20 \
            --extra-helm-set "global.host=${{ steps.setup.outputs.ingress-host }}" \
            ${IMAGE_TAGS_ARGS[@]+"${IMAGE_TAGS_ARGS[@]}"} \
            ${ENV_FILE_ARGS[@]+"${ENV_FILE_ARGS[@]}"} \
            ${CHART_REF_ARGS[@]+"${CHART_REF_ARGS[@]}"} \
            ${EXTRA_HELM_ARGS[@]+"${EXTRA_HELM_ARGS[@]}"} \
            --log-level info

      - name: Display Docker Image Git Commits
        id: display-image-commits
        if: always()
        continue-on-error: true
        run: |
          OUTPUT=$($GITHUB_WORKSPACE/scripts/list-chart-image-commits.sh)
          echo "$OUTPUT"
          echo "$OUTPUT" >> $GITHUB_STEP_SUMMARY

      ##########################################################################################################
      # Misc
      ##########################################################################################################

      - name: CI Setup - Update GitHub deployment status
        if: inputs.flow == 'install'
        continue-on-error: true # Deployment status is cosmetic; don't fail the job if the API 404s (e.g. merge queue refs).
        uses: bobheadxi/deployments@648679e8e4915b27893bd7dbc35cb504dc915bc8 # v1
        with:
          step: finish
          token: ${{ steps.setup.outputs.cluster-auth-token }}
          status: ${{ job.status }}
          deployment_id: ${{ steps.deployment.outputs.deployment_id }}
          env_url: https://${{ steps.setup.outputs.ingress-host }}
          env: ${{ steps.setup.outputs.identifier }}
          ref: ${{ inputs.caller-git-ref }}

      - name: Calculate Playwright sharding matrix
        id: calc-playwright-matrix
        run: |
          # Decide how many shards to use for the Playwright E2E test-suite.
          if [[ "${{ inputs.run-all-e2e-tests }}" != "true" ]]; then
            SHARD_TOTAL="[4]"
            SHARD_INDEX_LIST="[1,2,3,4]"
          else
            SHARD_TOTAL="[36]"
            SHARD_INDEX_LIST="[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36]"
          fi

          echo "SHARD_TOTAL=${SHARD_TOTAL}" | tee -a "$GITHUB_ENV"
          echo "SHARD_INDEX=${SHARD_INDEX_LIST}" | tee -a "$GITHUB_ENV"

      - name: 🚨 Get failed Pods info 🚨
        if: failure()
        uses: ./.github/actions/failed-pods-info

  upgrade:
    if: always() && github.event.action != 'closed' && (needs.install.result == 'success' || needs.install.result == 'skipped') && inputs.flow != 'install'
    name: upgrade for ${{ inputs.flow }} on ${{ inputs.distro-platform }} - ${{ inputs.shortname }}
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/camunda/team-distribution/ci-runner:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --user root
    defaults:
      run:
        shell: bash
    needs: [install]
    outputs:
      vars-identifier: ${{ steps.setup.outputs.identifier }}
      vars-ingress-host: ${{ steps.setup.outputs.ingress-host }}
      vars-namespace: ${{ steps.setup.outputs.namespace }}
      SHARD_TOTAL: ${{ env.SHARD_TOTAL }}
      SHARD_INDEX: ${{ env.SHARD_INDEX }}
    permissions:
      contents: read
      id-token: write
      deployments: write
      packages: read
    env:
      TEST_CLUSTER_TYPE: ${{ inputs.distro-type || inputs.cluster-type }}

    steps:
      - name: Info - ℹ️ Print workflow inputs ℹ️
        env:
          GITHUB_CONTEXT: ${{ toJson(inputs) }}
        run: |
          if [[ -n "${INITIAL_CLAIM_VALUE}" ]]; then
            echo "::add-mask::${INITIAL_CLAIM_VALUE}"
          fi
          echo "Workflow Inputs:"
          echo "${GITHUB_CONTEXT}" | jq '."extra-values" = "<Check below>"'
          echo "Workflow Inputs - Extra Values:"
          echo "${GITHUB_CONTEXT}" | jq -r '."extra-values"'

      - name: CI Setup - Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          # This is needed to load repo GH composite actions if the workflow triggered by workflow_call.
          repository: camunda/camunda-platform-helm
          ref: ${{ inputs.camunda-helm-git-ref }}

      - name: CI Setup - Configure Git safe directory
        run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"

      # Tools (golang, helm, kubectl, oc, task, yq, zbctl) are pre-installed in the CI runner container image
      # See: .github/docker/ci-runner/Dockerfile
      # The per-scenario helm-version input below can override the pre-baked helm when set in ci-test-config.yaml.
      # When no override is provided, fall back to the helm version pinned in .tool-versions.

      - name: CI Setup - Resolve Helm version
        id: helm-resolve
        run: |
          version="${{ inputs.helm-version }}"
          if [[ -z "$version" ]]; then
            version="$(awk '$1=="helm"{print $2}' .tool-versions)"
          fi
          prebaked="$(helm version --short --template '{{.Version}}' 2>/dev/null | sed 's/^v//' | cut -d'+' -f1)"
          echo "version=$version" >> "$GITHUB_OUTPUT"
          if [[ "$version" == "$prebaked" ]]; then
            echo "needs-install=false" >> "$GITHUB_OUTPUT"
            echo "Pre-baked helm $prebaked matches requested $version — skipping override"
          else
            echo "needs-install=true" >> "$GITHUB_OUTPUT"
            echo "Pre-baked helm $prebaked != requested $version — installing override"
          fi

      - name: CI Setup - Override Helm version
        if: steps.helm-resolve.outputs.needs-install == 'true'
        env:
          HELM_VERSION: ${{ steps.helm-resolve.outputs.version }}
        run: |
          tarball="helm-v${HELM_VERSION}-linux-amd64.tar.gz"
          curl -fsSL "https://get.helm.sh/${tarball}" -o "/tmp/${tarball}"
          curl -fsSL "https://get.helm.sh/${tarball}.sha256sum" -o "/tmp/${tarball}.sha256sum"
          (cd /tmp && sha256sum --check "${tarball}.sha256sum")
          tar -xzf "/tmp/${tarball}" -C /tmp
          mv /tmp/linux-amd64/helm /usr/local/bin/helm
          rm -rf "/tmp/${tarball}" "/tmp/${tarball}.sha256sum" /tmp/linux-amd64
          helm version --short

      - name: CI Setup - Shared integration test setup
        id: setup
        uses: ./.github/actions/integration-test-setup
        with:
          vault-secret-mapping:        ${{ inputs.vault-secret-mapping }}
          distro-platform:             ${{ inputs.distro-platform }}
          auth-data:                   ${{ inputs.auth-data }}
          deployment-ttl:              ${{ env.CI_DEPLOYMENT_TTL }}
          flow:                        ${{ inputs.flow }}
          identifier:                  ${{ inputs.identifier }}
          camunda-helm-dir:            ${{ inputs.camunda-helm-dir }}
          camunda-helm-upgrade-version: ${{ inputs.camunda-helm-upgrade-version }}
          namespace-prefix:            ${{ inputs.namespace-prefix }}
          infra-type:                  ${{ inputs.infra-type }}
          values-enterprise:           ${{ inputs.values-enterprise }}
          values-digest:               ${{ inputs.values-digest }}
          camunda-version-previous:    ${{ inputs.camunda-version-previous }}
          upgrade-step:                "true"
        env:
          VAULT_ADDR:       ${{ secrets.VAULT_ADDR }}
          VAULT_ROLE_ID:    ${{ secrets.VAULT_ROLE_ID }}
          VAULT_SECRET_ID:  ${{ secrets.VAULT_SECRET_ID }}
          GH_APP_ID:        ${{ secrets.GH_APP_ID_DISTRO_CI_MANAGE_GH_ENVS }}
          GH_APP_KEY:       ${{ secrets.GH_APP_PRIVATE_KEY_DISTRO_CI_MANAGE_GH_ENVS }}
          ROSA_URL:         ${{ secrets[inputs.server-url] }}
          ROSA_USER:        ${{ secrets[inputs.username] }}
          ROSA_PASS:        ${{ secrets[inputs.password] }}
          DISTRO_CI_DOCKER_USERNAME_DOCKERHUB: ${{ secrets.DISTRO_CI_DOCKER_USERNAME_DOCKERHUB }}
          DISTRO_CI_DOCKER_PASSWORD_DOCKERHUB: ${{ secrets.DISTRO_CI_DOCKER_PASSWORD_DOCKERHUB }}

      ##########################################################################################################
      # Upgrade
      ##########################################################################################################
      - name: Helm - Upgrade - Configure Helm with Entra Values
        if: (inputs.auth == 'oidc' || inputs.test-identity == 'oidc') && inputs.values-config == '{}'
        timeout-minutes: 5
        env:
          TEST_CHART_FLOW: ${{ inputs.flow }}
          TEST_INGRESS_HOST: ${{ steps.setup.outputs.ingress-host }}
          TEST_VALUES_SCENARIO: "${{ inputs.scenario }}"
          TEST_AUTH_TYPE: ${{ inputs.auth }}
          INFRA_TYPE: ${{ inputs.infra-type }}
          EXTRA_VALUES: ${{ inputs.extra-values }}
          # Selection + composition model environment variables (new - preferred)
          TEST_IDENTITY: ${{ inputs.test-identity }}
          TEST_PERSISTENCE: ${{ inputs.test-persistence }}
          TEST_PLATFORM: ${{ inputs.test-platform || inputs.distro-platform }}
          TEST_FEATURES: ${{ inputs.test-features }}
          TEST_HELM_EXTRA_ARGS: >-
            ${{ env.TEST_HELM_EXTRA_ARGS }}
            ${{ steps.setup.outputs.legacy-ingress-arg }}
            --set global.host=${{ steps.setup.outputs.ingress-host }}
        run: |
          echo "Extra values from workflow:"
          echo "$EXTRA_VALUES" | tee /tmp/extra-values-file.yaml
          cd ./scripts/deploy-camunda
          go run . entra update-redirect-uris \
            --ingress-host "$TEST_INGRESS_HOST" \
            --log-level info
          go run . entra ensure-venom-app \
            --namespace "$TEST_NAMESPACE" \
            --log-level info | tee /tmp/entra-output.txt
          # Capture VENOM_CLIENT_ID and CONNECTORS_CLIENT_ID from Go CLI stdout.
          grep '^VENOM_CLIENT_ID=' /tmp/entra-output.txt >> "$GITHUB_ENV"
          grep '^CONNECTORS_CLIENT_ID=' /tmp/entra-output.txt >> "$GITHUB_ENV"

      # NOTE: helm package + pre-upgrade scripts are handled by `deploy-camunda matrix run`.
      # The runner deploys directly from the chart directory (no `helm package` needed) and
      # auto-discovers per-version pre-upgrade scripts via versionmatrix.HasPreUpgradeScript.

      - name: Cluster Setup - Upgrade - Configure TLS Certificates/Secrets
        uses: ./.github/actions/cluster-setup-secrets
        with:
          distro-platform: ${{ inputs.distro-platform }}
          chart-path: ${{ env.CHART_PATH }}
          namespace: ${{ env.TEST_NAMESPACE }}

      - name: Apply Vault mapped Secret
        if: inputs.vault-secret-mapping != ''
        run: |
          kubectl -n "$TEST_NAMESPACE" apply -f /tmp/vault-mapped-secrets.yaml

      - name: Helm - OCI Registry Login (Upgrade)
        if: inputs.helmChartVersion != ''
        run: |
          printf '%s' "${TEST_DOCKER_PASSWORD_CAMUNDA_CLOUD}" | helm registry login registry.camunda.cloud \
            --username "${TEST_DOCKER_USERNAME_CAMUNDA_CLOUD}" \
            --password-stdin

      - name: Helm - Upgrade - 🌟 Upgrade Camunda chart 🌟
        env:
          TEST_CHART_FLOW: ${{ inputs.flow }}
          # CAMUNDA_HOSTNAME is required by layered values (e.g. base.yaml contains $CAMUNDA_HOSTNAME)
          CAMUNDA_HOSTNAME: ${{ steps.setup.outputs.ingress-host }}
          # License key for Camunda Platform.
          E2E_TESTS_LICENSE_KEY: ${{ secrets.E2E_TESTS_LICENSE_KEY || env.E2E_TESTS_LICENSE_KEY }}
          CAMUNDA_HELM_DIR: ${{ inputs.camunda-helm-dir }}
          INFRA_TYPE: ${{ inputs.infra-type }}
          # Selection + composition model environment variables.
          TEST_IDENTITY: ${{ inputs.test-identity }}
          TEST_PERSISTENCE: ${{ inputs.test-persistence }}
          TEST_PLATFORM: ${{ inputs.test-platform || inputs.distro-platform }}
          TEST_FEATURES: ${{ inputs.test-features }}
          TEST_QA: ${{ inputs.test-qa }}
          TEST_UPGRADE: ${{ inputs.test-upgrade }}
          TEST_IMAGE_TAGS: ${{ inputs.include-image-tags }}
          TEST_VALUES_SCENARIO: "${{ inputs.scenario }}"
          VALUES_CONFIG: ${{ inputs.values-config }}
          HELM_CHART_VERSION: ${{ inputs.helmChartVersion }}
        run: |
          set -euo pipefail
          CHART_VERSION="${CAMUNDA_HELM_DIR#camunda-platform-}"

          EXTRA_HELM_ARGS=()
          if [[ -n "${E2E_TESTS_LICENSE_KEY:-}" ]]; then
            LICENSE_FILE=/tmp/license-key.txt
            printf '%s' "${E2E_TESTS_LICENSE_KEY}" > "$LICENSE_FILE"
            if grep -q '@param global.license.key' "${{ env.CHART_PATH }}/values.yaml" 2>/dev/null; then
              EXTRA_HELM_ARGS+=(--extra-helm-arg=--set-file=global.license.key="$LICENSE_FILE")
            else
              EXTRA_HELM_ARGS+=(--extra-helm-arg=--set-file=global.license.secret.inlineSecret="$LICENSE_FILE")
            fi
          fi

          LEGACY_INGRESS_ARG="${{ steps.setup.outputs.legacy-ingress-arg }}"
          if [[ -n "$LEGACY_INGRESS_ARG" ]]; then
            # shellcheck disable=SC2206
            LEGACY_TOKENS=($LEGACY_INGRESS_ARG)
            for tok in "${LEGACY_TOKENS[@]}"; do
              EXTRA_HELM_ARGS+=(--extra-helm-arg="$tok")
            done
          fi

          # OCI chart override: when helmChartVersion is set, upgrade to the published
          # OCI artifact instead of the local git chart directory. Step 1 (installing
          # the previous version) still uses the Helm repo — only Step 2 uses the OCI ref.
          CHART_REF_ARGS=()
          if [[ -n "${HELM_CHART_VERSION:-}" ]]; then
            CHART_REF_ARGS+=(--chart-ref "oci://registry.camunda.cloud/team-distribution/camunda-platform")
            CHART_REF_ARGS+=(--chart-version "${HELM_CHART_VERSION}")
          fi

          # Image version source selection (same logic as install block above).
          IMAGE_TAGS_ARGS=()
          ENV_FILE_ARGS=()
          if [[ "${TEST_IMAGE_TAGS:-true}" == "false" ]]; then
            IMAGE_TAGS_ARGS=(--disable-image-tags)
          elif [[ -n "${VALUES_CONFIG:-}" && "${VALUES_CONFIG}" != "{}" ]]; then
            echo "${VALUES_CONFIG}" | jq -r 'to_entries[] | "\(.key)=\(.value)"' > /tmp/values-config.env
            ENV_FILE_ARGS=(--env-file /tmp/values-config.env)
          fi

          cd ./scripts/deploy-camunda
          go run . matrix run \
            --repo-root "${GITHUB_WORKSPACE}" \
            --include-disabled \
            --scenario-filter "${{ inputs.scenario }}" \
            --shortname-filter "${{ inputs.shortname }}" \
            --shortname-exact \
            --flow-filter "${{ inputs.flow }}" \
            --versions "${CHART_VERSION}" \
            --platform "${{ inputs.distro-platform }}" \
            --namespace-override "${TEST_NAMESPACE}" \
            --skip-dependency-update=false \
            --timeout 20 \
            --extra-helm-set "global.host=${{ steps.setup.outputs.ingress-host }}" \
            ${IMAGE_TAGS_ARGS[@]+"${IMAGE_TAGS_ARGS[@]}"} \
            ${ENV_FILE_ARGS[@]+"${ENV_FILE_ARGS[@]}"} \
            ${CHART_REF_ARGS[@]+"${CHART_REF_ARGS[@]}"} \
            ${EXTRA_HELM_ARGS[@]+"${EXTRA_HELM_ARGS[@]}"} \
            --log-level info

      - name: Display Docker Image Git Commits (Upgrade)
        id: display-image-commits-upgrade
        if: always()
        continue-on-error: true
        run: |
          OUTPUT=$($GITHUB_WORKSPACE/scripts/list-chart-image-commits.sh)
          echo "$OUTPUT"
          echo "$OUTPUT" >> $GITHUB_STEP_SUMMARY

      ##########################################################################################################
      # Misc
      ##########################################################################################################

      - name: 🚨 Get failed Pods info 🚨
        if: failure()
        uses: ./.github/actions/failed-pods-info

  playwright-e2e-tests-after-install:
    name: Playwright e2e after install - ${{ inputs.flow }} on ${{ inputs.distro-platform }} - ${{ inputs.shortname }} (${{ matrix.shardIndex }} of ${{ matrix.shardTotal }})
    needs: [install]
    runs-on: gcp-core-8-release
    container:
      image: ghcr.io/camunda/team-distribution/playwright-runner:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --ipc=host --user root
    defaults:
      run:
        shell: bash
    if: ${{ inputs.test-enabled && inputs.e2e-enabled && inputs.flow == 'install' && !inputs.skip-e2e }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        shardIndex: [1]
        shardTotal: [1]
    permissions:
      contents: read
      id-token: write
      deployments: write
      packages: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: camunda/camunda-platform-helm
          ref: ${{ inputs.camunda-helm-git-ref }}
      - name: Import Vault secrets
        if: inputs.distro-platform == 'gke'
        id: test-credentials-secret
        uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0
        with:
          url: ${{ secrets.VAULT_ADDR }}
          method: approle
          roleId: ${{ secrets.VAULT_ROLE_ID }}
          secretId: ${{ secrets.VAULT_SECRET_ID }}
          secrets: |
            secret/data/products/distribution/ci DISTRO_CI_GCP_GKE_CLUSTER_NAME;
            secret/data/products/distribution/ci DISTRO_CI_GCP_GKE_CLUSTER_LOCATION;
            secret/data/products/distribution/ci DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER;
            secret/data/products/distribution/ci DISTRO_CI_GCP_SERVICE_ACCOUNT;
          exportEnv: true

      - name: Run Playwright E2E Tests
        uses: ./.github/actions/playwright-e2e-tests
        env:
          GH_APP_ID:        ${{ secrets.GH_APP_ID_DISTRO_CI_MANAGE_GH_ENVS }}
          GH_APP_KEY:       ${{ secrets.GH_APP_PRIVATE_KEY_DISTRO_CI_MANAGE_GH_ENVS }}
          GKE_CLUSTER_NAME: ${{ env.DISTRO_CI_GCP_GKE_CLUSTER_NAME }}
          GKE_CLUSTER_LOC:  ${{ env.DISTRO_CI_GCP_GKE_CLUSTER_LOCATION }}
          GKE_WIP:          ${{ env.DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER }}
          GKE_SA:           ${{ env.DISTRO_CI_GCP_SERVICE_ACCOUNT }}
          ROSA_URL:         ${{ secrets[inputs.server-url] }}
          ROSA_USER:        ${{ secrets[inputs.username] }}
          ROSA_PASS:        ${{ secrets[inputs.password] }}
          CI_DEPLOYMENT_TTL: ${{ env.CI_DEPLOYMENT_TTL }}
        with:
          camunda-helm-git-ref: ${{ inputs.camunda-helm-git-ref }}
          camunda-helm-dir: ${{ inputs.camunda-helm-dir }}
          camunda-helm-upgrade-version: ${{ inputs.camunda-helm-upgrade-version }}
          flow: ${{ inputs.flow }}
          distro-platform: ${{ inputs.distro-platform }}
          infra-type: ${{ inputs.infra-type }}
          identifier: ${{ inputs.identifier }}
          namespace-prefix: ${{ inputs.namespace-prefix }}
          deployment-ttl: ${{ env.CI_DEPLOYMENT_TTL }}
          auth: ${{ inputs.auth }}
          auth-data: ${{ inputs.auth-data }}
          exclude: ${{ inputs.exclude }}
          test-stage: after-install
          shard-index: ${{ matrix.shardIndex }}
          scenario: ${{ inputs.scenario }}

  # Shadow job: runs the full E2E suite for the dedicated 'shadow-e2e' scenario.
  # Tier 2 (merge queue only). Non-blocking during observation period.
  shadow-e2e-full-suite:
    name: "[shadow] Full e2e - ${{ inputs.shortname }}"
    needs: [install]
    runs-on: gcp-core-32-longrunning-big-ssd
    container:
      image: ghcr.io/camunda/team-distribution/playwright-runner:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --ipc=host --user root
    defaults:
      run:
        shell: bash
    if: ${{ inputs.test-enabled && inputs.scenario == 'shadow-e2e' }}
    continue-on-error: true
    timeout-minutes: 30
    permissions:
      contents: read
      id-token: write
      deployments: write
      packages: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: camunda/camunda-platform-helm
          ref: ${{ inputs.camunda-helm-git-ref }}

      - name: Import Vault secrets
        id: test-credentials-secret
        uses: hashicorp/vault-action@4c06c5ccf5c0761b6029f56cfb1dcf5565918a3b # v3.4.0
        with:
          url: ${{ secrets.VAULT_ADDR }}
          method: approle
          roleId: ${{ secrets.VAULT_ROLE_ID }}
          secretId: ${{ secrets.VAULT_SECRET_ID }}
          secrets: |
            secret/data/products/distribution/ci DISTRO_CI_GCP_GKE_CLUSTER_NAME;
            secret/data/products/distribution/ci DISTRO_CI_GCP_GKE_CLUSTER_LOCATION;
            secret/data/products/distribution/ci DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER;
            secret/data/products/distribution/ci DISTRO_CI_GCP_SERVICE_ACCOUNT;
          exportEnv: true

      - name: Run full E2E suite (shadow)
        uses: ./.github/actions/playwright-e2e-tests
        env:
          GH_APP_ID:        ${{ secrets.GH_APP_ID_DISTRO_CI_MANAGE_GH_ENVS }}
          GH_APP_KEY:       ${{ secrets.GH_APP_PRIVATE_KEY_DISTRO_CI_MANAGE_GH_ENVS }}
          GKE_CLUSTER_NAME: ${{ env.DISTRO_CI_GCP_GKE_CLUSTER_NAME }}
          GKE_CLUSTER_LOC:  ${{ env.DISTRO_CI_GCP_GKE_CLUSTER_LOCATION }}
          GKE_WIP:          ${{ env.DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER }}
          GKE_SA:           ${{ env.DISTRO_CI_GCP_SERVICE_ACCOUNT }}
          ROSA_URL:         ${{ secrets[inputs.server-url] }}
          ROSA_USER:        ${{ secrets[inputs.username] }}
          ROSA_PASS:        ${{ secrets[inputs.password] }}
          CI_DEPLOYMENT_TTL: ${{ env.CI_DEPLOYMENT_TTL }}
          PLAYWRIGHT_POD_RETRY_MAX_ATTEMPTS: "1"
        with:
          camunda-helm-git-ref: ${{ inputs.camunda-helm-git-ref }}
          camunda-helm-dir: ${{ inputs.camunda-helm-dir }}
          camunda-helm-upgrade-version: ${{ inputs.camunda-helm-upgrade-version }}
          flow: ${{ inputs.flow }}
          distro-platform: ${{ inputs.distro-platform }}
          infra-type: ${{ inputs.infra-type }}
          identifier: ${{ inputs.identifier }}
          namespace-prefix: ${{ inputs.namespace-prefix }}
          deployment-ttl: ${{ env.CI_DEPLOYMENT_TTL }}
          auth: ${{ inputs.auth }}
          auth-data: ${{ inputs.auth-data }}
          exclude: ${{ inputs.exclude }}
          test-stage: after-install
          shard-index: "1"
          scenario: ${{ inputs.scenario }}
          run-smoke-tests: "false"

  playwright-e2e-tests-after-upgrade:
    name: Playwright e2e after upgrade - ${{ inputs.flow }} on ${{ inputs.distro-platform }} - ${{ inputs.shortname }} (${{ matrix.shardIndex }} of ${{ matrix.shardTotal }})
    needs: [upgrade]
    runs-on: gcp-core-8-release
    container:
      image: ghcr.io/camunda/team-distribution/playwright-runner:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --ipc=host --user root
    defaults:
      run:
        shell: bash
    if: ${{ inputs.test-enabled && inputs.e2e-enabled && inputs.flow != 'install' && !inputs.skip-e2e }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        shardIndex: [1]
        shardTotal: [1]
    permissions:
      contents: read
      id-token: write
      deployments: write
      packages: read
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: camunda/camunda-platform-helm
          ref: ${{ inputs.camunda-helm-git-ref }}

      - name: Import Vault secrets
        if: inputs.distro-platform == 'gke'
        id: test-credentials-secret
        uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0
        with:
          url: ${{ secrets.VAULT_ADDR }}
          method: approle
          roleId: ${{ secrets.VAULT_ROLE_ID }}
          secretId: ${{ secrets.VAULT_SECRET_ID }}
          secrets: |
            secret/data/products/distribution/ci DISTRO_CI_GCP_GKE_CLUSTER_NAME;
            secret/data/products/distribution/ci DISTRO_CI_GCP_GKE_CLUSTER_LOCATION;
            secret/data/products/distribution/ci DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER;
            secret/data/products/distribution/ci DISTRO_CI_GCP_SERVICE_ACCOUNT;
          exportEnv: true

      - name: Run Playwright E2E Tests
        uses: ./.github/actions/playwright-e2e-tests
        env:
          GH_APP_ID:        ${{ secrets.GH_APP_ID_DISTRO_CI_MANAGE_GH_ENVS }}
          GH_APP_KEY:       ${{ secrets.GH_APP_PRIVATE_KEY_DISTRO_CI_MANAGE_GH_ENVS }}
          GKE_CLUSTER_NAME: ${{ env.DISTRO_CI_GCP_GKE_CLUSTER_NAME }}
          GKE_CLUSTER_LOC:  ${{ env.DISTRO_CI_GCP_GKE_CLUSTER_LOCATION }}
          GKE_WIP:          ${{ env.DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER }}
          GKE_SA:           ${{ env.DISTRO_CI_GCP_SERVICE_ACCOUNT }}
          ROSA_URL:         ${{ secrets[inputs.server-url] }}
          ROSA_USER:        ${{ secrets[inputs.username] }}
          ROSA_PASS:        ${{ secrets[inputs.password] }}
          CI_DEPLOYMENT_TTL: ${{ env.CI_DEPLOYMENT_TTL }}
        with:
          camunda-helm-git-ref: ${{ inputs.camunda-helm-git-ref }}
          camunda-helm-dir: ${{ inputs.camunda-helm-dir }}
          camunda-helm-upgrade-version: ${{ inputs.camunda-helm-upgrade-version }}
          flow: ${{ inputs.flow }}
          distro-platform: ${{ inputs.distro-platform }}
          infra-type: ${{ inputs.infra-type }}
          identifier: ${{ inputs.identifier }}
          namespace-prefix: ${{ inputs.namespace-prefix }}
          deployment-ttl: ${{ env.CI_DEPLOYMENT_TTL }}
          auth: ${{ inputs.auth }}
          auth-data: ${{ inputs.auth-data }}
          exclude: ${{ inputs.exclude }}
          test-stage: after-upgrade
          shard-index: ${{ matrix.shardIndex }}
          scenario: ${{ inputs.scenario }}

  merge-e2e-reports:
    name: Merge E2E Reports - ${{ inputs.flow }} on ${{ inputs.distro-platform }} - ${{ inputs.shortname }}
    # Run if e2e is enabled AND at least one test job completed (success or failure, not skipped/cancelled)
    if: ${{ !cancelled() && inputs.e2e-enabled && (needs.playwright-e2e-tests-after-install.result == 'success' || needs.playwright-e2e-tests-after-install.result == 'failure' || needs.playwright-e2e-tests-after-upgrade.result == 'success' || needs.playwright-e2e-tests-after-upgrade.result == 'failure') }}
    needs: [playwright-e2e-tests-after-install, playwright-e2e-tests-after-upgrade]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: camunda/camunda-platform-helm
          ref: ${{ inputs.camunda-helm-git-ref }}

      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
        with:
          node-version: lts/*

      - name: Download blob reports from GitHub Actions Artifacts
        id: download-reports
        continue-on-error: true
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          path: all-blob-reports
          pattern: blob-report-${{ inputs.identifier }}-${{ inputs.flow }}-${{ github.run_id }}-${{ github.run_attempt }}-*
          merge-multiple: true

      - name: Validate and cleanup blob reports
        run: |
          echo "Validating blob report ZIP files..."
          
          # Check if download succeeded and directory exists with files
          if [[ "${{ steps.download-reports.outcome }}" != "success" ]] || [[ ! -d "all-blob-reports" ]]; then
            echo "⚠️ No blob reports were downloaded (download outcome: ${{ steps.download-reports.outcome }})"
            mkdir -p all-blob-reports
            exit 0
          fi
          
          cd all-blob-reports
          
          for zip_file in *.zip; do
            if [[ -f "$zip_file" ]]; then
              if ! unzip -t "$zip_file" > /dev/null 2>&1; then
                echo "⚠️ Removing corrupted/incomplete zip file: $zip_file"
                rm -f "$zip_file"
              else
                echo "✅ Valid: $zip_file"
              fi
            fi
          done
          
          remaining_files=$(ls -1 *.zip 2>/dev/null | wc -l || echo "0")
          echo "Valid blob reports remaining: $remaining_files"
          
          if [[ "$remaining_files" -eq 0 ]]; then
            echo "⚠️ No valid blob reports found to merge"
            exit 0
          fi

      - name: Merge into HTML Report
        run: |
          # Check if there are any valid reports to merge
          if [[ ! -d "all-blob-reports" ]] || [[ -z "$(ls -A all-blob-reports/*.zip 2>/dev/null)" ]]; then
            echo "No valid blob reports found, skipping merge"
            mkdir -p playwright-report
            echo "<html><body><h1>No E2E test reports available</h1><p>All blob reports were corrupted or no tests ran.</p></body></html>" > playwright-report/index.html
            exit 0
          fi
          
          npm install -g playwright
          npx playwright merge-reports --reporter html ./all-blob-reports

      - name: Upload HTML report
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: e2e-html-report-${{ inputs.identifier }}-${{ inputs.flow }}-${{ github.run_id }}-${{ github.run_attempt }}-${{ inputs.shortname }}
          path: playwright-report
          retention-days: 14

  # Record a successful scenario result as a commit status on the PR HEAD commit.
  # This enables progressive CI result caching: future merge queue attempts can
  # skip scenarios that already passed for the same content hash.
  record-result:
    name: Record cache result - ${{ inputs.flow }} on ${{ inputs.distro-platform }} - ${{ inputs.shortname }}
    needs: [install, playwright-e2e-tests-after-install, playwright-e2e-tests-after-upgrade, merge-e2e-reports]
    # Only record when:
    # 1. All prerequisite jobs succeeded (or were skipped — e.g., e2e tests skipped for skip-e2e scenarios)
    #    Note: we check both 'failure' AND 'cancelled' to prevent false positives when
    #    tests are cancelled (e.g., timeout). A cancelled job result is 'cancelled', not 'failure'.
    # 2. PR HEAD SHA is available (set by the init job for merge_group and pull_request events)
    # 3. Camunda version is set (needed for content hash computation)
    if: >-
      ${{
        !cancelled() &&
        !contains(needs.*.result, 'failure') &&
        !contains(needs.*.result, 'cancelled') &&
        inputs.pr-head-sha != '' &&
        inputs.camunda-version != ''
      }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      statuses: write
    steps:
      - name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          repository: camunda/camunda-platform-helm
          ref: ${{ inputs.camunda-helm-git-ref }}

      - name: Setup Go
        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5
        with:
          go-version-file: scripts/ci-result-cache/go.mod
          cache: true
          cache-dependency-path: scripts/ci-result-cache/go.sum

      - name: Record passing result
        env:
          GITHUB_TOKEN: ${{ github.token }}
          GITHUB_REPOSITORY: ${{ github.repository }}
        run: |
          cd scripts/ci-result-cache && go build -o /tmp/ci-result-cache .
          /tmp/ci-result-cache record \
            --sha "${{ inputs.pr-head-sha }}" \
            --version "${{ inputs.camunda-version }}" \
            --shortname "${{ inputs.shortname }}" \
            --flow "${{ inputs.flow }}" \
            --repo-root "${{ github.workspace }}" \
            --target-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

  cleanup:
    name: Cleanup - ${{ inputs.flow }} on ${{ inputs.distro-platform }} - ${{ inputs.shortname }}
    needs: [merge-e2e-reports, shadow-e2e-full-suite]
    if: ${{ always() && (contains(needs.*.result, 'success') || contains(needs.*.result, 'skipped') || contains(needs.*.result, 'cancelled')) }}
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/camunda/team-distribution/ci-runner:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --user root
    defaults:
      run:
        shell: bash
    permissions:
      contents: read
      id-token: write
      deployments: write
      packages: read
    steps:
      - name: CI Cleanup - 🚨 Get failed Pods info 🚨
        if: failure()
        uses: ./.github/actions/failed-pods-info

      - name: CI Setup - Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
        with:
          # This is needed to load repo GH composite actions if the workflow triggered by workflow_call.
          repository: camunda/camunda-platform-helm
          ref: ${{ inputs.camunda-helm-git-ref }}

      - name: CI Setup - Configure Git safe directory
        run: git config --global --add safe.directory "${GITHUB_WORKSPACE}"

      - name: CI Setup - ℹ️ Set workflow vars for both install and upgrade ℹ️
        id: vars
        uses: ./.github/actions/workflow-vars
        with:
          deployment-ttl: ${{ env.CI_DEPLOYMENT_TTL }}
          setup-flow: ${{ inputs.flow }}
          platform: ${{ inputs.distro-platform }}
          identifier-base: ${{ inputs.identifier }}
          chart-dir: ${{ inputs.camunda-helm-dir }}
          chart-upgrade-version: ${{ inputs.camunda-helm-upgrade-version }}
          prefix: ${{ inputs.namespace-prefix }}

      - name: CI Setup - Set test type vars
        id: test-type-vars
        uses: ./.github/actions/test-type-vars
        with:
          chart-dir: "${{ inputs.camunda-helm-dir }}"
          infra-type: "${{ inputs.infra-type }}"
          platform: "${{ inputs.distro-platform }}"
          flow: "${{ inputs.flow }}"

      - name: Import Vault secrets
        if: inputs.distro-platform == 'gke'
        id: test-credentials-secret
        uses: hashicorp/vault-action@892a26828f195e65540a40b4768ae4571f51ebfc # v4.0.0
        with:
          url: ${{ secrets.VAULT_ADDR }}
          method: approle
          roleId: ${{ secrets.VAULT_ROLE_ID }}
          secretId: ${{ secrets.VAULT_SECRET_ID }}
          secrets: |
            secret/data/products/distribution/ci DISTRO_CI_GCP_GKE_CLUSTER_NAME;
            secret/data/products/distribution/ci DISTRO_CI_GCP_GKE_CLUSTER_LOCATION;
            secret/data/products/distribution/ci DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER;
            secret/data/products/distribution/ci DISTRO_CI_GCP_SERVICE_ACCOUNT;
            secret/data/products/distribution/ci DISTRO_AWS_OPENSEARCH_USERNAME | OPENSEARCH_USERNAME;
            secret/data/products/distribution/ci DISTRO_AWS_OPENSEARCH_PASSWORD | OPENSEARCH_PASSWORD;
            secret/data/products/distribution/ci AUTH0_DOMAIN;
            secret/data/products/distribution/ci AUTH0_MGMT_CLIENT_ID;
            secret/data/products/distribution/ci AUTH0_MGMT_CLIENT_SECRET;
            secret/data/products/distribution/ci ENTRA_APP_CLIENT_ID;
            secret/data/products/distribution/ci ENTRA_APP_CLIENT_SECRET;
            secret/data/products/distribution/ci ENTRA_APP_OBJECT_ID;
            secret/data/products/distribution/ci ENTRA_APP_DIRECTORY_ID;
          exportEnv: true

      - name: CI Setup - Authenticate to cluster
        id: cluster-auth
        uses: ./.github/actions/cluster-auth
        with:
          platform: ${{ inputs.distro-platform }}
          auth-data: ${{ inputs.auth-data }}
        env:
          GH_APP_ID:        ${{ secrets.GH_APP_ID_DISTRO_CI_MANAGE_GH_ENVS }}
          GH_APP_KEY:       ${{ secrets.GH_APP_PRIVATE_KEY_DISTRO_CI_MANAGE_GH_ENVS }}
          GKE_CLUSTER_NAME: ${{ env.DISTRO_CI_GCP_GKE_CLUSTER_NAME }}
          GKE_CLUSTER_LOC:  ${{ env.DISTRO_CI_GCP_GKE_CLUSTER_LOCATION }}
          GKE_WIP:          ${{ env.DISTRO_CI_GCP_WORKLOAD_IDENTITY_PROVIDER }}
          GKE_SA:           ${{ env.DISTRO_CI_GCP_SERVICE_ACCOUNT }}
          ROSA_URL:         ${{ secrets[inputs.server-url] }}
          ROSA_USER:        ${{ secrets[inputs.username] }}
          ROSA_PASS:        ${{ secrets[inputs.password] }}
          CLUSTER_NAME:     ${{ env.INFRA_CLUSTER_NAME }}

      - name: CI Cleanup - Cleanup GitHub deployment
        if: always() && (env.CI_DEPLOYMENT_TTL == '' || inputs.distro-type != 'kubernetes')
        continue-on-error: true # We don't want to fail if the deployment is not found.
        uses: bobheadxi/deployments@648679e8e4915b27893bd7dbc35cb504dc915bc8 # v1
        with:
          step: delete-env
          token: ${{ steps.cluster-auth.outputs.token }}
          env: ${{ steps.vars.outputs.identifier }}
          ref: ${{ inputs.caller-git-ref }}

      - name: CI Cleanup - Delete venom Entra app registration
        if: always() && (inputs.auth == 'oidc' || inputs.scenario == 'oidc' || inputs.test-identity == 'oidc')
        continue-on-error: true
        run: |
          cd ./scripts/deploy-camunda
          go run . entra cleanup-venom-app \
            --namespace "$TEST_NAMESPACE" \
            --log-level info

      - name: CI Cleanup - Delete Auth0 OIDC clients
        # Mirrors the Entra cleanup above. The auth0 scenario provisions
        # per-component first-party clients in the tenant via
        # `auth0 ensure-clients`; namespace deletion does not touch the
        # tenant, so without this step orphans accumulate across PR runs.
        # AUTH0_DOMAIN / AUTH0_MGMT_CLIENT_ID / AUTH0_MGMT_CLIENT_SECRET
        # are already exported to $GITHUB_ENV by the integration-test-setup
        # action's vault mapping.
        if: always() && inputs.scenario == 'auth0'
        continue-on-error: true
        run: |
          cd ./scripts/deploy-camunda
          go run . auth0 cleanup-clients \
            --namespace "$TEST_NAMESPACE" \
            --domain "$AUTH0_DOMAIN" \
            --log-level info

      - name: CI Cleanup
        if: always() && inputs.always-delete-namespace
        run: |
          kubectl annotate ns $TEST_NAMESPACE test-flow=${{ inputs.flow }} --overwrite || true
          kubectl annotate ns $TEST_NAMESPACE build-failed=true --overwrite || true

          if [[ "$(kubectl get ns $TEST_NAMESPACE --ignore-not-found)" != "" && "${{ env.CI_DEPLOYMENT_TTL }}" == "" ]]; then
            kubectl annotate ns $TEST_NAMESPACE cleaner/ttl=1s --overwrite=true
            kubectl annotate ns $TEST_NAMESPACE janitor/ttl=1s --overwrite=true
            scripts/apply-ttl-to-elasticsearch-indexes.sh --prefix "${GITHUB_WORKFLOW_JOB_ID}" --ttl 1s --url "https://${INFRA_ES_HOST}-pool-${ES_POOL_INDEX}" --user "elastic" --elasticsearch-namespace "${INFRA_ES_NAMESPACE}-pool-${ES_POOL_INDEX}" --debug || echo "Warning: Elasticsearch index cleanup failed (credentials inaccessible), skipping..."
            scripts/apply-ttl-to-elasticsearch-indexes.sh --prefix "${GITHUB_WORKFLOW_JOB_ID}" --ttl 1s --url "${OPENSEARCH_PROTOCOL}://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}" --user "${OPENSEARCH_USERNAME}" --pass "${OPENSEARCH_PASSWORD}" --debug || echo "Warning: OpenSearch index cleanup failed, skipping..."
          else
            if [[ "${{ inputs.distro-type }}" == "kubernetes" && "${{ env.CI_DEPLOYMENT_TTL }}" != "" ]]; then
              kubectl annotate ns $TEST_NAMESPACE cleaner/ttl=${CI_DEPLOYMENT_TTL} --overwrite=true || true
              kubectl annotate ns $TEST_NAMESPACE janitor/ttl=${CI_DEPLOYMENT_TTL} --overwrite=true || true
            else
              kubectl annotate ns $TEST_NAMESPACE cleaner/ttl=1s --overwrite=true || true
              kubectl annotate ns $TEST_NAMESPACE janitor/ttl=1s --overwrite=true || true
              scripts/apply-ttl-to-elasticsearch-indexes.sh --prefix "${GITHUB_WORKFLOW_JOB_ID}" --ttl 1s --url "https://${INFRA_ES_HOST}-pool-${ES_POOL_INDEX}" --user "elastic" --elasticsearch-namespace "${INFRA_ES_NAMESPACE}-pool-${ES_POOL_INDEX}" --debug || echo "Warning: Elasticsearch index cleanup failed (credentials inaccessible), skipping..."
              scripts/apply-ttl-to-elasticsearch-indexes.sh --prefix "${GITHUB_WORKFLOW_JOB_ID}" --ttl 1s --url "${OPENSEARCH_PROTOCOL}://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}" --user "${OPENSEARCH_USERNAME}" --pass "${OPENSEARCH_PASSWORD}" --debug || echo "Warning: OpenSearch index cleanup failed, skipping..."
            fi
          fi

      - name: CI Cleanup - Delete Elasticsearch templates for this CI run
        if: always()
        continue-on-error: true
        run: |
          set -euo pipefail

          ES_URL="https://${INFRA_ES_HOST}-pool-${ES_POOL_INDEX}"
          ES_NS="${INFRA_ES_NAMESPACE}-pool-${ES_POOL_INDEX}"

          # Extract the 6-char hex suffix from the namespace.
          # Namespace patterns:
          #   camunda-pr-XXXX-intg-X-X-gke-XXXXX-{hex}
          #   {prefix}-{hex}-upgm
          HEX_PREFIX="${TEST_NAMESPACE##*-}"
          # For upgrade namespaces ending in -upgm, the hex is the second-to-last segment.
          if [[ "$HEX_PREFIX" == "upgm" ]]; then
            HEX_PREFIX="$(echo "$TEST_NAMESPACE" | sed 's/-upgm$//' | sed 's/.*-//')"
          fi

          # Validate it looks like a 6-char hex string.
          if ! [[ "$HEX_PREFIX" =~ ^[0-9a-f]{6}$ ]]; then
            echo "Warning: Could not extract a valid 6-char hex prefix from namespace '${TEST_NAMESPACE}' (got '${HEX_PREFIX}'). Skipping ES template cleanup."
            exit 0
          fi

          echo "Cleaning up ES templates with hex prefix: ${HEX_PREFIX}"

          # Get ES password from K8s secret.
          ES_PASS="$(scripts/get-credientials-from-cluster.sh --namespace "$ES_NS")" || {
            echo "Warning: Failed to retrieve ES credentials. Skipping ES template cleanup."
            exit 0
          }

          CURL_AUTH=(-u "elastic:${ES_PASS}")
          CURL_OPTS=(--silent --show-error --connect-timeout 10 --max-time 30 --retry 2 --retry-delay 2 --retry-connrefused)

          deleted_index=0
          deleted_component=0
          deleted_legacy=0
          failed=0

          # --- Delete V2 (composable) index templates matching the hex prefix ---
          v2_templates=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" "${ES_URL}/_index_template" | jq -r --arg pfx "$HEX_PREFIX" '.index_templates[]?.name // empty | select(startswith($pfx + "-"))') || true
          if [[ -n "$v2_templates" ]]; then
            echo "Found $(echo "$v2_templates" | wc -l | tr -d ' ') V2 index templates to delete"
            # ES DELETE _index_template does not support comma-separated names, but supports wildcards.
            # Delete using the hex prefix wildcard which matches all templates for this CI run.
            http_code=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" -o /dev/null -w "%{http_code}" -X DELETE "${ES_URL}/_index_template/${HEX_PREFIX}-*") || true
            if [[ "$http_code" =~ ^2 ]]; then
              deleted_index=$(echo "$v2_templates" | wc -l | tr -d ' ')
              echo "Deleted ${deleted_index} V2 index templates"
            else
              echo "Warning: Failed to delete V2 index templates (HTTP ${http_code})"
              ((failed++)) || true
            fi
          else
            echo "No V2 index templates found for prefix ${HEX_PREFIX}"
          fi

          # --- Delete component templates matching the hex prefix ---
          comp_templates=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" "${ES_URL}/_component_template" | jq -r --arg pfx "$HEX_PREFIX" '.component_templates[]?.name // empty | select(startswith($pfx + "-"))') || true
          if [[ -n "$comp_templates" ]]; then
            echo "Found $(echo "$comp_templates" | wc -l | tr -d ' ') component templates to delete"
            http_code=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" -o /dev/null -w "%{http_code}" -X DELETE "${ES_URL}/_component_template/${HEX_PREFIX}-*") || true
            if [[ "$http_code" =~ ^2 ]]; then
              deleted_component=$(echo "$comp_templates" | wc -l | tr -d ' ')
              echo "Deleted ${deleted_component} component templates"
            else
              echo "Warning: Failed to delete component templates (HTTP ${http_code})"
              ((failed++)) || true
            fi
          else
            echo "No component templates found for prefix ${HEX_PREFIX}"
          fi

          # --- Delete legacy (v1) templates matching the hex prefix ---
          legacy_templates=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" "${ES_URL}/_template" | jq -r --arg pfx "$HEX_PREFIX" 'keys[] | select(startswith($pfx + "-"))') || true
          if [[ -n "$legacy_templates" ]]; then
            echo "Found $(echo "$legacy_templates" | wc -l | tr -d ' ') legacy templates to delete"
            http_code=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" -o /dev/null -w "%{http_code}" -X DELETE "${ES_URL}/_template/${HEX_PREFIX}-*") || true
            if [[ "$http_code" =~ ^2 ]]; then
              deleted_legacy=$(echo "$legacy_templates" | wc -l | tr -d ' ')
              echo "Deleted ${deleted_legacy} legacy templates"
            else
              echo "Warning: Failed to delete legacy templates (HTTP ${http_code})"
              ((failed++)) || true
            fi
          else
            echo "No legacy templates found for prefix ${HEX_PREFIX}"
          fi

          echo "ES template cleanup complete: v2_index=${deleted_index} component=${deleted_component} legacy=${deleted_legacy} failures=${failed}"

      - name: CI Cleanup - Delete OpenSearch templates for this CI run
        if: always()
        continue-on-error: true
        run: |
          set -euo pipefail

          if [[ -z "${OPENSEARCH_USERNAME:-}" || -z "${OPENSEARCH_PASSWORD:-}" ]]; then
            echo "OpenSearch credentials not available. Skipping OS template cleanup."
            exit 0
          fi

          OS_URL="${OPENSEARCH_PROTOCOL}://${OPENSEARCH_HOST}:${OPENSEARCH_PORT}"

          HEX_PREFIX="${TEST_NAMESPACE##*-}"
          if [[ "$HEX_PREFIX" == "upgm" || "$HEX_PREFIX" == "upgp" ]]; then
            HEX_PREFIX="$(echo "$TEST_NAMESPACE" | sed "s/-${HEX_PREFIX}\$//" | sed 's/.*-//')"
          fi

          if ! [[ "$HEX_PREFIX" =~ ^[0-9a-f]{6}$ ]]; then
            echo "Warning: Could not extract a valid 6-char hex prefix from namespace '${TEST_NAMESPACE}' (got '${HEX_PREFIX}'). Skipping OS template cleanup."
            exit 0
          fi

          echo "Cleaning up OS templates with hex prefix: ${HEX_PREFIX}"

          CURL_AUTH=(-u "${OPENSEARCH_USERNAME}:${OPENSEARCH_PASSWORD}")
          CURL_OPTS=(--silent --show-error --connect-timeout 10 --max-time 60 --retry 2 --retry-delay 2 --retry-connrefused)

          deleted_index=0
          deleted_component=0
          deleted_legacy=0
          failed=0

          v2_templates=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" "${OS_URL}/_index_template" | jq -r --arg pfx "$HEX_PREFIX" '.index_templates[]?.name // empty | select(startswith($pfx + "-"))') || true
          if [[ -n "$v2_templates" ]]; then
            echo "Found $(echo "$v2_templates" | wc -l | tr -d ' ') V2 index templates to delete"
            http_code=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" -o /dev/null -w "%{http_code}" -X DELETE "${OS_URL}/_index_template/${HEX_PREFIX}-*") || true
            if [[ "$http_code" =~ ^2 ]]; then
              deleted_index=$(echo "$v2_templates" | wc -l | tr -d ' ')
              echo "Deleted ${deleted_index} V2 index templates"
            else
              echo "Warning: Failed to delete V2 index templates (HTTP ${http_code})"
              ((failed++)) || true
            fi
          else
            echo "No V2 index templates found for prefix ${HEX_PREFIX}"
          fi

          comp_templates=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" "${OS_URL}/_component_template" | jq -r --arg pfx "$HEX_PREFIX" '.component_templates[]?.name // empty | select(startswith($pfx + "-"))') || true
          if [[ -n "$comp_templates" ]]; then
            echo "Found $(echo "$comp_templates" | wc -l | tr -d ' ') component templates to delete"
            http_code=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" -o /dev/null -w "%{http_code}" -X DELETE "${OS_URL}/_component_template/${HEX_PREFIX}-*") || true
            if [[ "$http_code" =~ ^2 ]]; then
              deleted_component=$(echo "$comp_templates" | wc -l | tr -d ' ')
              echo "Deleted ${deleted_component} component templates"
            else
              echo "Warning: Failed to delete component templates (HTTP ${http_code})"
              ((failed++)) || true
            fi
          else
            echo "No component templates found for prefix ${HEX_PREFIX}"
          fi

          legacy_templates=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" "${OS_URL}/_template" | jq -r --arg pfx "$HEX_PREFIX" 'keys[] | select(startswith($pfx + "-"))') || true
          if [[ -n "$legacy_templates" ]]; then
            echo "Found $(echo "$legacy_templates" | wc -l | tr -d ' ') legacy templates to delete"
            http_code=$(curl "${CURL_OPTS[@]}" "${CURL_AUTH[@]}" -o /dev/null -w "%{http_code}" -X DELETE "${OS_URL}/_template/${HEX_PREFIX}-*") || true
            if [[ "$http_code" =~ ^2 ]]; then
              deleted_legacy=$(echo "$legacy_templates" | wc -l | tr -d ' ')
              echo "Deleted ${deleted_legacy} legacy templates"
            else
              echo "Warning: Failed to delete legacy templates (HTTP ${http_code})"
              ((failed++)) || true
            fi
          else
            echo "No legacy templates found for prefix ${HEX_PREFIX}"
          fi

          echo "OS template cleanup complete: v2_index=${deleted_index} component=${deleted_component} legacy=${deleted_legacy} failures=${failed}"