[CRDB-62682] helm: make hook images configurable for air-gapped support

This commit replaces hardcoded kubectl images in pre-upgrade validation hooks with
configurable values via `hooks.image` for air-gapped environments.

Image manifest which lists all the images used in both charts  and a script
for mirroring images are also added.

Author: NishanthNalluri

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file.
 
+## [Unreleased]
+### Added
+- Hook images (`bitnami/kubectl`, `dtzar/helm-kubectl`) are now configurable via
+  `hooks.kubectlImage.{registry,repository,tag,pullPolicy}` in both charts. This
+  unblocks air-gapped deployments where pulling from public registries is not possible.
+- Added `cockroachdb-parent/images.txt` manifest listing all container images
+  required by both charts, including operator-managed runtime images.
+- Added `scripts/mirror-images.sh` to mirror images from the manifest into an
+  internal registry using `crane` or `skopeo`.
+
 ## [cockroachdb-parent-26.1.1-preview+2] 2026-03-26
 ### Changed
 - **API Version Migration**: The operator now uses an image that removes `v1alpha1` entirely and keeps only `v1beta1`.

Makefile

@@ -130,7 +130,7 @@ test/e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin
 	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Single region tests failed with exit code $$?" && exit 1)
 
 test/e2e/migrate: bin/cockroach bin/kubectl bin/helm bin/migration-helper build/self-signer test/cluster/up/3
-	@PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/migrate/... || EXIT_CODE=$$?; \
+	@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v ./tests/e2e/migrate/... || EXIT_CODE=$$?; \
 	$(MAKE) test/cluster/down; \
 	exit $${EXIT_CODE:-0}
 

build/templates/cockroachdb-parent/charts/cockroachdb/values.yaml

@@ -544,6 +544,24 @@ cockroachdb:
     #   # Possible Values: disabled, primary, standby
     #   mode: "disabled"
 
+# hooks captures the configuration for Helm hook jobs (pre-upgrade validation, etc.).
+# All hook jobs use the image defined here by default. Override these
+# values to use images from an internal registry in air-gapped environments.
+hooks:
+  # kubectlImage is the container image used by hook jobs that run kubectl commands.
+  # If a future hook requires a different image (e.g., cockroach sql), add a new
+  # sibling key (e.g., hooks.cockroachImage) rather than overloading this one.
+  kubectlImage:
+    # registry defines the container registry for the hook image.
+    registry: docker.io
+    # repository defines the image repository for the hook container.
+    repository: dtzar/helm-kubectl
+    # tag defines the image tag for the hook container. Pinned for reproducibility
+    # in air-gapped environments.
+    tag: "3.19"
+    # pullPolicy defines the image pull policy for the hook container.
+    pullPolicy: IfNotPresent
+
 k8s:
   # nameOverride overrides the name of the chart. If not set, the chart name will be used.
   # For example, if the chart name is "cockroachdb" and nameOverride is set to "crdb",

build/templates/cockroachdb-parent/charts/operator/values.yaml

@@ -38,3 +38,20 @@ appLabel: "cockroach-operator"
 # startup; certs rotate on every pod restart. When false (default), Helm provisions a stable
 # cockroach-operator-certs Secret. Switching requires deleting that Secret first.
 selfSignedOperatorCerts: false
+# hooks captures the configuration for Helm hook jobs (pre-upgrade validation, etc.).
+# All hook jobs use the image defined here by default. Override these
+# values to use images from an internal registry in air-gapped environments.
+hooks:
+  # kubectlImage is the container image used by hook jobs that run kubectl commands.
+  # If a future hook requires a different image (e.g., cockroach sql), add a new
+  # sibling key (e.g., hooks.cockroachImage) rather than overloading this one.
+  kubectlImage:
+    # registry defines the container registry for the hook image.
+    registry: docker.io
+    # repository defines the image repository for the hook container.
+    repository: dtzar/helm-kubectl
+    # tag defines the image tag for the hook container. Pinned for reproducibility
+    # in air-gapped environments.
+    tag: "3.19"
+    # pullPolicy defines the image pull policy for the hook container.
+    pullPolicy: IfNotPresent

cockroachdb-parent/charts/cockroachdb/templates/pre-upgrade-validation.yaml

@@ -21,7 +21,8 @@ spec:
       restartPolicy: Never
       containers:
         - name: validation
-          image: bitnami/kubectl:latest
+          image: "{{ .Values.hooks.kubectlImage.registry }}/{{ .Values.hooks.kubectlImage.repository }}:{{ .Values.hooks.kubectlImage.tag }}"
+          imagePullPolicy: {{ .Values.hooks.kubectlImage.pullPolicy }}
           command:
             - /bin/bash
             - -c

cockroachdb-parent/charts/cockroachdb/values.yaml

@@ -545,6 +545,24 @@ cockroachdb:
     #   # Possible Values: disabled, primary, standby
     #   mode: "disabled"
 
+# hooks captures the configuration for Helm hook jobs (pre-upgrade validation, etc.).
+# All hook jobs use the image defined here by default. Override these
+# values to use images from an internal registry in air-gapped environments.
+hooks:
+  # kubectlImage is the container image used by hook jobs that run kubectl commands.
+  # If a future hook requires a different image (e.g., cockroach sql), add a new
+  # sibling key (e.g., hooks.cockroachImage) rather than overloading this one.
+  kubectlImage:
+    # registry defines the container registry for the hook image.
+    registry: docker.io
+    # repository defines the image repository for the hook container.
+    repository: dtzar/helm-kubectl
+    # tag defines the image tag for the hook container. Pinned for reproducibility
+    # in air-gapped environments.
+    tag: "3.19"
+    # pullPolicy defines the image pull policy for the hook container.
+    pullPolicy: IfNotPresent
+
 k8s:
   # nameOverride overrides the name of the chart. If not set, the chart name will be used.
   # For example, if the chart name is "cockroachdb" and nameOverride is set to "crdb",

cockroachdb-parent/charts/operator/templates/pre-upgrade-validation.yaml

@@ -22,7 +22,8 @@ spec:
       restartPolicy: Never
       containers:
         - name: pre-upgrade-check
-          image: dtzar/helm-kubectl:latest
+          image: "{{ .Values.hooks.kubectlImage.registry }}/{{ .Values.hooks.kubectlImage.repository }}:{{ .Values.hooks.kubectlImage.tag }}"
+          imagePullPolicy: {{ .Values.hooks.kubectlImage.pullPolicy }}
           command: ["/bin/bash", "-c"]
           args:
             - |

cockroachdb-parent/charts/operator/values.yaml

@@ -39,3 +39,20 @@ appLabel: "cockroach-operator"
 # startup; certs rotate on every pod restart. When false (default), Helm provisions a stable
 # cockroach-operator-certs Secret. Switching requires deleting that Secret first.
 selfSignedOperatorCerts: false
+# hooks captures the configuration for Helm hook jobs (pre-upgrade validation, etc.).
+# All hook jobs use the image defined here by default. Override these
+# values to use images from an internal registry in air-gapped environments.
+hooks:
+  # kubectlImage is the container image used by hook jobs that run kubectl commands.
+  # If a future hook requires a different image (e.g., cockroach sql), add a new
+  # sibling key (e.g., hooks.cockroachImage) rather than overloading this one.
+  kubectlImage:
+    # registry defines the container registry for the hook image.
+    registry: docker.io
+    # repository defines the image repository for the hook container.
+    repository: dtzar/helm-kubectl
+    # tag defines the image tag for the hook container. Pinned for reproducibility
+    # in air-gapped environments.
+    tag: "3.19"
+    # pullPolicy defines the image pull policy for the hook container.
+    pullPolicy: IfNotPresent

cockroachdb-parent/images.txt

@@ -0,0 +1,38 @@
+# CockroachDB Helm Charts - Container Image Manifest
+#
+# This file lists all container images required by the CockroachDB Helm charts.
+# Use it to mirror images into an internal registry for air-gapped environments.
+#
+# Format: <registry>/<repository>:<tag> or <registry>/<repository>@<digest>
+#
+# To mirror all images, run:
+#   scripts/mirror-images.sh --source-file cockroachdb-parent/images.txt --target-registry <your-registry>
+
+# === Operator chart images ===
+
+# CockroachDB Operator
+us-docker.pkg.dev/releases-prod/self-hosted/cockroachdb-operator@sha256:62796d0ad9c87cb0f1f85ebcbc905235ca6c1fc3549bf2a62f5c534af0497332
+
+# Operator pre-upgrade validation hook
+docker.io/dtzar/helm-kubectl:3.19
+
+# === CockroachDB chart images ===
+
+# CockroachDB database
+docker.io/cockroachdb/cockroach:v26.1.2
+
+# Certificate self-signer
+gcr.io/cockroachlabs-helm-charts/cockroach-self-signer-cert:1.9
+
+# CockroachDB pre-upgrade validation hook
+docker.io/dtzar/helm-kubectl:3.19
+
+# === Operator-managed images (embedded in operator binary) ===
+# These images are pulled by the operator at runtime, not referenced in Helm values.
+# They must also be mirrored for air-gapped environments.
+
+# Init container
+us-docker.pkg.dev/releases-prod/self-hosted/init-container@sha256:bcfc9312af84c7966f017c2325981b30314c0c293491f942e54da1667bedaf69
+
+# Cert reloader (inotifywait)
+us-docker.pkg.dev/releases-prod/self-hosted/inotifywait@sha256:94db3d416e3df8d1ee2605f05b0526b3bd7217ae69b4eeb147929fe0b77aeafc

scripts/mirror-images.sh

@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+#
+# Mirror all container images required by CockroachDB Helm charts to an
+# internal registry. Designed for air-gapped and restricted environments.
+#
+# Prerequisites: crane (https://github.com/google/go-containerregistry/tree/main/cmd/crane)
+#   or skopeo (https://github.com/containers/skopeo)
+#
+# Usage:
+#   ./scripts/mirror-images.sh --target-registry my-registry.internal.io
+#   ./scripts/mirror-images.sh --target-registry my-registry.internal.io --source-file cockroachdb-parent/images.txt
+#   ./scripts/mirror-images.sh --target-registry my-registry.internal.io --tool skopeo
+#   ./scripts/mirror-images.sh --target-registry my-registry.internal.io --dry-run
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+
+TARGET_REGISTRY=""
+SOURCE_FILE="${REPO_ROOT}/cockroachdb-parent/images.txt"
+TOOL="crane"
+DRY_RUN=false
+
+usage() {
+  cat <<EOF
+Usage: $(basename "$0") [OPTIONS]
+
+Mirror CockroachDB Helm chart images to an internal registry.
+
+Options:
+  --target-registry REGISTRY   Target registry to push images to (required)
+  --source-file FILE           Path to images.txt manifest (default: cockroachdb-parent/images.txt)
+  --tool TOOL                  Tool to use for mirroring: crane or skopeo (default: crane)
+  --dry-run                    Print commands without executing
+  -h, --help                   Show this help message
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --target-registry) TARGET_REGISTRY="$2"; shift 2 ;;
+    --source-file)     SOURCE_FILE="$2"; shift 2 ;;
+    --tool)            TOOL="$2"; shift 2 ;;
+    --dry-run)         DRY_RUN=true; shift ;;
+    -h|--help)         usage; exit 0 ;;
+    *)                 echo "Unknown option: $1"; usage; exit 1 ;;
+  esac
+done
+
+if [[ -z "${TARGET_REGISTRY}" ]]; then
+  echo "Error: --target-registry is required"
+  usage
+  exit 1
+fi
+
+if [[ ! -f "${SOURCE_FILE}" ]]; then
+  echo "Error: source file not found: ${SOURCE_FILE}"
+  exit 1
+fi
+
+if [[ "${TOOL}" != "crane" && "${TOOL}" != "skopeo" ]]; then
+  echo "Error: --tool must be 'crane' or 'skopeo'"
+  exit 1
+fi
+
+if ! command -v "${TOOL}" &>/dev/null; then
+  echo "Error: ${TOOL} not found in PATH"
+  echo "Install instructions:"
+  if [[ "${TOOL}" == "crane" ]]; then
+    echo "  https://github.com/google/go-containerregistry/tree/main/cmd/crane#installation"
+  else
+    echo "  https://github.com/containers/skopeo/blob/main/install.md"
+  fi
+  exit 1
+fi
+
+# Compute the target image reference from a source reference.
+# Strips the source registry and prepends the target registry.
+# For digest references (@sha256:...), preserves the digest.
+compute_target() {
+  local src="$1"
+  local repo_and_ref
+
+  # Strip the registry prefix (everything before the first slash that isn't part of a path)
+  # Examples:
+  #   us-docker.pkg.dev/releases-prod/self-hosted/cockroachdb-operator@sha256:abc -> cockroachdb-operator@sha256:abc
+  #   docker.io/cockroachdb/cockroach:v26.1.2 -> cockroachdb/cockroach:v26.1.2
+  #   gcr.io/cockroachlabs-helm-charts/cockroach-self-signer-cert:1.9 -> cockroach-self-signer-cert:1.9
+  case "${src}" in
+    us-docker.pkg.dev/*/*/*)
+      # Google Artifact Registry: us-docker.pkg.dev/project/repo/image
+      # Strip us-docker.pkg.dev/project/repo/ to keep only image:tag or image@digest
+      local without_host="${src#us-docker.pkg.dev/}"
+      local without_project="${without_host#*/}"
+      repo_and_ref="${without_project#*/}"
+      ;;
+    gcr.io/*)
+      # GCR: gcr.io/project/image
+      repo_and_ref="${src#gcr.io/*/}"
+      ;;
+    docker.io/*)
+      repo_and_ref="${src#docker.io/}"
+      ;;
+    *)
+      # Fallback: strip first path segment as registry
+      repo_and_ref="${src#*/}"
+      ;;
+  esac
+
+  echo "${TARGET_REGISTRY}/${repo_and_ref}"
+}
+
+echo "Mirroring images from: ${SOURCE_FILE}"
+echo "Target registry: ${TARGET_REGISTRY}"
+echo "Tool: ${TOOL}"
+echo ""
+
+SUCCESS=0
+FAILED=0
+
+while IFS= read -r line; do
+  # Skip comments and blank lines
+  [[ "${line}" =~ ^[[:space:]]*# ]] && continue
+  [[ -z "${line// /}" ]] && continue
+
+  src="${line}"
+  dst="$(compute_target "${src}")"
+
+  echo "Mirroring: ${src}"
+  echo "      --> ${dst}"
+
+  if [[ "${DRY_RUN}" == true ]]; then
+    if [[ "${TOOL}" == "crane" ]]; then
+      echo "  [dry-run] crane copy ${src} ${dst}"
+    else
+      echo "  [dry-run] skopeo copy docker://${src} docker://${dst}"
+    fi
+    echo ""
+    continue
+  fi
+
+  if [[ "${TOOL}" == "crane" ]]; then
+    if crane copy "${src}" "${dst}"; then
+      echo "  OK"
+      ((++SUCCESS))
+    else
+      echo "  FAILED"
+      ((++FAILED))
+    fi
+  else
+    if skopeo copy "docker://${src}" "docker://${dst}"; then
+      echo "  OK"
+      ((++SUCCESS))
+    else
+      echo "  FAILED"
+      ((++FAILED))
+    fi
+  fi
+  echo ""
+done < "${SOURCE_FILE}"
+
+echo "================================"
+echo "Mirroring complete: ${SUCCESS} succeeded, ${FAILED} failed"
+
+if [[ "${FAILED}" -gt 0 ]]; then
+  exit 1
+fi