preload everything in gha due to rate-limiting

Signed-off-by: Zoe Blevins <zblevins@nvidia.com>

Author: zsblevins

deploy/configs/ci-integration-test.yaml

@@ -77,6 +77,34 @@ services:
   config_store: true
   nautobot: true
 
+images:
+  # CI preloads third-party runtime images into Kind through the host Docker daemon.
+  # This avoids anonymous in-cluster pulls from Docker Hub during Helm's --wait window.
+  kind_preload_images:
+    - docker.io/hashicorp/http-echo:1.0
+    - docker.io/alpine/kubectl:1.35.4
+    - docker.io/library/redis:7-alpine
+    - docker.io/library/nats:2.10-alpine
+    - docker.io/natsio/nats-box:0.14.3
+    - docker.io/temporalio/server:1.29
+    - docker.io/temporalio/admin-tools:1.29
+    - docker.io/temporalio/ui:v2.37.4
+    - docker.io/nginxinc/nginx-unprivileged:1.27
+    - docker.io/envoyproxy/gateway:v1.6.5
+    - docker.io/envoyproxy/ratelimit:c8765e89
+    - docker.io/envoyproxy/envoy:distroless-v1.36.5
+    - quay.io/jetstack/cert-manager-controller:v1.20.2
+    - quay.io/jetstack/cert-manager-webhook:v1.20.2
+    - quay.io/jetstack/cert-manager-cainjector:v1.20.2
+    - quay.io/jetstack/cert-manager-startupapicheck:v1.20.2
+    - quay.io/jetstack/cert-manager-acmesolver:v1.20.2
+    - ghcr.io/cloudnative-pg/cloudnative-pg:1.29.0
+    - ghcr.io/cloudnative-pg/postgresql:18.0-system-trixie
+  overrides:
+    postgresql:
+      repository: ghcr.io/cloudnative-pg/postgresql
+      tag: 18.0-system-trixie
+
 infrastructure:
   gateway: envoyGateway
   tls: true

docs/install/index.mdx

@@ -173,6 +173,8 @@ nv-config-manager-installer deploy nv-config-manager-install.yaml \
 
 For a fresh Kind deployment that builds local images, the local-image flags and the three operator install flags are required unless those operators are already installed and images are already available to the cluster. If the Kind cluster name is not `nv-config-manager`, pass the actual name with `--kind-cluster`.
 
+In CI environments where in-cluster pulls are unreliable or rate limited, `--load-kind` can also preload selected third-party runtime images before Helm starts. Add image references under `images.kind_preload_images` in the installer YAML, or pass a comma- or space-separated override with `NVCM_KIND_PRELOAD_IMAGES`.
+
 | Flag | Default | Description |
 | :--- | :------ | :---------- |
 | `--chart-dir` | `deploy/helm` | Path to the Helm chart |

installer/src/nv_config_manager_installer/deployer.py

@@ -282,9 +282,16 @@ def _docker_server_platform() -> str:
     return f"linux/{arch}"
 
 
+def _docker_image_id(image: str) -> str:
+    """Return Docker's local image ID for an image reference."""
+    result = _run(["docker", "image", "inspect", "--format", "{{.Id}}", image], check=True)
+    return result.stdout.strip()
+
+
 _DEFAULT_IMAGE_REGISTRY = "nvcr.io/nvidian/cfa"
 _CNPG_OPERATOR_IMAGE_TAG = "1.29.0"
 _PROMETHEUS_OPERATOR_CRDS_CHART_VERSION = "28.0.1"
+_KIND_PRELOAD_IMAGES_ENV = "NVCM_KIND_PRELOAD_IMAGES"
 _KIND_PRELOAD_IMAGES = (LOADER_POD_IMAGE,)
 _KIND_PRELOAD_PLATFORM_IMAGES: dict[str, dict[str, str]] = {
     LOADER_POD_IMAGE: {
@@ -299,6 +306,36 @@ def _kind_preload_source_image(image: str, platform_name: str) -> str:
     return _KIND_PRELOAD_PLATFORM_IMAGES.get(image, {}).get(platform_name, image)
 
 
+def _env_kind_preload_images() -> list[str]:
+    """Return additional Kind preload images from the environment."""
+    value = os.environ.get(_KIND_PRELOAD_IMAGES_ENV, "")
+    return [part for part in value.replace(",", " ").split() if part]
+
+
+def _kind_preload_images(config: NVConfigManagerInstallConfig) -> list[str]:
+    """Return the effective ordered list of images to preload into Kind."""
+    images: list[str] = []
+    seen: set[str] = set()
+    for image in (
+        *_KIND_PRELOAD_IMAGES,
+        *config.images.kind_preload_images,
+        *_env_kind_preload_images(),
+    ):
+        image = image.strip()
+        if not image or image in seen:
+            continue
+        seen.add(image)
+        images.append(image)
+    return images
+
+
+def _kind_node_names(cluster: str) -> list[str]:
+    """Return Docker container names for nodes in a Kind cluster."""
+    result = _run(["kind", "get", "nodes", "--name", cluster], check=True)
+    nodes = [line.strip() for line in result.stdout.splitlines() if line.strip()]
+    return nodes or [f"{cluster}-control-plane"]
+
+
 def _strip_image_registry(repository: str) -> str:
     """Return repository path with any registry host removed."""
     first, sep, rest = repository.partition("/")
@@ -457,6 +494,90 @@ def _run_logged(
     return subprocess.CompletedProcess(cmd, proc.returncode, stdout_text, stderr_text)
 
 
+def _run_logged_pipe(
+    source_cmd: list[str],
+    sink_cmd: list[str],
+    step: DeployStep,
+    callback: DeployCallback,
+    *,
+    check: bool = True,
+    timeout: int | None = 600,
+) -> subprocess.CompletedProcess[str]:
+    """Run source_cmd with stdout piped to sink_cmd, streaming command output."""
+    source_str = " ".join(source_cmd[:4]) + ("..." if len(source_cmd) > 4 else "")
+    sink_str = " ".join(sink_cmd[:4]) + ("..." if len(sink_cmd) > 4 else "")
+    callback.on_log(f"$ {source_str} | {sink_str}")
+
+    source = subprocess.Popen(source_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    sink = subprocess.Popen(
+        sink_cmd,
+        stdin=source.stdout,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    if source.stdout:
+        source.stdout.close()
+
+    stdout_lines: list[str] = []
+    stderr_lines: list[str] = []
+    source_stderr_lines: list[str] = []
+    deadline = time.monotonic() + timeout if timeout else None
+    sel = selectors.DefaultSelector()
+    try:
+        if source.stderr:
+            sel.register(source.stderr, selectors.EVENT_READ, "source-stderr")
+        if sink.stdout:
+            sel.register(sink.stdout, selectors.EVENT_READ, "sink-stdout")
+        if sink.stderr:
+            sel.register(sink.stderr, selectors.EVENT_READ, "sink-stderr")
+
+        while sel.get_map():
+            for proc, cmd in ((source, source_cmd), (sink, sink_cmd)):
+                _check_deadline(deadline, proc, cmd, timeout)
+            wait = None if deadline is None else max(0.0, deadline - time.monotonic())
+            for key, _ in sel.select(timeout=wait):
+                line_bytes = key.fileobj.readline()  # type: ignore[union-attr]
+                if not line_bytes:
+                    sel.unregister(key.fileobj)
+                    continue
+                line = line_bytes.decode(errors="replace").rstrip("\n")
+                step.output.append(line)
+                callback.on_log(line)
+                if key.data == "sink-stdout":
+                    stdout_lines.append(line)
+                elif key.data == "sink-stderr":
+                    stderr_lines.append(line)
+                else:
+                    source_stderr_lines.append(line)
+
+        source.wait()
+        sink.wait()
+    except BaseException:
+        source.kill()
+        sink.kill()
+        source.wait()
+        sink.wait()
+        raise
+    finally:
+        sel.close()
+
+    stdout_text = "\n".join(stdout_lines)
+    stderr_text = "\n".join(stderr_lines)
+    source_stderr_text = "\n".join(source_stderr_lines)
+
+    if check and sink.returncode != 0:
+        raise subprocess.CalledProcessError(sink.returncode, sink_cmd, stdout_text, stderr_text)
+    if check and source.returncode != 0:
+        raise subprocess.CalledProcessError(
+            source.returncode,
+            source_cmd,
+            "",
+            source_stderr_text,
+        )
+
+    return subprocess.CompletedProcess(sink_cmd, sink.returncode, stdout_text, stderr_text)
+
+
 def _short_pod_text(value: str, *, limit: int = 140) -> str:
     """Return a compact single-line pod status detail."""
     value = " ".join(value.split())
@@ -1193,10 +1314,13 @@ def _load_kind(self) -> None:
                 timeout=300,
             )
         platform_name = _docker_server_platform()
-        for img in _KIND_PRELOAD_IMAGES:
+        for img in _kind_preload_images(self.config):
             # Pull arch-scoped official images when available. Docker can keep
             # multi-platform tags as manifest lists even after --platform,
-            # which makes kind's image import fail on missing platform content.
+            # which makes kind's default --all-platforms import fail on missing
+            # platform content. Import helper images directly into node
+            # containerd with the selected platform instead. Avoid ctr's
+            # digest refs here; the tag is what Kubernetes needs to resolve.
             source_img = _kind_preload_source_image(img, platform_name)
             self.callback.on_log(
                 f"Pulling helper image {source_img} for platform {platform_name}..."
@@ -1207,23 +1331,50 @@ def _load_kind(self) -> None:
                 self.callback,
                 timeout=300,
             )
-            if source_img != img:
-                self.callback.on_log(f"Tagging helper image {source_img} as {img}...")
-                _run_logged(
-                    ["docker", "tag", source_img, img],
-                    step,
-                    self.callback,
-                    timeout=300,
-                )
-            self.callback.on_log(f"Loading helper image {img} into Kind cluster {cluster}...")
+            image_id = _docker_image_id(source_img)
+            self.callback.on_log(f"Tagging selected-platform helper image {image_id} as {img}...")
             _run_logged(
-                ["kind", "load", "docker-image", img, "--name", cluster],
+                ["docker", "tag", image_id, img],
                 step,
                 self.callback,
                 timeout=300,
             )
+            self._load_kind_helper_image(img, cluster, platform_name, step)
         self._finish_step(step)
 
+    def _load_kind_helper_image(
+        self,
+        image: str,
+        cluster: str,
+        platform_name: str,
+        step: DeployStep,
+    ) -> None:
+        """Load a helper image into Kind node containerd for one platform."""
+        nodes = _kind_node_names(cluster)
+        for node in nodes:
+            self.callback.on_log(f"Loading helper image {image} into Kind node {node}...")
+            _run_logged_pipe(
+                ["docker", "save", image],
+                [
+                    "docker",
+                    "exec",
+                    "--privileged",
+                    "-i",
+                    node,
+                    "ctr",
+                    "--namespace=k8s.io",
+                    "images",
+                    "import",
+                    "--platform",
+                    platform_name,
+                    "--snapshotter=overlayfs",
+                    "-",
+                ],
+                step,
+                self.callback,
+                timeout=300,
+            )
+
     def _operator_bundle_root(self) -> Path | None:
         """Return the airgap bundle root if local charts/manifests are available."""
         chart_dir = Path(self.options.chart_dir).expanduser()

installer/src/nv_config_manager_installer/schema.py

@@ -580,6 +580,7 @@ class ImagesConfig(BaseModel):
     registry: str = "nvcr.io/nvidian/cfa"
     tag: str = ""
     pull_policy: str = "IfNotPresent"
+    kind_preload_images: list[str] = Field(default_factory=list)
     pull_secret: ImagePullSecret = Field(default_factory=ImagePullSecret)
     overrides: dict[str, ImageOverride] = Field(default_factory=dict)
 
@@ -927,9 +928,11 @@ def _prune_ztp_storage(ztp_storage: dict[str, Any]) -> None:
 
 
 def _prune_images(images: dict[str, Any]) -> None:
+    if not images.get("kind_preload_images"):
+        images.pop("kind_preload_images", None)
     if images.get("source") != ImageSource.LOCAL.value:
         return
-    _replace_with_keys(images, {"source"})
+    _replace_with_keys(images, {"source", "kind_preload_images"})
 
 
 def _prune_redfish(redfish: dict[str, Any]) -> None: