fix buildkit behavior + code rabbit feedback

Signed-off-by: Zoe Blevins <zblevins@nvidia.com>

Author: zsblevins

.github/workflows/kind-integration.yml

@@ -120,15 +120,36 @@ jobs:
         run: |
           for attempt in 1 2 3; do
             echo "Attempt ${attempt} of 3"
-            if timeout --kill-after=1m 15m uv run pytest "${TEST_PATH}" -v \
+            results_file="integration-test-results-${attempt}.xml"
+            set +e
+            timeout --kill-after=1m 15m uv run pytest "${TEST_PATH}" -v \
               --ci \
               --nv-config-manager-namespace "${NV_CONFIG_MANAGER_NAMESPACE}" \
               --tb=short \
-              --junitxml=integration-test-results.xml; then
+              --junitxml="${results_file}"
+            test_status=$?
+            set -e
+
+            if [ "${test_status}" -eq 0 ]; then
+              cp -f "${results_file}" integration-test-results.xml
               exit 0
             fi
-            if [ "${attempt}" -lt 3 ]; then
-              sleep 10
+
+            if [ -f "${results_file}" ]; then
+              cp -f "${results_file}" integration-test-results.xml
+            fi
+
+            if [ "${test_status}" -eq 124 ] || \
+              { [ "${test_status}" -ge 128 ] && [ "${test_status}" -le 192 ]; }; then
+              echo "Attempt ${attempt} failed with retryable status ${test_status}"
+              if [ "${attempt}" -lt 3 ]; then
+                sleep 10
+                continue
+              fi
+              exit "${test_status}"
+            else
+              echo "Attempt ${attempt} failed with non-retryable status ${test_status}"
+              exit "${test_status}"
             fi
           done
           exit 1

installer/src/nv_config_manager_installer/deployer.py

@@ -925,6 +925,7 @@ def _build_images(self) -> None:
                 apt_mirror_args += ["--build-arg", f"{env_var}={val}"]
 
         build_env = {**os.environ, "DOCKER_BUILDKIT": "1"}
+        build_output_args = ["--load"] if build_env.get("BUILDX_BUILDER") else []
         build_commands: list[_ParallelCommand] = []
         for name, dockerfile, context in images:
             build_tag = f"{name}:local"
@@ -936,6 +937,7 @@ def _build_images(self) -> None:
                         "build",
                         "--provenance=false",
                         "--progress=plain",
+                        *build_output_args,
                         "--build-context",
                         "scripts=build/",
                         *apt_mirror_args,

installer/tests/test_deployer.py

@@ -462,6 +462,7 @@ def fake_run(cmd, **kwargs):
             return MagicMock(returncode=0, stdout="", stderr="")
 
         monkeypatch.setenv("NVCM_DOCKER_BUILD_PARALLELISM", "2")
+        monkeypatch.delenv("BUILDX_BUILDER", raising=False)
         monkeypatch.setattr(
             "nv_config_manager_installer.deployer._run_logged_parallel",
             fake_run_logged_parallel,
@@ -486,12 +487,45 @@ def fake_run(cmd, **kwargs):
             command.cmd[:4] == ["docker", "build", "--provenance=false", "--progress=plain"]
             for command in commands
         )
+        assert all("--load" not in command.cmd for command in commands)
         assert all(command.timeout == 900 for command in commands)
         assert all(command.env and command.env["DOCKER_BUILDKIT"] == "1" for command in commands)
         assert len(run_commands) == 6
         assert all(cmd[:2] == ["docker", "tag"] for cmd in run_commands)
         assert deployer._local_image_tags["nv-config-manager-ui"].startswith("sha-")
 
+    def test_build_images_loads_buildx_container_outputs(self, monkeypatch):
+        parallel_calls: list[list[_ParallelCommand]] = []
+
+        def fake_run_logged_parallel(commands, step, callback, *, max_parallel, **kwargs):
+            parallel_calls.append(commands)
+            for command in commands:
+                callback.on_log(f"[{command.label}] completed in 0s")
+
+        monkeypatch.setenv("BUILDX_BUILDER", "ci-builder")
+        monkeypatch.setattr(
+            "nv_config_manager_installer.deployer._run_logged_parallel",
+            fake_run_logged_parallel,
+        )
+        monkeypatch.setattr(
+            "nv_config_manager_installer.deployer._get_image_digest_tag",
+            lambda image: "",
+        )
+
+        deployer = Deployer(
+            _make_config(),
+            DeployOptions(build_images=True),
+            RecordingCallback(),
+        )
+        deployer._build_images()
+
+        commands = parallel_calls[0]
+        assert len(commands) == 6
+        assert all("--load" in command.cmd for command in commands)
+        assert all(
+            command.env and command.env["BUILDX_BUILDER"] == "ci-builder" for command in commands
+        )
+
 
 class TestKindImageLoading:
     def test_load_kind_tags_arch_specific_loader_image_as_canonical(self, monkeypatch):