refactor(agent): Skip CpusetMems on non-NUMA hosts and clean dummy tombstone

Addresses review feedback on PR #11222:
- Skip CpusetMems entirely when libnuma reports NUMA unsupported or
  the host has a single node, so non-NUMA Linux hosts and macOS/WSL
  dev environments do not get CpusetMems="0" pinned on every container.
- Add a unit test covering the non-NUMA short-circuit branch.
- Remove the twin commented-out CpusetMems tombstone from
  src/ai/backend/agent/dummy/intrinsic.py.

Refs #11217
Refs #11222

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: rapsealk

src/ai/backend/agent/docker/intrinsic.py

@@ -437,18 +437,23 @@ async def generate_docker_args(
             "Cpus": len(cores),
             "CpusetCpus": ",".join(sorted_core_ids),
         }
-        allocated_nodes: set[int] = set()
-        for core in cores:
-            node = libnuma.node_of_cpu(core)
-            if node < 0:
-                allocated_nodes.clear()
-                break
-            allocated_nodes.add(node)
-        # Pin memory only when the CPU allocation is fully node-local.
-        # For multi-node CPU allocations, intentionally leave CpusetMems unset
-        # so Docker/kernel default NUMA memory placement policy can apply.
-        if len(allocated_nodes) == 1:
-            host_config["CpusetMems"] = str(next(iter(allocated_nodes)))
+        # Skip CpusetMems entirely when NUMA is unsupported (non-Linux hosts,
+        # Linux without libnuma.so, Docker Desktop, WSL, etc.) or when the host
+        # exposes a single node; libnuma.node_of_cpu would otherwise fall back
+        # to 0 and cause every container to be pinned to "CpusetMems": "0".
+        if libnuma.num_nodes() > 1:
+            allocated_nodes: set[int] = set()
+            for core in cores:
+                node = libnuma.node_of_cpu(core)
+                if node < 0:
+                    allocated_nodes.clear()
+                    break
+                allocated_nodes.add(node)
+            # Pin memory only when the CPU allocation is fully node-local.
+            # For multi-node CPU allocations, intentionally leave CpusetMems unset
+            # so Docker/kernel default NUMA memory placement policy can apply.
+            if len(allocated_nodes) == 1:
+                host_config["CpusetMems"] = str(next(iter(allocated_nodes)))
         return {
             "HostConfig": host_config,
         }

src/ai/backend/agent/dummy/intrinsic.py

@@ -145,7 +145,6 @@ async def generate_docker_args(
                 "CpuQuota": int(100_000 * len(cores)),
                 "Cpus": ",".join(sorted_core_ids),
                 "CpusetCpus": ",".join(sorted_core_ids),
-                # 'CpusetMems': f'{resource_spec.numa_node}',
             },
         }
 

tests/unit/agent/test_docker_intrinsic.py

@@ -632,13 +632,28 @@ def _device_alloc(core_ids: list[int]) -> dict[SlotName, dict[DeviceId, Decimal]
 
     @staticmethod
     @contextmanager
-    def _patch_node_of_cpu(core_to_node: dict[int, int]) -> Generator[None, None, None]:
+    def _patch_node_of_cpu(
+        core_to_node: dict[int, int],
+        *,
+        num_nodes: int = 2,
+    ) -> Generator[None, None, None]:
         """Patch libnuma.node_of_cpu; return -1 for any core missing from the map
         (matches real libnuma's behavior for unknown cores when NUMA info is
-        unavailable)."""
-        with patch(
-            "ai.backend.agent.docker.intrinsic.libnuma.node_of_cpu",
-            side_effect=lambda core: core_to_node.get(core, -1),
+        unavailable).
+
+        Also patches libnuma.num_nodes to report a multi-node host by default
+        so the NUMA-aware branch is exercised. Tests covering the non-NUMA
+        short-circuit can pass ``num_nodes=1``.
+        """
+        with (
+            patch(
+                "ai.backend.agent.docker.intrinsic.libnuma.num_nodes",
+                return_value=num_nodes,
+            ),
+            patch(
+                "ai.backend.agent.docker.intrinsic.libnuma.node_of_cpu",
+                side_effect=lambda core: core_to_node.get(core, -1),
+            ),
         ):
             yield
 
@@ -710,3 +725,27 @@ async def test_unknown_or_negative_node_omits_cpuset_mems(
         # Sanity: core-list plumbing still works.
         assert host_config["Cpus"] == 2
         assert host_config["CpusetCpus"] == "0,1"
+
+    async def test_non_numa_host_omits_cpuset_mems(
+        self,
+        cpu_plugin: CPUPlugin,
+    ) -> None:
+        """On non-NUMA / non-Linux hosts (macOS, Docker Desktop, WSL, Linux
+        without libnuma.so) libnuma.num_nodes() reports 1 and node_of_cpu()
+        hardcodes 0. The plugin must short-circuit before inspecting per-core
+        nodes so containers are not unconditionally pinned to CpusetMems="0".
+        """
+        # node_of_cpu would return 0 for every core on a non-NUMA host; assert
+        # we never reach that branch by mapping cores to a bogus node that
+        # would otherwise produce a stale CpusetMems assignment.
+        with self._patch_node_of_cpu({0: 0, 1: 0}, num_nodes=1):
+            result = await cpu_plugin.generate_docker_args(
+                AsyncMock(),
+                self._device_alloc([0, 1]),
+            )
+
+        host_config = result["HostConfig"]
+        assert "CpusetMems" not in host_config
+        # Sanity: core-list plumbing still works.
+        assert host_config["Cpus"] == 2
+        assert host_config["CpusetCpus"] == "0,1"