Set per-process memory limits on getdeps build subprocesses

Summary:
getdeps computes build parallelism from available RAM but never
enforces the per-job memory budget. If a compiler or linker exceeds
its expected usage, the Linux OOM killer can terminate arbitrary
processes including the user's terminal.

Add RLIMIT_AS enforcement via preexec_fn on build subprocesses. The
limit is set to 3x job_weight_mib to account for virtual address
space being larger than resident memory. This causes runaway
processes to get std::bad_alloc/ENOMEM instead of triggering the
OOM killer. The limit is inherited by all child processes (ninja,
compiler invocations).

Reviewed By: jbower-fb, bigfootjon

Differential Revision: D94983478

fbshipit-source-id: 44ac784351f55e1b1a67ef43c8beb1ddf805d596

Author: r-barnes

build/fbcode_builder/getdeps/builder.py

@@ -24,7 +24,7 @@
 from .dyndeps import create_dyn_dep_munger
 from .envfuncs import add_path_entry, Env, path_search
 from .fetcher import copy_if_different, is_public_commit
-from .runcmd import run_cmd
+from .runcmd import make_memory_limit_preexec_fn, run_cmd
 
 if typing.TYPE_CHECKING:
     from .buildopts import BuildOptions
@@ -100,6 +100,7 @@ def _run_cmd(
         env=None,
         use_cmd_prefix: bool = True,
         allow_fail: bool = False,
+        preexec_fn=None,
     ) -> int:
         if env:
             e = self.env.copy()
@@ -120,6 +121,7 @@ def _run_cmd(
             cwd=cwd or self.build_dir,
             log_file=log_file,
             allow_fail=allow_fail,
+            preexec_fn=preexec_fn,
         )
 
     def _reconfigure(self, reconfigure: bool) -> bool:
@@ -229,7 +231,7 @@ def build(self, reconfigure: bool) -> None:
             dep_munger.emit_dev_run_script(script_path, dep_dirs)
 
     @property
-    def num_jobs(self) -> int:
+    def _job_weight_mib(self) -> int:
         # This is a hack, but we don't have a "defaults manifest" that we can
         # customize per platform.
         # TODO: Introduce some sort of defaults config that can select by
@@ -241,14 +243,25 @@ def num_jobs(self) -> int:
             # 1.5 GiB is a lot to assume, but it's typical of Facebook-style C++.
             # Some manifests are even heavier and should override.
             default_job_weight = 1536
-        return self.build_opts.get_num_jobs(
-            int(
-                self.manifest.get(
-                    "build", "job_weight_mib", default_job_weight, ctx=self.ctx
-                )
+        return int(
+            self.manifest.get(
+                "build", "job_weight_mib", default_job_weight, ctx=self.ctx
             )
         )
 
+    @property
+    def num_jobs(self) -> int:
+        return self.build_opts.get_num_jobs(self._job_weight_mib)
+
+    @property
+    def memory_limit_preexec_fn(self):
+        """Return a preexec_fn that caps per-process virtual memory.
+
+        Uses the same job_weight_mib that controls parallelism, so the memory
+        limit is consistent with the parallelism budget.
+        """
+        return make_memory_limit_preexec_fn(self._job_weight_mib)
+
     def run_tests(
         self,
         schedule_type,
@@ -933,6 +946,7 @@ def _build(self, reconfigure: bool) -> None:
                 str(self.num_jobs),
             ],
             env=env,
+            preexec_fn=self.memory_limit_preexec_fn,
         )
 
     def _build_targets(self, targets: typing.Sequence[str]) -> None:
@@ -969,7 +983,7 @@ def _build_targets(self, targets: typing.Sequence[str]) -> None:
             ]
         )
 
-        self._check_cmd(cmd, env=env)
+        self._check_cmd(cmd, env=env, preexec_fn=self.memory_limit_preexec_fn)
 
     def _get_missing_test_executables(
         self, test_filter: Optional[str], env: Env, ctest: Optional[str]

build/fbcode_builder/getdeps/runcmd.py

@@ -5,6 +5,8 @@
 
 # pyre-unsafe
 
+from __future__ import annotations
+
 import os
 import select
 import subprocess
@@ -19,6 +21,53 @@ class RunCommandError(Exception):
     pass
 
 
+def make_memory_limit_preexec_fn(
+    job_weight_mib: int,
+) -> object | None:
+    """Create a preexec_fn that sets a per-process virtual memory limit.
+
+    When getdeps spawns build commands (cmake -> ninja -> N compiler processes),
+    the parallelism is computed from available RAM divided by job_weight_mib.
+    However, there is no enforcement of that budget: if a compiler or linker
+    process exceeds its expected memory usage, the system can run out of RAM
+    and the Linux OOM killer may terminate arbitrary processes — including the
+    user's shell or terminal.
+
+    This function returns a callable suitable for subprocess.Popen's preexec_fn
+    parameter. It runs in each child process after fork() but before exec(),
+    setting RLIMIT_AS (virtual address space limit) so that a runaway process
+    gets a failed allocation (std::bad_alloc / ENOMEM) instead of triggering
+    the OOM killer. The limit is inherited by all descendant processes (ninja,
+    compiler invocations, etc.).
+
+    The per-process limit is set to job_weight_mib * 10. The 10x multiplier
+    accounts for the fact that RLIMIT_AS caps virtual address space, which is
+    typically 2-4x larger than resident (physical) memory for C++ compilers
+    due to memory-mapped files, shared libraries, and address space reservations
+    that don't consume physical RAM. The multiplier is intentionally generous:
+    the goal is a safety net that catches genuine runaways before the OOM killer
+    fires, not a tight per-job budget.
+
+    Only applies on Linux, where the OOM killer is the problem. Returns None
+    on other platforms.
+    """
+    if sys.platform != "linux":
+        return None
+
+    # Each job is budgeted job_weight_mib of physical RAM. Virtual address
+    # space is typically 2-4x RSS. Use 10x as a generous safety net: tight
+    # enough to stop a runaway process before the OOM killer fires, but loose
+    # enough to avoid false positives from normal virtual memory overhead.
+    limit_bytes = job_weight_mib * 10 * 1024 * 1024
+
+    def _set_memory_limit():
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (limit_bytes, limit_bytes))
+
+    return _set_memory_limit
+
+
 def _print_env_diff(env, log_fn) -> None:
     current_keys = set(os.environ.keys())
     wanted_env = set(env.keys())
@@ -48,7 +97,9 @@ def check_cmd(cmd, **kwargs) -> None:
         raise RuntimeError(f"Failure exit code {rc} for command {cmd}")
 
 
-def run_cmd(cmd, env=None, cwd=None, allow_fail: bool = False, log_file=None) -> int:
+def run_cmd(
+    cmd, env=None, cwd=None, allow_fail: bool = False, log_file=None, preexec_fn=None
+) -> int:
     def log_to_stdout(msg):
         sys.stdout.buffer.write(msg.encode(errors="surrogateescape"))
 
@@ -60,15 +111,25 @@ def log_function(msg):
                 log_to_stdout(msg)
 
             return _run_cmd(
-                cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_function
+                cmd,
+                env=env,
+                cwd=cwd,
+                allow_fail=allow_fail,
+                log_fn=log_function,
+                preexec_fn=preexec_fn,
             )
     else:
         return _run_cmd(
-            cmd, env=env, cwd=cwd, allow_fail=allow_fail, log_fn=log_to_stdout
+            cmd,
+            env=env,
+            cwd=cwd,
+            allow_fail=allow_fail,
+            log_fn=log_to_stdout,
+            preexec_fn=preexec_fn,
         )
 
 
-def _run_cmd(cmd, env, cwd, allow_fail, log_fn) -> int:
+def _run_cmd(cmd, env, cwd, allow_fail, log_fn, preexec_fn=None) -> int:
     log_fn("---\n")
     try:
         cmd_str = " \\\n+      ".join(shellquote(arg) for arg in cmd)
@@ -106,7 +167,12 @@ def _run_cmd(cmd, env, cwd, allow_fail, log_fn) -> int:
 
     try:
         p = subprocess.Popen(
-            cmd, env=env, cwd=cwd, stdout=stdout, stderr=subprocess.STDOUT
+            cmd,
+            env=env,
+            cwd=cwd,
+            stdout=stdout,
+            stderr=subprocess.STDOUT,
+            preexec_fn=preexec_fn,
         )
     except (TypeError, ValueError, OSError) as exc:
         log_fn("error running `%s`: %s" % (cmd_str, exc))