Merge pull request #1595 from merkelmarrow/1-build-dir-nfs-robustness-pr

Harden build dir deletion against transient NFS failures [CI 1/9]

Author: auphelia

src/finn/transformation/fpgadataflow/cleanup.py

@@ -29,9 +29,9 @@
 
 import os
 import qonnx.custom_op.registry as registry
-import shutil
 from qonnx.transformation.base import Transformation
 
+from finn.util.basic import robust_rmtree
 from finn.util.fpgadataflow import is_fpgadataflow_node
 
 
@@ -45,12 +45,12 @@ def apply(self, model):
         # delete PYNQ project, if any
         vivado_pynq_proj_dir = model.get_metadata_prop("vivado_pynq_proj")
         if vivado_pynq_proj_dir is not None and os.path.isdir(vivado_pynq_proj_dir):
-            shutil.rmtree(vivado_pynq_proj_dir)
+            robust_rmtree(vivado_pynq_proj_dir)
         model.set_metadata_prop("vivado_pynq_proj", "")
         # delete IP stitching project, if any
         ipstitch_path = model.get_metadata_prop("vivado_stitch_proj")
         if ipstitch_path is not None and os.path.isdir(ipstitch_path):
-            shutil.rmtree(ipstitch_path)
+            robust_rmtree(ipstitch_path)
         model.set_metadata_prop("vivado_stitch_proj", "")
         for node in model.graph.node:
             op_type = node.op_type
@@ -61,22 +61,22 @@ def apply(self, model):
                     # delete code_gen_dir from cppsim
                     code_gen_dir = inst.get_nodeattr("code_gen_dir_cppsim")
                     if os.path.isdir(code_gen_dir):
-                        shutil.rmtree(code_gen_dir)
+                        robust_rmtree(code_gen_dir)
                     inst.set_nodeattr("code_gen_dir_cppsim", "")
                     inst.set_nodeattr("executable_path", "")
                     # delete code_gen_dir from ipgen and project folder
                     code_gen_dir = inst.get_nodeattr("code_gen_dir_ipgen")
                     ipgen_path = inst.get_nodeattr("ipgen_path")
                     if os.path.isdir(code_gen_dir):
-                        shutil.rmtree(code_gen_dir)
+                        robust_rmtree(code_gen_dir)
                     if os.path.isdir(ipgen_path):
-                        shutil.rmtree(ipgen_path)
+                        robust_rmtree(ipgen_path)
                     inst.set_nodeattr("code_gen_dir_ipgen", "")
                     inst.set_nodeattr("ipgen_path", "")
                     # delete Java HotSpot Performance data log
                     for d_name in os.listdir("/tmp/"):
                         if "hsperfdata" in d_name:
-                            shutil.rmtree("/tmp/" + str(d_name))
+                            robust_rmtree("/tmp/" + str(d_name))
 
                 except KeyError:
                     # exception if op_type is not supported

src/finn/util/basic.py

@@ -26,11 +26,14 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import errno
 import os
 import re
+import shutil
 import subprocess
 import sys
 import tempfile
+import time
 from qonnx.core.modelwrapper import ModelWrapper
 from qonnx.custom_op.registry import getCustomOp
 from qonnx.util.basic import gen_finn_dt_tensor, roundup_to_integer_multiple
@@ -159,16 +162,38 @@ def make_build_dir(prefix=""):
     Use this function instead of tempfile.mkdtemp to ensure any generated files
     will survive on the host after the FINN Docker container exits."""
     try:
-        tmpdir = tempfile.mkdtemp(prefix=prefix)
-        newdir = tmpdir.replace("/tmp", os.environ["FINN_BUILD_DIR"])
-        os.makedirs(newdir)
-        return newdir
+        build_dir = os.environ["FINN_BUILD_DIR"]
     except KeyError:
         raise Exception(
             """Environment variable FINN_BUILD_DIR must be set
-        correctly. Please ensure you have launched the Docker contaier correctly.
+        correctly. Please ensure you have launched the Docker container correctly.
         """
         )
+    os.makedirs(build_dir, exist_ok=True)
+    new_dir = tempfile.mkdtemp(prefix=prefix, dir=build_dir)
+    os.chmod(new_dir, 0o755)
+    return new_dir
+
+
+def robust_rmtree(path, retries=6, initial_delay=0.1, backoff=2.0):
+    """Remove a directory tree with retries for transient NFS cleanup races.
+    Retries ``ENOTEMPTY``/``EBUSY``. Other errors propagate immediately.
+    """
+    if not path or not os.path.exists(path):
+        return
+    delay = initial_delay
+    for attempt in range(retries):
+        try:
+            shutil.rmtree(path)
+            return
+        except FileNotFoundError:
+            return
+        except OSError as exc:
+            transient = exc.errno in (errno.ENOTEMPTY, errno.EBUSY)
+            if not transient or attempt == retries - 1:
+                raise
+            time.sleep(delay)
+            delay *= backoff
 
 
 class CppBuilder:

tests/fpgadataflow/test_fifosizing.py

@@ -30,7 +30,6 @@
 import pytest
 
 import json
-import shutil
 import torch
 from brevitas.export import export_qonnx
 from onnx import TensorProto, helper
@@ -46,7 +45,7 @@
 import finn.builder.build_dataflow_config as build_cfg
 from finn.transformation.fpgadataflow.set_fifo_depths import InsertAndSetFIFODepths
 from finn.transformation.fpgadataflow.specialize_layers import SpecializeLayers
-from finn.util.basic import make_build_dir
+from finn.util.basic import make_build_dir, robust_rmtree
 from finn.util.test import get_trained_network_and_ishape
 
 
@@ -111,8 +110,8 @@ def test_fifosizing_linear(method, topology):
             node1_inst = getCustomOp(node1)
             assert node0_inst.get_nodeattr("depth") == node1_inst.get_nodeattr("depth")
 
-    shutil.rmtree(tmp_output_dir)
-    shutil.rmtree(tmp_output_dir_cmp)
+    robust_rmtree(tmp_output_dir)
+    robust_rmtree(tmp_output_dir_cmp)
 
 
 @pytest.mark.slow

tests/fpgadataflow/test_split_large_fifos.py

@@ -30,7 +30,6 @@
 import pytest
 
 import json
-import shutil
 import torch
 from brevitas.export import export_qonnx
 from qonnx.core.modelwrapper import ModelWrapper
@@ -39,7 +38,7 @@
 import finn.builder.build_dataflow as build
 import finn.builder.build_dataflow_config as build_cfg
 from finn.transformation.fpgadataflow.set_fifo_depths import get_fifo_split_configs
-from finn.util.basic import make_build_dir
+from finn.util.basic import make_build_dir, robust_rmtree
 from finn.util.test import get_trained_network_and_ishape
 
 
@@ -103,7 +102,7 @@ def test_split_large_fifos(depth):
         assert fifo_depth == golden_cfg[i % len(golden_cfg)][0]
         assert fifo_depth > 1
 
-    shutil.rmtree(tmp_output_dir)
+    robust_rmtree(tmp_output_dir)
 
 
 def test_split_large_fifo_configs():

tests/util/test_robust_rmtree.py

@@ -0,0 +1,112 @@
+############################################################################
+# Copyright (C) 2026, Advanced Micro Devices, Inc.
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+#
+############################################################################
+
+import pytest
+
+import errno
+import os
+import shutil
+import time
+from pathlib import Path
+
+from finn.util.basic import make_build_dir, robust_rmtree
+
+
+@pytest.mark.util
+def test_robust_rmtree_succeeds_first_attempt():
+    target = make_build_dir("test_rmtree_")
+    sub = os.path.join(target, "sub")
+    os.makedirs(sub)
+    Path(os.path.join(sub, "f")).write_text("x")
+    robust_rmtree(target)
+    assert not os.path.exists(target)
+
+
+@pytest.mark.util
+def test_robust_rmtree_missing_path_is_noop():
+    target = make_build_dir("test_rmtree_")
+    robust_rmtree(os.path.join(target, "does_not_exist"))
+    robust_rmtree("")
+    robust_rmtree(None)
+    robust_rmtree(target)
+
+
+@pytest.mark.util
+@pytest.mark.parametrize("transient_errno", [errno.ENOTEMPTY, errno.EBUSY])
+def test_robust_rmtree_retries_on_transient_then_succeeds(monkeypatch, transient_errno):
+    target = make_build_dir("test_rmtree_")
+    Path(os.path.join(target, "f")).write_text("x")
+
+    state = {"calls": 0}
+    real_rmtree = shutil.rmtree
+
+    def flaky(path, *a, **kw):
+        state["calls"] += 1
+        if state["calls"] < 3:
+            raise OSError(transient_errno, "fake", str(path))
+        return real_rmtree(path, *a, **kw)
+
+    monkeypatch.setattr(shutil, "rmtree", flaky)
+    monkeypatch.setattr(time, "sleep", lambda _s: None)
+
+    robust_rmtree(target, retries=5)
+
+    assert state["calls"] == 3
+    assert not os.path.exists(target)
+
+
+@pytest.mark.util
+def test_robust_rmtree_propagates_non_transient_oserror(monkeypatch):
+    target = make_build_dir("test_rmtree_")
+
+    calls = []
+
+    def always_eacces(path, *a, **kw):
+        calls.append(path)
+        raise OSError(errno.EACCES, "fake", str(path))
+
+    monkeypatch.setattr(shutil, "rmtree", always_eacces)
+    monkeypatch.setattr(time, "sleep", lambda _s: None)
+
+    with pytest.raises(OSError) as excinfo:
+        robust_rmtree(target, retries=5)
+
+    assert excinfo.value.errno == errno.EACCES
+    # No retry: an errno outside (ENOTEMPTY, EBUSY) propagates on the first attempt.
+    assert len(calls) == 1
+
+
+@pytest.mark.util
+def test_robust_rmtree_raises_after_retries_exhausted(monkeypatch):
+    target = make_build_dir("test_rmtree_")
+
+    calls = []
+
+    def always_enotempty(path, *a, **kw):
+        calls.append(path)
+        raise OSError(errno.ENOTEMPTY, "fake", str(path))
+
+    monkeypatch.setattr(shutil, "rmtree", always_enotempty)
+    monkeypatch.setattr(time, "sleep", lambda _s: None)
+
+    with pytest.raises(OSError) as excinfo:
+        robust_rmtree(target, retries=4)
+
+    assert excinfo.value.errno == errno.ENOTEMPTY
+    assert len(calls) == 4
+
+
+@pytest.mark.util
+def test_robust_rmtree_tolerates_filenotfounderror(monkeypatch):
+    target = make_build_dir("test_rmtree_")
+
+    def fnf(path, *a, **kw):
+        raise FileNotFoundError(str(path))
+
+    monkeypatch.setattr(shutil, "rmtree", fnf)
+    robust_rmtree(target)