FEAT - Pruning skrub-data folder to remove old data ops execution reports

CHANGES.rst

@@ -29,6 +29,8 @@ New features
 
 Changes
 -------
+- The :meth: `DataOp.skb.full_report` method now deletes reports created with
+  ``output_dir=None`` after 7 days. :pr:`1657` by :user: `Simon Dierickx <simon.dierickx>`.
 - The :func: `tabular_pipeline` uses a :class:`SquashingScaler` instead of a
   :class:`StandardScaler` for centering and scaling numerical features
   when linear models are used.

skrub/_data_ops/_inspection.py

@@ -80,11 +80,9 @@ def node_report(data_op, mode="preview", environment=None, **report_kwargs):
 def _get_output_dir(output_dir, overwrite):
     now = datetime.datetime.now().strftime("%Y-%m-%dT%H%M%S")
     if output_dir is None:
-        output_dir = (
-            datasets.get_data_dir()
-            / "execution_reports"
-            / f"full_data_op_report_{now}_{random_string()}"
-        )
+        output_base_dir = datasets.get_data_dir() / "execution_reports"
+        output_dir = output_base_dir / f"full_data_op_report_{now}_{random_string()}"
+        _utils.prune_folder(output_base_dir)
     else:
         output_dir = Path(output_dir).expanduser().resolve()
         if output_dir.exists():

skrub/_data_ops/_skrub_namespace.py

@@ -1472,7 +1472,15 @@ def full_report(
         This creates a report showing the computation graph, and for each
         intermediate computation, some information (the line of code where it
         was defined, the time it took to run, and more) and a display of the
-        intermediate result (or error).
+        intermediate result (or error). By default, the report is stored in
+        a timestamped subdirectory of the skrub data folder.
+
+        .. note::
+            When this function is invoked reports starting with ``full_data_op_report_``
+            that are stored in the skrub data folder are automatically deleted after 7 days.
+            This is to avoid accumulating too many reports over time. If you want to keep
+            specific reports, please specify an output directory.
+
 
         Parameters
         ----------
@@ -1486,7 +1494,9 @@ def full_report(
 
         output_dir : str or Path or None (default=None)
             Directory where to store the report. If ``None``, a timestamped
-            subdirectory will be created in the skrub data directory.
+            subdirectory will be created in the skrub data directory. Note
+            that the reports created with ``output_dir=None`` are automatically
+            deleted after 7 days.
 
         overwrite : bool (default=False)
             What to do if the output directory already exists. If

skrub/_data_ops/_utils.py

@@ -1,5 +1,9 @@
 import enum
+import os
+import shutil
+import time
 import traceback
+import warnings
 
 from joblib.externals import cloudpickle
 
@@ -62,3 +66,25 @@ def format_exception(e):
 def format_exception_only(e):
     """compatibility for python < 3.10"""
     return traceback.format_exception_only(type(e), e)
+
+
+def prune_folder(path: str):
+    if not os.path.exists(path):
+        return
+    time_threshold = time.time() - 7 * 24 * 3600  # 7 days ago
+
+    folders = (
+        (f, os.path.getmtime(os.path.join(path, f)))
+        for f in os.listdir(path)
+        if os.path.isdir(os.path.join(path, f)) and f.startswith("full_data_op_report_")
+    )
+    for dir_path, dir_time in folders:
+        try:
+            if dir_time < time_threshold:
+                shutil.rmtree(os.path.join(path, dir_path))
+        except Exception as e:
+            warnings.warn(
+                "Skrub wants to delete an old folder in the skrub data folder: "
+                f"Could not delete {dir_path}:\n"
+                + "".join(format_exception_only(e))
+            )

skrub/_data_ops/tests/test_utils.py

@@ -0,0 +1,61 @@
+import os
+import tempfile
+import time
+
+import pytest
+
+from skrub._data_ops import _utils
+
+
+def test_prune_folder_with_standard_name_dirs():
+    eight_days_ago = time.time() - 8 * 24 * 3600
+    with tempfile.TemporaryDirectory() as tmpdir:
+        for i in range(3):
+            dirname = os.path.join(tmpdir, f"full_data_op_report_{i}")
+            os.mkdir(dirname)
+            if i < 2:
+                os.utime(dirname, (eight_days_ago, eight_days_ago))
+
+        assert len(os.listdir(tmpdir)) == 3
+
+        _utils.prune_folder(tmpdir)
+
+        remaining_items = os.listdir(tmpdir)
+        assert len(remaining_items) == 1
+
+
+def test_prune_folder_with_nonstandard_name_dirs():
+    eight_days_ago = time.time() - 8 * 24 * 3600
+    with tempfile.TemporaryDirectory() as tmpdir:
+        dirname = os.path.join(tmpdir, "other_report")
+        os.mkdir(dirname)
+        os.utime(dirname, (eight_days_ago, eight_days_ago))
+
+        assert len(os.listdir(tmpdir)) == 1
+
+        _utils.prune_folder(tmpdir)
+
+        remaining_items = os.listdir(tmpdir)
+        assert len(remaining_items) == 1
+        assert remaining_items[0] == "other_report"
+
+
+def test_prune_folder_catch_exception(monkeypatch):
+    eight_days_ago = time.time() - 8 * 24 * 3600
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        dirname = os.path.join(tmpdir, "full_data_op_report_test")
+        os.mkdir(dirname)
+        os.utime(dirname, (eight_days_ago, eight_days_ago))
+
+        def mock_rmtree(path, *args, **kwargs):
+            raise OSError("Cannot delete folder")
+
+        monkeypatch.setattr("shutil.rmtree", mock_rmtree)
+
+        assert len(os.listdir(tmpdir)) == 1
+
+        with pytest.warns(UserWarning, match="Could not delete"):
+            _utils.prune_folder(tmpdir)
+
+        monkeypatch.undo()