feat(rest): limit workspace file listing and add forced GC (#657)

Introduce WORKSPACE_DISPLAY_FILE_LIMIT env var to cap the
number of files returned by workspace listing endpoints,
raising BadRequest when the limit is exceeded. Add
FORCE_GARBAGE_COLLECTION env var to allow forcing manual
gc.collect() before selected workspace operations (ls, list,
rm, delete).

Closes #644

Author: Michael Buchar

docs/openapi.json

@@ -1658,7 +1658,7 @@
             "type": "integer"
           },
           {
-            "description": "Filter workflow workspace files.",
+            "description": "Filter workflow workspace files by file name, size, or modification date.",
             "in": "query",
             "name": "search",
             "required": false,
@@ -1706,7 +1706,7 @@
             }
           },
           "400": {
-            "description": "Request failed. The incoming data specification seems malformed."
+            "description": "Request failed. The request parameters are invalid or the filtered result set exceeds the configured display limit."
           },
           "404": {
             "description": "Request failed. Workflow does not exist.",

reana_workflow_controller/config.py

@@ -18,6 +18,7 @@
 )
 from reana_db.models import JobStatus, RunStatus
 from distutils.util import strtobool
+from typing import List
 
 from reana_workflow_controller.version import __version__
 
@@ -465,3 +466,39 @@ def _parse_interactive_sessions_environments(env_var):
 
 MAX_WORKFLOW_SHARING_MESSAGE_LENGTH = 5000
 """Maximum length of the user-provided message when sharing a workflow."""
+
+
+def _parse_comma_separated_list(value: str) -> List[str]:
+    """Parse comma-separated env var values into a list of strings."""
+    if not value:
+        return []
+    return [x.strip() for x in value.split(",") if x.strip()]
+
+
+WORKSPACE_DISPLAY_FILE_LIMIT = int(os.getenv("WORKSPACE_DISPLAY_FILE_LIMIT", "100000"))
+"""Maximum number of file entries returned by workspace listing endpoints."""
+
+_VALID_GC_COMMANDS = {"ls", "list", "rm", "delete"}
+"""Valid FORCE_GARBAGE_COLLECTION command values."""
+
+_gc_env = os.getenv("FORCE_GARBAGE_COLLECTION", "")
+FORCE_GARBAGE_COLLECTION = _parse_comma_separated_list(_gc_env)
+"""Comma-separated list of commands that trigger a manual `gc.collect()` before operations.
+
+Example:
+  $ export FORCE_GARBAGE_COLLECTION=ls,list,rm,delete
+
+Supported values:
+- ls: trigger `gc.collect()` before listing workspace files
+- list: trigger `gc.collect()` before listing all workflows
+- rm: trigger `gc.collect()` before removing workspace files
+- delete: trigger `gc.collect()` before deleting workflows
+"""
+_invalid_gc = sorted(set(FORCE_GARBAGE_COLLECTION) - _VALID_GC_COMMANDS)
+if _invalid_gc:
+    valid_gc_values = ", ".join(sorted(_VALID_GC_COMMANDS))
+    invalid_gc_values = ", ".join(_invalid_gc)
+    raise ValueError(
+        "Invalid FORCE_GARBAGE_COLLECTION values: "
+        f"{invalid_gc_values}. Valid values: {valid_gc_values}"
+    )

reana_workflow_controller/rest/utils.py

@@ -9,7 +9,7 @@
 """REANA Workflow Controller workflows REST API."""
 
 import difflib
-import fs
+import gc
 import json
 import logging
 import mimetypes
@@ -63,6 +63,8 @@
     PROGRESS_STATUSES,
     REANA_GITLAB_HOST,
     PREVIEWABLE_MIME_TYPE_PREFIXES,
+    FORCE_GARBAGE_COLLECTION,
+    WORKSPACE_DISPLAY_FILE_LIMIT,
 )
 from reana_workflow_controller.consumer import _update_workflow_status
 from reana_workflow_controller.errors import (
@@ -73,6 +75,12 @@
 )
 from reana_workflow_controller.workflow_run_manager import KubernetesWorkflowRunManager
 
+WORKSPACE_FILE_LISTING_FILTER_HINT = (
+    "Please use more specific filters to narrow the results. "
+    "Available filters: file name, size, or last-modified."
+)
+"""Neutral filter hint shown when workspace listings exceed the display limit."""
+
 
 def start_workflow(workflow, parameters):
     """Start a workflow."""
@@ -222,6 +230,9 @@ def remove_workflow_jobs_from_cache(workflow):
 
 def delete_workflow(workflow, all_runs=False, workspace=False):
     """Delete workflow."""
+    if "delete" in FORCE_GARBAGE_COLLECTION:
+        gc.collect()
+
     if workflow.status in [
         RunStatus.created,
         RunStatus.finished,
@@ -434,8 +445,16 @@ def list_directory_files(
     workspace_path: str, search: Dict[str, List[str]] = None
 ) -> List[dict]:
     """Return a list of files inside a given workspace."""
+    if "ls" in FORCE_GARBAGE_COLLECTION:
+        gc.collect()
     file_list = []
     for file_name in workspace.walk(workspace_path, include_dirs=False):
+        if len(file_list) >= WORKSPACE_DISPLAY_FILE_LIMIT:
+            raise BadRequest(
+                "Too many files to display "
+                f"(limit={WORKSPACE_DISPLAY_FILE_LIMIT}). "
+                f"{WORKSPACE_FILE_LISTING_FILTER_HINT}"
+            )
         st = workspace.lstat(workspace_path, file_name)
         file_info = {
             "name": file_name,
@@ -465,10 +484,12 @@ def remove_files_recursive_wildcard(workspace_path, path_or_pattern):
     :param workspace_path: Directory to delete files from.
     :param path_or_pattern: Wildcard pattern to use for the removal.
     :return: Dictionary with the results:
-       - dictionary with names of succesfully deleted files and their sizes
+       - dictionary with names of successfully deleted files and their sizes
        - dictionary with names of failed deletions and corresponding
        error messages.
     """
+    if "rm" in FORCE_GARBAGE_COLLECTION:
+        gc.collect()
     deleted = {"deleted": {}, "failed": {}}
     for file_name in workspace.glob_or_walk_directory(
         workspace_path, path_or_pattern, topdown=False
@@ -488,12 +509,20 @@ def list_files_recursive_wildcard(workspace_path, path_or_pattern, search=None):
     :param workspace_path: Directory to list files from.
     :param path_or_pattern: Wildcard pattern to use for the listing.
     :return: Dictionary with the results:
-       - dictionary with names of succesfully listed files and their sizes
+       - dictionary with names of successfully listed files and their sizes
        - dictionary with names of failed listing and corresponding
        error messages.
     """
+    if "ls" in FORCE_GARBAGE_COLLECTION:
+        gc.collect()
     list_files_recursive = []
     for path in workspace.glob_or_walk_directory(workspace_path, path_or_pattern):
+        if len(list_files_recursive) >= WORKSPACE_DISPLAY_FILE_LIMIT:
+            raise BadRequest(
+                "Too many files to display "
+                f"(limit={WORKSPACE_DISPLAY_FILE_LIMIT}). "
+                f"{WORKSPACE_FILE_LISTING_FILTER_HINT}"
+            )
         st = workspace.lstat(workspace_path, path)
         raw_size = st.st_size
         mtime = st.st_mtime

reana_workflow_controller/rest/workflows.py

@@ -11,7 +11,7 @@
 import datetime
 import json
 import logging
-import re
+import gc
 from typing import Optional
 from uuid import uuid4
 
@@ -44,6 +44,7 @@
     REANA_URL,
     DEFAULT_NAME_FOR_WORKFLOWS,
     MAX_WORKFLOW_SHARING_MESSAGE_LENGTH,
+    FORCE_GARBAGE_COLLECTION,
 )
 from reana_workflow_controller.errors import (
     REANAWorkflowControllerError,
@@ -308,6 +309,9 @@ def get_workflows(args, paginate=None):  # noqa
     shared_by: Optional[str] = args.get("shared_by")
     shared_with: Optional[str] = args.get("shared_with")
 
+    if "list" in FORCE_GARBAGE_COLLECTION:
+        gc.collect()
+
     if shared_by and shared_with:
         message = "You cannot filter by shared_by and shared_with at the same time."
         return (jsonify({"message": message}), 400)

reana_workflow_controller/rest/workflows_workspace.py

@@ -20,7 +20,7 @@
 )
 from fs.errors import CreateFailed
 from werkzeug.datastructures import FileStorage
-from werkzeug.exceptions import NotFound
+from werkzeug.exceptions import BadRequest, NotFound
 
 from reana_commons import workspace
 from reana_commons.errors import REANAWorkspaceError
@@ -417,7 +417,7 @@ def get_files(workflow_id_or_name, paginate=None):  # noqa
           type: integer
         - name: search
           in: query
-          description: Filter workflow workspace files.
+          description: Filter workflow workspace files by file name, size, or modification date.
           required: false
           type: string
       responses:
@@ -448,7 +448,8 @@ def get_files(workflow_id_or_name, paginate=None):  # noqa
                           type: string
         400:
           description: >-
-            Request failed. The incoming data specification seems malformed.
+            Request failed. The request parameters are invalid or the filtered
+            result set exceeds the configured display limit.
         404:
           description: >-
             Request failed. Workflow does not exist.
@@ -487,6 +488,8 @@ def get_files(workflow_id_or_name, paginate=None):  # noqa
         pagination_dict = paginate(file_list)
         return jsonify(pagination_dict), 200
 
+    except json.JSONDecodeError:
+        return jsonify({"message": "Malformed request."}), 400
     except ValueError:
         return (
             jsonify(
@@ -501,6 +504,8 @@ def get_files(workflow_id_or_name, paginate=None):  # noqa
         )
     except KeyError:
         return jsonify({"message": "Malformed request."}), 400
+    except BadRequest as e:
+        return jsonify({"message": e.description}), e.code
     except REANAWorkspaceError as e:
         return jsonify({"message": str(e)}), 400
     except FileNotFoundError:

tests/test_config.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of REANA.
+# Copyright (C) 2025, 2026 CERN.
+#
+# REANA is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+import importlib
+
+import pytest
+
+import reana_workflow_controller.config as config
+
+
+def _reload_config():
+    """Reload configuration module after environment changes."""
+    return importlib.reload(config)
+
+
+def test_parse_comma_separated_list():
+    assert config._parse_comma_separated_list("") == []
+    assert config._parse_comma_separated_list("ls") == ["ls"]
+    assert config._parse_comma_separated_list("ls,list") == ["ls", "list"]
+    assert config._parse_comma_separated_list(" ls, list ,rm,, ") == [
+        "ls",
+        "list",
+        "rm",
+    ]
+
+    parsed = config._parse_comma_separated_list("ls,list")
+    assert "ls" in parsed
+    assert "list" in parsed
+    assert "l" not in parsed  # ensures no more substring matching
+
+
+def test_force_garbage_collection_rejects_invalid_values(monkeypatch):
+    """Test FORCE_GARBAGE_COLLECTION rejects unsupported command values."""
+    with monkeypatch.context() as m:
+        m.setenv("FORCE_GARBAGE_COLLECTION", "lis,delet")
+        with pytest.raises(ValueError) as exc_info:
+            _reload_config()
+        message = str(exc_info.value)
+        assert "Invalid FORCE_GARBAGE_COLLECTION values:" in message
+        assert "delet" in message
+        assert "lis" in message
+        assert "Valid values: delete, list, ls, rm" in message
+
+    _reload_config()
+
+
+def test_force_garbage_collection_accepts_valid_values(monkeypatch):
+    """Test FORCE_GARBAGE_COLLECTION accepts supported command values."""
+    with monkeypatch.context() as m:
+        m.setenv("FORCE_GARBAGE_COLLECTION", "ls,list,rm,delete")
+        reloaded_config = _reload_config()
+        assert reloaded_config.FORCE_GARBAGE_COLLECTION == [
+            "ls",
+            "list",
+            "rm",
+            "delete",
+        ]
+
+    _reload_config()