App locking hardening

README.md

@@ -45,11 +45,15 @@ The CS3 Contents Manager consists of several key components:
 
 ### JupyterLab Extension
 
-The bundled labextension (`@cs3org/cs3-jupyter`) provides three plugins:
+The bundled labextension (`@cs3org/cs3-jupyter-client`) provides four plugins:
 
 - **Spaces** - sidebar panel listing CERNBox Spaces (projects) the user has access to
 - **Shares** - sidebar panel showing incoming and outgoing CERNBox shared folders
 - **Storage Quota** - progress bar at the bottom of the file browser showing storage usage
+- **Locking** - notifies the server when documents are opened and closed, so the
+  CS3 lock is held while a document is open and released when the last session
+  closes it. Without the labextension installed, locks are only taken on save
+  and released when they expire (`lock_expiration`).
 
 ## Installation
 
@@ -95,19 +99,48 @@ Add the following to your `jupyter_server_config.py`:
 from cs3_jupyter.cs3largefilemanager import CS3LargeFileManager
 
 c.ServerApp.contents_manager_class = CS3LargeFileManager
-c.CS3FileManagerMixin.host = '<host>'
-c.CS3FileManagerMixin.tus_enabled = False
-c.CS3FileManagerMixin.ssl_enabled = False
-c.CS3FileManagerMixin.token_path = '/path/to/oauth.token'
-c.CS3FileManagerMixin.auth_login_type = 'bearer'
-c.CS3FileManagerMixin.authtokenvalidity = 3600
-c.CS3FileManagerMixin.lock_not_impl = False
-c.CS3FileManagerMixin.lock_as_attr = False
-c.CS3FileManagerMixin.root_path = '/eos/user/r/rwelande'
-c.CS3FileManagerMixin.client_id = 'rwelande'
+c.CS3Mixin.host = '<host>'
+# Keep TUS disabled while locking is enabled: cs3-python-client sends a
+# misspelled X-Lock_Holder header on the TUS branch (cs3client/file.py), so
+# locked writes would be rejected by EOS holder matching.
+c.CS3Mixin.tus_enabled = False
+c.CS3Mixin.ssl_enabled = False
+c.CS3Mixin.token_path = '/path/to/oauth.token'
+c.CS3Mixin.auth_login_type = 'bearer'
+c.CS3Mixin.authtokenvalidity = 3600
+c.CS3Mixin.lock_not_impl = False
+c.CS3Mixin.lock_by_setting_attr = False
+c.CS3Mixin.root_path = '/eos/user/r/rwelande'
+c.CS3Mixin.client_id = 'rwelande'
 c.CS3LargeFileManager.max_copy_folder_size_mb = 500
 ```
 
+Configure the traits on `CS3Mixin`: it is the only class in the MRO of both
+contents managers, so the same section applies to `CS3LargeFileManager` and
+`CS3HybridLargeFileManager`. (`c.CS3FileManagerMixin.*` is silently ignored by
+the hybrid manager, which does not inherit from it.)
+
+### Locking
+
+Files being edited are locked in the storage through the CS3 APIs, so other
+applications (sync clients, web office, ...) cannot write to them concurrently.
+
+- `lock_app_name` (default `jupyter-rtc`): the CS3 lock holder ("app name").
+  EOS enforces write locks by app name, and lowercases it - keep it lowercase.
+- `lock_holder_suffix_client_id` (default `True`): appends `-<client_id>` to the
+  holder, making locks per-user (`jupyter-rtc-rwelande`). Set it to `False` on
+  all servers to share one holder (`jupyter-rtc`) so multiple users can
+  collaborate on the same locked files.
+- `lock_value` (default `jupyter_rtc_lock`): the shared lock id, required by
+  reva to refresh or release a lock.
+- `lock_expiration` (default `300` seconds): locks not refreshed within this
+  window expire in the storage; it is also the session-tracker heartbeat timeout.
+
+The `/lock` API tracks how many sessions have a document open (`POST
+/lock?path=...&session_id=...` on open and as heartbeat, `DELETE` on close) and
+releases the reva lock when the last session leaves. A background task refreshes
+locks for tracked sessions and unlocks documents whose sessions went stale.
+
 ### Authentication
 
 The CS3 Contents Manager supports OAuth token-based authentication. Set up your authentication:

cs3_jupyter/cs3mixin.py

@@ -1,24 +1,29 @@
 from __future__ import annotations
 
+import asyncio
 import jwt
 import os
 from configparser import ConfigParser
 from typing import Any
 
+from anyio.to_thread import run_sync
 from cs3client.auth import Auth
 from cs3client.cs3client import CS3Client
 from traitlets import Bool, Int, Unicode
 from traitlets.config.configurable import LoggingConfigurable
 
-from .cs3vfs.statuscodehandler import StatusCodeHandler
+
+from .cs3vfs.statuscodehandler import StatusCodeHandler, FileLockedError
 from .cs3vfs.cs3versions import CS3FileVersions
-from .cs3vfs.cs3groups import CS3Groups
+from .cs3vfs.cs3vfs import CS3VirtualFileSystem
 from .cs3vfs.cs3sharing import CS3Sharing
 from .cs3vfs.cs3spaces import CS3Spaces
+from .cs3vfs.cs3groups import CS3Groups
 from .cs3vfs.cs3users import CS3Users
-from .cs3vfs.cs3vfs import CS3VirtualFileSystem
+from .cs3vfs.cs3lock import CS3Lock
+from .sessiontracker import SessionTracker
 
-class CS3BaseMixin(CS3Groups, CS3Users, CS3Spaces, CS3Sharing, CS3FileVersions, LoggingConfigurable):
+class CS3Mixin(CS3VirtualFileSystem, CS3Groups, CS3Users, CS3Spaces, CS3Sharing, CS3FileVersions, CS3Lock, LoggingConfigurable):
     """Owns the shared CS3Client/Auth and persistent service instances."""
 
     host = Unicode(config=True, help="CS3 host address")
@@ -29,6 +34,27 @@ class CS3BaseMixin(CS3Groups, CS3Users, CS3Spaces, CS3Sharing, CS3FileVersions,
         config=True,
         help="Path to OAuth token file",
     )
+    lock_expiration = Int(
+        default_value=300,
+        config=True,
+        help="Lock expiration time in seconds"
+    )
+    lock_app_name = Unicode(
+        default_value="jupyter-rtc",
+        config=True,
+        help="String to use for application-level locking (EOS lowercases app names, keep it lowercase)"
+    )
+    lock_holder_suffix_client_id = Bool(
+        default_value=True,
+        config=True,
+        help="Append '-<client_id>' to lock_app_name to make the lock holder per-user. "
+             "Set to False on all servers to share one holder, letting them collaborate on locked files."
+    )
+    lock_value = Unicode(
+        default_value="jupyter_rtc_lock",
+        config=True,
+        help="Value to use for application-level locking"
+    )
     root_path = Unicode(default_value="", config=True, help="CS3 root path for the user")
     auth_login_type = Unicode(
         default_value="bearer", config=True, help="Authentication login type"
@@ -37,23 +63,38 @@ class CS3BaseMixin(CS3Groups, CS3Users, CS3Spaces, CS3Sharing, CS3FileVersions,
         default_value=3600, config=True, help="Authentication token validity in seconds"
     )
     lock_not_impl = Bool(default_value=False, config=True, help="Lock not implemented flag")
-    lock_as_attr = Bool(default_value=False, config=True, help="Lock as attribute flag")
-    cs3_token = Unicode(default_value="", config=True, help="CS3 authentication token")
+    lock_by_setting_attr = Bool(default_value=False, config=True, help="Fall back to advisory xattr locks when the storage does not implement locking")
     client_id = Unicode(default_value="", config=True, help="CS3 client ID (can be set in config)")
 
 
     def __init__(self, **kwargs: Any):
         self.status_handler = StatusCodeHandler()
+        self.cs3_token = ""
+        self._lock_refresher = None
+        self._stat_cache: dict[str, tuple[Any, float]] = {}
+        self._pending_paths: dict[str, float] = {}  # changed via CS3, mount may lag
         super().__init__(**kwargs)
         self._read_token_file()
         self._config = self._create_cs3_config()
         self.client = CS3Client(self._config, "cs3client", self.log)
         self.auth = Auth(self.client)
         self.auth.set_client_id(self.client_id)
         self.auth.set_client_secret(self.cs3_token)
-
+        self.session_tracker = SessionTracker(heartbeat_timeout_seconds=self.lock_expiration)
         self.log.debug(f"CS3ClientMixinBase initialized with path: {self.root_path}")
 
+    @property
+    def lock_holder(self) -> str:
+        """The lock holder (CS3 lock app_name) identifying this server's locks.
+
+        Evaluated lazily so it reflects traitlets config. Reva's EOS driver
+        matches lock holders by app_name alone, so per-user holders serialize
+        users while a shared holder lets all servers co-own locks.
+        """
+        if self.lock_holder_suffix_client_id and self.client_id:
+            return f"{self.lock_app_name}-{self.client_id}"
+        return self.lock_app_name
+
     def get_user_path(self) -> str:
         return self.root_path
 
@@ -83,9 +124,44 @@ def _create_cs3_config(self) -> ConfigParser:
         cs3config.set("cs3client", "auth_login_type", self.auth_login_type)
         cs3config.set("cs3client", "authtokenvalidity", str(self.authtokenvalidity))
         cs3config.set("cs3client", "lock_not_impl", str(self.lock_not_impl).lower())
-        cs3config.set("cs3client", "lock_as_attr", str(self.lock_as_attr).lower())
+        cs3config.set("cs3client", "lock_by_setting_attr", str(self.lock_by_setting_attr).lower())
+        cs3config.set("cs3client", "lock_expiration", str(self.lock_expiration))
         return cs3config
 
+    def ensure_lock_refresher(self) -> None:
+        """Start the background lock refresher if it is not already running."""
+        if self._lock_refresher is None or self._lock_refresher.done():
+            self._lock_refresher = asyncio.ensure_future(self._lock_refresh_loop())
+
+    async def _lock_refresh_loop(self) -> None:
+        """Keep locks of open documents alive and release locks whose sessions all went stale.
+
+        Runs while the session tracker has entries; exits when idle (restarted
+        lazily by the next POST /lock).
+        """
+        interval = max(self.lock_expiration // 3, 10)
+        while True:
+            await asyncio.sleep(interval)
+            async with self._lock_mutex:
+                active, expired = self.session_tracker.sweep()
+                # A storage error must never kill the refresher: it is the only
+                # thing keeping open documents locked and stale ones released.
+                for path in expired:
+                    try:
+                        await run_sync(self.unlock, path, self.lock_holder, self.lock_value)
+                    except FileLockedError:
+                        # Not ours anymore - nothing to release.
+                        pass
+                    except OSError as e:
+                        self.log.warning(f"Could not release lock on {path}: {e}")
+                for path in active:
+                    try:
+                        await run_sync(self.refresh_lock, path, self.lock_holder, self.lock_value, self.lock_value)
+                    except OSError as e:
+                        self.log.warning(f"Lock refresh lost on {path}: {e}")
+            if not active and not expired:
+                return
+
     def _decode_token(self) -> dict:
         _, token = self.auth.get_token()
         return jwt.decode(
@@ -101,7 +177,3 @@ def user_idp(self) -> str:
     @property
     def user_opaque_id(self) -> str:
         return self._decode_token()["user"]["id"]["opaque_id"]
-
-class CS3Mixin(CS3BaseMixin, CS3VirtualFileSystem):
-    """CS3Mixin combines the base mixin with the virtual file system operations."""
-    pass

cs3_jupyter/cs3vfs/cs3file.py

@@ -38,10 +38,10 @@ def _load_content(self) -> None:
         """Load file content from CS3."""
         try:
             if 'b' in self.mode:
-                result = self.cs3_vfs._read_file(self.path, "byte")
+                result = self.cs3_vfs.read_file(self.path, "byte")
                 self._content = result[0]
             else:
-                result = self.cs3_vfs._read_file(self.path, "text")
+                result = self.cs3_vfs.read_file(self.path, "text")
                 self._content = result[0]
         except Exception:
             if 'r' in self.mode:
@@ -96,8 +96,7 @@ def flush(self) -> None:
             else:
                 format = "base64"
                 content = base64.encodebytes(self._content).decode("ascii")
-
-            self.cs3_vfs._save_file(self.path, content, format)
+            self.cs3_vfs.vfs_save_file(self.path, content, format)
             self._modified = False
 
     def close(self) -> None: