migrate irods pyfilesystem2 filesource to fsspec backed filesource

lib/galaxy/dependencies/__init__.py

@@ -294,6 +294,9 @@ def check_rspace_client(self):
     def check_fs_irods(self):
         return "irods" in self.file_sources
 
+    def check_mangofs(self):
+        return self.check_fs_irods()
+
     def check_watchdog(self):
         install_set = {"auto", "True", "true", "polling", True}
         return self.config["watch_tools"] in install_set or self.config["watch_tool_data_dir"] in install_set

lib/galaxy/dependencies/conditional-requirements.txt

@@ -35,7 +35,7 @@ adlfs
 huggingface_hub
 omero-py #type: omero
 iiif-fsspec # type: iiif
-fs-irods # type: irods, depends on python-irodsclient
+mangofs @ git+https://gitlab.kuleuven.be/setit/rdm/mango-fsspec.git@1.1.0 # type: irods, depends on python-irodsclient
 mavedb-fsspec>=0.1.0 # type: mavedb
 
 # Vault backend

lib/galaxy/files/sources/irods.py

@@ -1,31 +1,26 @@
 import os
-from fnmatch import fnmatch
+import posixpath
 
-import fs
-import fs.errors
-
-from galaxy.exceptions import (
-    AuthenticationRequired,
-    MessageException,
-)
 from galaxy.files.models import (
-    AnyRemoteEntry,
-    BaseFileSourceConfiguration,
-    BaseFileSourceTemplateConfiguration,
     FilesSourceRuntimeContext,
 )
+from galaxy.files.sources._fsspec import (
+    CacheOptionsDictType,
+    FsspecBaseFileSourceConfiguration,
+    FsspecBaseFileSourceTemplateConfiguration,
+    FsspecFilesSource,
+)
 from galaxy.util.config_templates import TemplateExpansion
-from ._pyfilesystem2 import PyFilesystem2FilesSource
 
 try:
-    from fs_irods import iRODSFS
     from irods.session import iRODSSession
+    from mangofs import IRODSFileSystem
 except ImportError:
-    iRODSFS = None
     iRODSSession = None
+    IRODSFileSystem = None
 
 
-class IrodsFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration):
+class IrodsFsspecFileSourceTemplateConfiguration(FsspecBaseFileSourceTemplateConfiguration):
     host: str | TemplateExpansion
     port: int | TemplateExpansion = 1247
     username: str | TemplateExpansion
@@ -45,7 +40,7 @@ class IrodsFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration):
     resource: str | TemplateExpansion | None = None
 
 
-class IrodsFileSourceConfiguration(BaseFileSourceConfiguration):
+class IrodsFsspecFileSourceConfiguration(FsspecBaseFileSourceConfiguration):
     host: str
     port: int = 1247
     username: str
@@ -65,70 +60,28 @@ class IrodsFileSourceConfiguration(BaseFileSourceConfiguration):
     resource: str | None = None
 
 
-class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfiguration, IrodsFileSourceConfiguration]):
+class IrodsFsspecFilesSource(
+    FsspecFilesSource[IrodsFsspecFileSourceTemplateConfiguration, IrodsFsspecFileSourceConfiguration]
+):
     plugin_type = "irods"
-    required_module = iRODSFS
-    required_package = "fs-irods"
-
-    template_config_class = IrodsFileSourceTemplateConfiguration
-    resolved_config_class = IrodsFileSourceConfiguration
-
-    def _iter_directory_entries(self, fs_handle, parent_path: str, normalized_query: str | None = None):
-        for raw_name in fs_handle.listdir(parent_path):
-            name = os.path.basename(str(raw_name).rstrip("/"))
-            if not name:
-                continue
-            if normalized_query and not fnmatch(name.lower(), f"*{normalized_query}*"):
-                continue
-            entry_path = fs.path.join(parent_path, name)
-            info = fs_handle.getinfo(entry_path, namespaces=["details"])
-            yield entry_path, info
-
-    def _list_non_recursive(
-        self,
-        fs_handle,
-        path: str,
-        limit: int | None = None,
-        offset: int | None = None,
-        query: str | None = None,
-    ) -> tuple[list[AnyRemoteEntry], int]:
-        normalized_query = query.lower() if query else None
-        entries = []
-        for _, info in self._iter_directory_entries(fs_handle, path, normalized_query):
-            entries.append(self._resource_info_to_dict(path, info))
-        count = len(entries)
-        if (page := self._to_page(limit, offset)) is not None:
-            entries = entries[page[0] : page[1]]
-        return entries, count
-
-    def _list(
+    required_module = IRODSFileSystem
+    required_package = "mangofs"
+
+    template_config_class = IrodsFsspecFileSourceTemplateConfiguration
+    resolved_config_class = IrodsFsspecFileSourceConfiguration
+
+    def _open_fs(
         self,
-        context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration],
-        path="/",
-        recursive=False,
-        write_intent: bool = False,
-        limit: int | None = None,
-        offset: int | None = None,
-        query: str | None = None,
-        sort_by: str | None = None,
-    ) -> tuple[list[AnyRemoteEntry], int]:
-        try:
-            with self._open_fs(context) as fs_handle:
-                if recursive:
-                    raise MessageException("Recursive listing is not supported for iRODS file sources.")
-                return self._list_non_recursive(fs_handle, path, limit, offset, query)
-        except fs.errors.PermissionDenied as e:
-            raise AuthenticationRequired(
-                f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}."
-            ) from e
-        except fs.errors.FSError as e:
-            raise MessageException(f"Problem listing file source path {path}. Reason: {e}") from e
-
-    def _open_fs(self, context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration]):
-        if iRODSFS is None or iRODSSession is None:
+        context: FilesSourceRuntimeContext[IrodsFsspecFileSourceConfiguration],
+        cache_options: CacheOptionsDictType,
+    ):
+        if IRODSFileSystem is None or iRODSSession is None:
             raise self.required_package_exception
 
-        config = context.config
+        session = self._open_session(context.config)
+        return IRODSFileSystem(session=session, **cache_options)
+
+    def _open_session(self, config: IrodsFsspecFileSourceConfiguration):
         session_kwargs = {
             "host": config.host,
             "port": config.port,
@@ -152,8 +105,33 @@ def _open_fs(self, context: FilesSourceRuntimeContext[IrodsFileSourceConfigurati
         session.connection_timeout = config.timeout
         if config.resource:
             session.default_resource = config.resource
-
-        return iRODSFS(session=session, root=config.root)
-
-
-__all__ = ("IrodsFilesSource",)
+        return session
+
+    def _to_filesystem_path(self, path: str, config: IrodsFsspecFileSourceConfiguration) -> str:
+        root = self._normalized_root(config)
+        path = path or "/"
+        if path.startswith("irods://"):
+            return path
+        if path == "/":
+            return root
+        return posixpath.join(root, path.lstrip("/"))
+
+    def _adapt_entry_path(self, filesystem_path: str, config: IrodsFsspecFileSourceConfiguration) -> str:
+        root = self._normalized_root(config)
+        filesystem_path = filesystem_path or "/"
+        if filesystem_path == root:
+            return "/"
+        root_prefix = root.rstrip("/") + "/"
+        if filesystem_path.startswith(root_prefix):
+            return "/" + filesystem_path[len(root_prefix) :]
+        return filesystem_path if filesystem_path.startswith("/") else f"/{filesystem_path}"
+
+    @staticmethod
+    def _normalized_root(config: IrodsFsspecFileSourceConfiguration) -> str:
+        if not config.root:
+            return "/"
+        root = os.path.normpath(config.root)
+        return root if root.startswith("/") else f"/{root}"
+
+
+__all__ = ("IrodsFsspecFilesSource",)

test/unit/app/dependencies/test_deps.py

@@ -97,6 +97,7 @@ def test_fs_configured():
         assert cds.check_dropboxdrivefs()
         assert cds.check_webdav4()
         assert cds.check_fs_irods()
+        assert cds.check_mangofs()
 
 
 def test_yaml_jobconf_runners():