migrate irods pyfilesystem2 filesource to fsspec backed filesource

Author: PlushZ

lib/galaxy/dependencies/__init__.py

@@ -291,7 +291,7 @@ def check_fs_basespace(self):
     def check_rspace_client(self):
         return "rspace" in self.file_sources
 
-    def check_fs_irods(self):
+    def check_mangofs(self):
         return "irods" in self.file_sources
 
     def check_watchdog(self):

lib/galaxy/dependencies/conditional-requirements.txt

@@ -35,7 +35,7 @@ adlfs
 huggingface_hub
 omero-py #type: omero
 iiif-fsspec # type: iiif
-fs-irods # type: irods, depends on python-irodsclient
+mangofs @ git+https://gitlab.kuleuven.be/setit/rdm/mango-fsspec.git@1.1.0 # type: irods, depends on python-irodsclient
 mavedb-fsspec>=0.1.0 # type: mavedb
 
 # Vault backend

lib/galaxy/files/sources/irods.py

@@ -1,51 +1,50 @@
 import os
-from fnmatch import fnmatch
-
-import fs
-import fs.errors
-
-from galaxy.exceptions import (
-    AuthenticationRequired,
-    MessageException,
+import posixpath
+from typing import (
+    Optional,
+    Union,
 )
+
 from galaxy.files.models import (
-    AnyRemoteEntry,
-    BaseFileSourceConfiguration,
-    BaseFileSourceTemplateConfiguration,
     FilesSourceRuntimeContext,
 )
+from galaxy.files.sources._fsspec import (
+    CacheOptionsDictType,
+    FsspecBaseFileSourceConfiguration,
+    FsspecBaseFileSourceTemplateConfiguration,
+    FsspecFilesSource,
+)
 from galaxy.util.config_templates import TemplateExpansion
-from ._pyfilesystem2 import PyFilesystem2FilesSource
 
 try:
-    from fs_irods import iRODSFS
     from irods.session import iRODSSession
+    from mangofs import IRODSFileSystem
 except ImportError:
-    iRODSFS = None
     iRODSSession = None
-
-
-class IrodsFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration):
-    host: str | TemplateExpansion
-    port: int | TemplateExpansion = 1247
-    username: str | TemplateExpansion
-    password: str | TemplateExpansion
-    zone: str | TemplateExpansion
-    root: str | TemplateExpansion | None = None
-    timeout: int | TemplateExpansion = 30
-    refresh_time: int | TemplateExpansion = 300
-    client_server_negotiation: str | TemplateExpansion | None = None
-    client_server_policy: str | TemplateExpansion | None = None
-    encryption_algorithm: str | TemplateExpansion | None = None
-    encryption_key_size: int | TemplateExpansion | None = None
-    encryption_num_hash_rounds: int | TemplateExpansion | None = None
-    encryption_salt_size: int | TemplateExpansion | None = None
-    ssl_verify_server: str | TemplateExpansion | None = None
-    ssl_ca_certificate_file: str | TemplateExpansion | None = None
-    resource: str | TemplateExpansion | None = None
-
-
-class IrodsFileSourceConfiguration(BaseFileSourceConfiguration):
+    IRODSFileSystem = None
+
+
+class IrodsFsspecFileSourceTemplateConfiguration(FsspecBaseFileSourceTemplateConfiguration):
+    host: Union[str, TemplateExpansion]
+    port: Union[int, TemplateExpansion] = 1247
+    username: Union[str, TemplateExpansion]
+    password: Union[str, TemplateExpansion]
+    zone: Union[str, TemplateExpansion]
+    root: Optional[Union[str, TemplateExpansion]] = None
+    timeout: Union[int, TemplateExpansion] = 30
+    refresh_time: Union[int, TemplateExpansion] = 300
+    client_server_negotiation: Optional[Union[str, TemplateExpansion]] = None
+    client_server_policy: Optional[Union[str, TemplateExpansion]] = None
+    encryption_algorithm: Optional[Union[str, TemplateExpansion]] = None
+    encryption_key_size: Optional[Union[int, TemplateExpansion]] = None
+    encryption_num_hash_rounds: Optional[Union[int, TemplateExpansion]] = None
+    encryption_salt_size: Optional[Union[int, TemplateExpansion]] = None
+    ssl_verify_server: Optional[Union[str, TemplateExpansion]] = None
+    ssl_ca_certificate_file: Optional[Union[str, TemplateExpansion]] = None
+    resource: Optional[Union[str, TemplateExpansion]] = None
+
+
+class IrodsFsspecFileSourceConfiguration(FsspecBaseFileSourceConfiguration):
     host: str
     port: int = 1247
     username: str
@@ -65,70 +64,28 @@ class IrodsFileSourceConfiguration(BaseFileSourceConfiguration):
     resource: str | None = None
 
 
-class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfiguration, IrodsFileSourceConfiguration]):
+class IrodsFsspecFilesSource(
+    FsspecFilesSource[IrodsFsspecFileSourceTemplateConfiguration, IrodsFsspecFileSourceConfiguration]
+):
     plugin_type = "irods"
-    required_module = iRODSFS
-    required_package = "fs-irods"
-
-    template_config_class = IrodsFileSourceTemplateConfiguration
-    resolved_config_class = IrodsFileSourceConfiguration
-
-    def _iter_directory_entries(self, fs_handle, parent_path: str, normalized_query: str | None = None):
-        for raw_name in fs_handle.listdir(parent_path):
-            name = os.path.basename(str(raw_name).rstrip("/"))
-            if not name:
-                continue
-            if normalized_query and not fnmatch(name.lower(), f"*{normalized_query}*"):
-                continue
-            entry_path = fs.path.join(parent_path, name)
-            info = fs_handle.getinfo(entry_path, namespaces=["details"])
-            yield entry_path, info
-
-    def _list_non_recursive(
-        self,
-        fs_handle,
-        path: str,
-        limit: int | None = None,
-        offset: int | None = None,
-        query: str | None = None,
-    ) -> tuple[list[AnyRemoteEntry], int]:
-        normalized_query = query.lower() if query else None
-        entries = []
-        for _, info in self._iter_directory_entries(fs_handle, path, normalized_query):
-            entries.append(self._resource_info_to_dict(path, info))
-        count = len(entries)
-        if (page := self._to_page(limit, offset)) is not None:
-            entries = entries[page[0] : page[1]]
-        return entries, count
-
-    def _list(
+    required_module = IRODSFileSystem
+    required_package = "mangofs"
+
+    template_config_class = IrodsFsspecFileSourceTemplateConfiguration
+    resolved_config_class = IrodsFsspecFileSourceConfiguration
+
+    def _open_fs(
         self,
-        context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration],
-        path="/",
-        recursive=False,
-        write_intent: bool = False,
-        limit: int | None = None,
-        offset: int | None = None,
-        query: str | None = None,
-        sort_by: str | None = None,
-    ) -> tuple[list[AnyRemoteEntry], int]:
-        try:
-            with self._open_fs(context) as fs_handle:
-                if recursive:
-                    raise MessageException("Recursive listing is not supported for iRODS file sources.")
-                return self._list_non_recursive(fs_handle, path, limit, offset, query)
-        except fs.errors.PermissionDenied as e:
-            raise AuthenticationRequired(
-                f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}."
-            ) from e
-        except fs.errors.FSError as e:
-            raise MessageException(f"Problem listing file source path {path}. Reason: {e}") from e
-
-    def _open_fs(self, context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration]):
-        if iRODSFS is None or iRODSSession is None:
+        context: FilesSourceRuntimeContext[IrodsFsspecFileSourceConfiguration],
+        cache_options: CacheOptionsDictType,
+    ):
+        if IRODSFileSystem is None or iRODSSession is None:
             raise self.required_package_exception
 
-        config = context.config
+        session = self._open_session(context.config)
+        return IRODSFileSystem(session=session, **cache_options)
+
+    def _open_session(self, config: IrodsFsspecFileSourceConfiguration):
         session_kwargs = {
             "host": config.host,
             "port": config.port,
@@ -152,8 +109,33 @@ def _open_fs(self, context: FilesSourceRuntimeContext[IrodsFileSourceConfigurati
         session.connection_timeout = config.timeout
         if config.resource:
             session.default_resource = config.resource
-
-        return iRODSFS(session=session, root=config.root)
-
-
-__all__ = ("IrodsFilesSource",)
+        return session
+
+    def _to_filesystem_path(self, path: str, config: IrodsFsspecFileSourceConfiguration) -> str:
+        root = self._normalized_root(config)
+        path = path or "/"
+        if path.startswith("irods://"):
+            return path
+        if path == "/":
+            return root
+        return posixpath.join(root, path.lstrip("/"))
+
+    def _adapt_entry_path(self, filesystem_path: str, config: IrodsFsspecFileSourceConfiguration) -> str:
+        root = self._normalized_root(config)
+        filesystem_path = filesystem_path or "/"
+        if filesystem_path == root:
+            return "/"
+        root_prefix = root.rstrip("/") + "/"
+        if filesystem_path.startswith(root_prefix):
+            return "/" + filesystem_path[len(root_prefix) :]
+        return filesystem_path if filesystem_path.startswith("/") else f"/{filesystem_path}"
+
+    @staticmethod
+    def _normalized_root(config: IrodsFsspecFileSourceConfiguration) -> str:
+        if not config.root:
+            return "/"
+        root = os.path.normpath(config.root)
+        return root if root.startswith("/") else f"/{root}"
+
+
+__all__ = ("IrodsFsspecFilesSource",)

lib/galaxy/files/templates/examples/production_irods.yml

@@ -0,0 +1,52 @@
+- id: irods
+  version: 0
+  name: iRODS
+  description: |
+    Use this template to connect an iRODS collection as a Galaxy file source for
+    importing and exporting datasets.
+
+    You need valid iRODS credentials and the target root collection path.
+  variables:
+    host:
+      label: iRODS host
+      type: string
+      help: Hostname of the iRODS server.
+    port:
+      label: iRODS port
+      type: integer
+      help: Port of the iRODS server.
+      default: 1247
+    zone:
+      label: iRODS zone
+      type: string
+      help: Zone used for authentication (for example teamZone).
+    username:
+      label: iRODS user
+      type: string
+      help: Username used to authenticate against iRODS.
+    subpath:
+      label: Subfolder path
+      type: string
+      optional: true
+      help: Optional subfolder under your personal directory (for example projectName).
+    writable:
+      label: Writable?
+      type: boolean
+      optional: true
+      help: Allow writing datasets back to iRODS.
+      default: false
+  secrets:
+    password:
+      label: Password
+      help: Password used to authenticate against iRODS.
+  configuration:
+    type: irods
+    host: "{{ variables.host }}"
+    port: "{{ variables.port }}"
+    username: "{{ variables.username }}"
+    password: "{{ secrets.password }}"
+    zone: "{{ variables.zone }}"
+    root: "/{{ variables.zone }}/home/{{ variables.username }}{% if variables.get('subpath') %}/{{ variables.get('subpath') | ensure_path_component }}{% endif %}"
+    timeout: 30
+    refresh_time: 300
+    writable: "{{ variables.writable }}"

test/unit/app/dependencies/test_deps.py

@@ -26,6 +26,7 @@
 - type: dropbox
 - type: googledrive
 - type: irods
+- type: irods-fsspec
 """
 JOB_CONF_YAML = """
 runners:
@@ -97,6 +98,7 @@ def test_fs_configured():
         assert cds.check_dropboxdrivefs()
         assert cds.check_webdav4()
         assert cds.check_fs_irods()
+        assert cds.check_mangofs()
 
 
 def test_yaml_jobconf_runners():