Rewrite resolved container image path via the compute environment

lib/galaxy/job_execution/compute_environment.py

@@ -53,6 +53,15 @@ def input_metadata_rewrite(self, dataset, metadata_value):
     def unstructured_path_rewrite(self, path):
         """Rewrite loc file paths, etc.."""
 
+    def container_path_rewrite(self, path):
+        """Rewrite a resolved container image path for the compute environment.
+
+        No-op by default; overridden where Galaxy and the compute environment
+        may resolve container images at different filesystem paths (e.g. Pulsar
+        in ``rewrite_parameters`` mode).
+        """
+        return None
+
     @abstractmethod
     def working_directory(self):
         """Job working directory (potentially remote)"""

lib/galaxy/jobs/runners/pulsar.py

@@ -589,6 +589,7 @@ def __prepare_job(self, job_wrapper: "MinimalJobWrapper", job_destination):
                     compute_tool_directory=remote_tool_directory,
                     compute_job_directory=remote_job_directory,
                 )
+                self._rewrite_container_for_compute_environment(container, compute_environment)
 
             # Pulsar handles ``create_tool_working_directory`` and
             # ``include_work_dir_outputs`` details.
@@ -620,6 +621,30 @@ def __prepare_job(self, job_wrapper: "MinimalJobWrapper", job_destination):
 
         return command_line, client, remote_job_config, compute_environment, remote_container
 
+    @staticmethod
+    def _rewrite_container_for_compute_environment(container, compute_environment):
+        """Rewrite the resolved container image path for the compute environment.
+
+        Container resolution runs against the Galaxy-side filesystem, but the
+        image is read on the compute node, which may expose e.g. CVMFS at a
+        different path (cvmfsexec mountrepo mode). Route the resolved image path
+        through ``ComputeEnvironment.container_path_rewrite`` (only populated in
+        ``rewrite_parameters`` mode) so a destination ``file_actions`` rule with
+        ``path_types: container`` can remap it.
+
+        No-op when there is no compute environment (the rewrite is a Pulsar
+        ``rewrite_parameters`` concept), when the image identifier is not a
+        filesystem path (e.g. a ``docker://`` or registry reference, which the
+        compute node resolves itself), or when no matching rule is configured.
+        """
+        if container is None or compute_environment is None:
+            return
+        if not container.image_identifier_is_path:
+            return
+        rewritten_container_id = compute_environment.container_path_rewrite(container.container_id)
+        if rewritten_container_id:
+            container.container_id = rewritten_container_id
+
     def __prepare_input_files_locally(self, job_wrapper):
         """Run task splitting commands locally."""
         prepare_input_files_cmds = getattr(job_wrapper, "prepare_input_files_cmds", None)
@@ -1322,6 +1347,14 @@ def unstructured_path_rewrite(self, parameter_value):
             # Did not need to rewrite, use original path or value.
             return None
 
+    def container_path_rewrite(self, container_path):
+        # Container images are resolved against the Galaxy-side filesystem but
+        # read on the compute node, which may expose them at a different path
+        # (e.g. cvmfsexec mountrepo mode). A destination ``file_actions`` rule
+        # with ``path_types: container`` remaps the image; unlike unstructured
+        # paths, container images are never staged.
+        return self.path_mapper.check_for_container_rewrite(container_path)
+
     def working_directory(self):
         return self._working_directory
 

lib/galaxy/tool_util/deps/container_classes.py

@@ -154,6 +154,18 @@ def source_environment(self) -> str:
             return SOURCE_CONDA_ACTIVATE
         return ""
 
+    @property
+    def image_identifier_is_path(self) -> bool:
+        """Whether ``container_id`` refers to a local filesystem path.
+
+        Most container identifiers are registry references (e.g.
+        ``quay.io/biocontainers/...`` or ``docker://...``) rather than paths.
+        Only when the identifier is a local path does it make sense to rewrite
+        it for a compute node with a different filesystem layout (e.g. a
+        Singularity/Apptainer image cached on CVMFS).
+        """
+        return False
+
     @abstractmethod
     def containerize_command(self, command: str) -> str:
         """
@@ -519,6 +531,14 @@ def docker_cache_path(cache_directory: str, container_id: str) -> str:
 class SingularityContainer(Container, HasDockerLikeVolumes):
     container_type = SINGULARITY_CONTAINER_TYPE
 
+    @property
+    def image_identifier_is_path(self) -> bool:
+        # A Singularity/Apptainer image identifier is either a local path (e.g.
+        # a ``.sif`` file or a CVMFS-cached image) or a URI with a scheme such
+        # as ``docker://``, ``library://``, ``shub://``, or ``oras://``. Only
+        # absolute paths are read from the (compute-node) filesystem.
+        return os.path.isabs(self.container_id)
+
     def get_singularity_target_kwds(self) -> dict[str, Any]:
         return dict(
             singularity_cmd=self.prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND),

test/unit/app/jobs/test_pulsar_runner.py

@@ -0,0 +1,60 @@
+"""Unit tests for Pulsar job runner utility methods."""
+
+from types import SimpleNamespace
+
+from galaxy.jobs.runners.pulsar import PulsarJobRunner
+
+
+def _container(container_id, image_identifier_is_path=True):
+    return SimpleNamespace(container_id=container_id, image_identifier_is_path=image_identifier_is_path)
+
+
+class _ComputeEnvironment:
+    """Minimal stand-in exposing only container_path_rewrite."""
+
+    def __init__(self, rewrites):
+        self._rewrites = rewrites
+
+    def container_path_rewrite(self, path):
+        return self._rewrites.get(path)
+
+
+IMAGE = "/cvmfs/singularity.galaxyproject.org/all/img"
+REWRITTEN = "/job/dir/.cvmfsexec/dist/cvmfs/singularity.galaxyproject.org/all/img"
+
+
+def test_rewrite_container_applies_compute_environment_rewrite():
+    container = _container(IMAGE)
+    compute_environment = _ComputeEnvironment({IMAGE: REWRITTEN})
+    PulsarJobRunner._rewrite_container_for_compute_environment(container, compute_environment)
+    assert container.container_id == REWRITTEN
+
+
+def test_rewrite_container_noop_without_matching_rule():
+    # container_path_rewrite returns None when no file_actions rule matches.
+    container = _container(IMAGE)
+    compute_environment = _ComputeEnvironment({})
+    PulsarJobRunner._rewrite_container_for_compute_environment(container, compute_environment)
+    assert container.container_id == IMAGE
+
+
+def test_rewrite_container_noop_when_identifier_not_a_path():
+    # A registry/docker:// identifier is resolved by the compute node itself;
+    # never route it through the path rewriter even if a rule would match.
+    container = _container("docker://quay.io/biocontainers/bwa", image_identifier_is_path=False)
+    compute_environment = _ComputeEnvironment({"docker://quay.io/biocontainers/bwa": REWRITTEN})
+    PulsarJobRunner._rewrite_container_for_compute_environment(container, compute_environment)
+    assert container.container_id == "docker://quay.io/biocontainers/bwa"
+
+
+def test_rewrite_container_noop_without_compute_environment():
+    # No compute environment => not rewrite_parameters mode; leave image as-is.
+    container = _container(IMAGE)
+    PulsarJobRunner._rewrite_container_for_compute_environment(container, None)
+    assert container.container_id == IMAGE
+
+
+def test_rewrite_container_noop_without_container():
+    # Should not raise when there is no resolved container.
+    compute_environment = _ComputeEnvironment({IMAGE: REWRITTEN})
+    PulsarJobRunner._rewrite_container_for_compute_environment(None, compute_environment)

test/unit/tool_util/test_container_classes.py

@@ -0,0 +1,37 @@
+"""Unit tests for Container classes."""
+
+from galaxy.tool_util.deps.container_classes import (
+    DockerContainer,
+    SingularityContainer,
+)
+
+
+def _build(container_class, container_id):
+    return container_class(
+        container_id=container_id,
+        app_info=None,
+        tool_info=None,
+        destination_info={},
+        job_info=None,
+        container_description=None,
+    )
+
+
+def test_docker_identifier_is_never_a_path():
+    container = _build(DockerContainer, "quay.io/biocontainers/bwa:0.7.17")
+    assert container.image_identifier_is_path is False
+
+
+def test_singularity_absolute_path_is_a_path():
+    container = _build(SingularityContainer, "/cvmfs/singularity.galaxyproject.org/all/bwa:0.7.17")
+    assert container.image_identifier_is_path is True
+
+
+def test_singularity_docker_uri_is_not_a_path():
+    container = _build(SingularityContainer, "docker://quay.io/biocontainers/bwa:0.7.17")
+    assert container.image_identifier_is_path is False
+
+
+def test_singularity_library_uri_is_not_a_path():
+    container = _build(SingularityContainer, "library://sylabsed/examples/lolcow")
+    assert container.image_identifier_is_path is False