🐛 COPY stash handles missing sources unlike the compressed modes

`RemoteStashFolderData.source_list` documents "the source files that were
stashed", but with `fail_on_missing=False` it kept the entries skipped for
being missing: record only what was copied. With `fail_on_missing=True` a
non-matching glob silently copied nothing: reject glob patterns, as the
compressed modes do.

Author: khsrali

src/aiida/engine/daemon/execmanager.py

@@ -501,16 +501,24 @@ async def stash_calculation(calculation: CalcJobNode, transport: Transport) -> N
         else:
             target_basepath = target_base / uuid[:2] / uuid[2:4] / uuid[4:]
 
-        async def _do_copy():
+        async def _do_copy() -> list[str]:
+            stashed_source_list: list[str] = []
             for source_filename in source_list:
                 if has_magic(source_filename):
+                    if fail_on_missing:
+                        msg = (
+                            'Stashing with glob patterns is not supported when fail_on_missing is True. '
+                            'Stashing failed.'
+                        )
+                        raise exceptions.StashingError(msg)
                     copy_instructions = []
                     for globbed_filename in await transport.glob_async(source_basepath / source_filename):
                         target_filepath = target_basepath / Path(globbed_filename).relative_to(source_basepath)
                         copy_instructions.append((globbed_filename, target_filepath))
                 else:
                     copy_instructions = [(source_basepath / source_filename, target_basepath / source_filename)]
 
+                stashed = False
                 for source_filepath, target_filepath in copy_instructions:
                     # If source is in a (nested) directory, create those directories first
                     target_dirname = target_filepath.parent
@@ -533,9 +541,13 @@ async def _do_copy():
                             f'Failed to copy {source_filepath} to {target_filepath}: {exc}'
                         ) from exc
                     EXEC_LOGGER.debug(f'Stashed from {source_filepath} to {target_filepath}')
+                    stashed = True
+                if stashed:
+                    stashed_source_list.append(source_filename)
+            return stashed_source_list
 
         try:
-            await _do_copy()
+            stashed_source_list = await _do_copy()
         except exceptions.StashingError as exception:
             await transport.rmtree_async(target_basepath)
             raise exception
@@ -546,7 +558,7 @@ async def _do_copy():
             computer=calculation.computer,
             target_basepath=str(target_basepath),
             stash_mode=StashMode(stash_mode),
-            source_list=source_list,
+            source_list=stashed_source_list,
             fail_on_missing=fail_on_missing,
         ).store()
 

tests/engine/daemon/test_execmanager.py

@@ -811,9 +811,10 @@ async def mock_compress_partial(*args, **kwargs):
     # Overwrites: a job can only replace its own leftover, see ``test_stashing_same_source_twice``
 
 
+@pytest.mark.parametrize('stash_mode', [StashMode.COPY.value, StashMode.COMPRESS_TARGZ.value])
 @pytest.mark.asyncio
-async def test_stashing_compress_skips_missing(generate_calcjob_node, tmp_path, monkeypatch):
-    """With ``fail_on_missing=False`` a compressed stash skips missing sources instead of failing silently."""
+async def test_stashing_skips_missing(generate_calcjob_node, stash_mode, tmp_path, monkeypatch):
+    """With ``fail_on_missing=False`` missing sources are skipped and the stash node records only what was stashed."""
     node = generate_calcjob_node()
     workdir = tmp_path / 'workdir'
     workdir.mkdir()
@@ -825,7 +826,7 @@ async def test_stashing_compress_skips_missing(generate_calcjob_node, tmp_path,
         {
             'source_list': ['present.out', 'missing.out', 'nomatch*'],
             'target_base': str(target_base),
-            'stash_mode': StashMode.COMPRESS_TARGZ.value,
+            'stash_mode': stash_mode,
         },
     )
 
@@ -838,15 +839,42 @@ def get_workdir(self, *args, **kwargs):
 
     with LocalTransport() as transport:
         await execmanager.stash_calculation(node, transport)
-        transport.extract(target_base / f'{node.uuid}.tar.gz', tmp_path / 'extracted')
-
-    assert [path.name for path in (tmp_path / 'extracted').iterdir()] == ['present.out']
+        if stash_mode == StashMode.COPY.value:
+            stashed = target_base / node.uuid[:2] / node.uuid[2:4] / node.uuid[4:]
+        else:
+            stashed = tmp_path / 'extracted'
+            transport.extract(target_base / f'{node.uuid}.{stash_mode}', stashed)
 
-    # The stash node must record only what actually went into the archive
+    assert [path.name for path in stashed.iterdir()] == ['present.out']
     remote_stash = node.base.links.get_outgoing(link_label_filter='remote_stash').one().node
     assert list(remote_stash.source_list) == ['present.out']
 
 
+@pytest.mark.parametrize('stash_mode', [StashMode.COPY.value, StashMode.COMPRESS_TARGZ.value])
+@pytest.mark.asyncio
+async def test_stashing_fail_on_missing_rejects_glob(generate_calcjob_node, stash_mode, tmp_path, monkeypatch):
+    """With ``fail_on_missing=True`` glob patterns are rejected, since a non-matching one cannot be told apart."""
+    node = generate_calcjob_node(workdir=tmp_path)
+    node.set_option(
+        'stash',
+        {
+            'source_list': ['*.out'],
+            'target_base': str(tmp_path / 'stash'),
+            'stash_mode': stash_mode,
+            'fail_on_missing': True,
+        },
+    )
+
+    class MockAuthInfo:
+        def get_workdir(self, *args, **kwargs):
+            return str(tmp_path)
+
+    monkeypatch.setattr(node, 'get_authinfo', MockAuthInfo)
+
+    with LocalTransport() as transport, pytest.raises(StashingError, match='glob patterns'):
+        await execmanager.stash_calculation(node, transport)
+
+
 @pytest.mark.parametrize('stash_mode', [StashMode.COPY.value, StashMode.COMPRESS_TARGZ.value])
 @pytest.mark.asyncio
 async def test_stashing_same_source_twice(generate_calcjob_node, aiida_localhost, stash_mode, tmp_path, monkeypatch):