Skip to content

Commit f43eeb3

Browse files
committed
♻️ PickledData: derive from Data, not SinglefileData (#7600)
`ShellJob` writes every `SinglefileData` input into the working directory, so passing a `PickledData` through the `nodes` port dropped raw pickle bytes there as an input file. Pickled bytes are not a file the caller supplied, and inheritance said they were. Store them in the node repository directly, under `obj.pickle`. The entry point still names the node type, so existing nodes keep loading; only the inherited `filename` attribute goes.
1 parent 794608f commit f43eeb3

3 files changed

Lines changed: 12 additions & 8 deletions

File tree

src/aiida/orm/nodes/data/pickled.py

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,24 @@
1212

1313
import importlib
1414
import importlib.metadata
15-
import io
1615
import sys
1716
import types
1817
import typing as t
1918

2019
from aiida.common.log import AIIDA_LOGGER
21-
from aiida.orm.nodes.data.singlefile import SinglefileData
20+
from aiida.orm.nodes.data.data import Data
2221

2322
__all__ = ('PickledData',)
2423

2524
LOGGER = AIIDA_LOGGER.getChild('pickled_data')
2625

2726
DEFAULT_PICKLER: str = 'dill'
2827

28+
FILENAME: str = 'obj.pickle'
29+
"""Name the pickled bytes are stored under in the node repository."""
2930

30-
class PickledData(SinglefileData):
31+
32+
class PickledData(Data):
3133
"""Data plugin to store (almost) any Python object by pickling it.
3234
3335
The pickler is chosen per node and recorded on it, so that a node knows how to unpickle itself::
@@ -37,6 +39,10 @@ class PickledData(SinglefileData):
3739
3840
Any module providing ``dumps`` and ``loads`` can be named. A pickler that does not follow that convention is
3941
used through a small adapter module rather than by subclassing this class.
42+
43+
This deliberately does not extend :class:`~aiida.orm.nodes.data.singlefile.SinglefileData`. Pickled bytes are
44+
not a file the caller supplied, and ``ShellJob`` writes every ``SinglefileData`` input into the working
45+
directory, which would dump the raw pickle there.
4046
"""
4147

4248
KEY_ATTRIBUTES_PICKLER: str = 'pickler'
@@ -68,8 +74,9 @@ def __init__(self, obj: t.Any, *, pickler: str = DEFAULT_PICKLER, **kwargs: t.An
6874
"""
6975
module = self._load_pickler(pickler)
7076

71-
super().__init__(file=io.BytesIO(module.dumps(obj, **kwargs)))
77+
super().__init__()
7278

79+
self.base.repository.put_object_from_bytes(module.dumps(obj, **kwargs), FILENAME)
7380
self.base.attributes.set(self.KEY_ATTRIBUTES_PICKLER, pickler)
7481
self.base.attributes.set(self.KEY_ATTRIBUTES_PICKLER_VERSION, self._get_distribution_version(pickler))
7582
self.base.attributes.set(self.KEY_ATTRIBUTES_PICKLER_KWARGS, kwargs)
@@ -189,7 +196,7 @@ def load(self) -> t.Any:
189196
:returns: The unpickled Python object.
190197
:raises ValueError: If the stored pickled object could not be unpickled.
191198
"""
192-
with self.open(mode='rb') as handle:
199+
with self.base.repository.open(FILENAME, mode='rb') as handle:
193200
pickled = handle.read()
194201

195202
try:

tests/orm/data/test_pickled.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ def test_records_pickling_environment():
3636
"""Test that the constructor records everything needed to decide whether this node can be unpickled here."""
3737
node = PickledData(None)
3838
assert node.base.attributes.all == {
39-
'filename': node.filename,
4039
'pickler': 'dill',
4140
'pickler_version': dill.__version__,
4241
'pickler_kwargs': {},

tests/orm/test_fields/fields_aiida.data.core.pickled.PickledData.yml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ ctime: QbNumericField('ctime', dtype=<class 'datetime.datetime'>, doc='The creat
55
time of the node')
66
description: QbStrField('description', dtype=<class 'str'>, doc='The node description')
77
extras: QbDictField('extras', dtype=dict[str, typing.Any], doc='The node extras')
8-
filename: QbStrField('attributes.filename', dtype=<class 'str'>, doc='The name of
9-
the stored file')
108
label: QbStrField('label', dtype=<class 'str'>, doc='The node label')
119
mtime: QbNumericField('mtime', dtype=<class 'datetime.datetime'>, doc='The modification
1210
time of the node')

0 commit comments

Comments
 (0)