Allow None in Shapes (#895)

Author: Tom-Willemsen

pyproject.toml

@@ -15,7 +15,8 @@ description = "Asynchronous Bluesky hardware abstraction code, compatible with c
 dependencies = [
     "numpy",
     "bluesky>=1.13.1rc2",
-    "event-model>=1.22.1",
+    # Unpin once an event-model release is made containing https://github.com/bluesky/event-model/pull/352
+    "event-model @ git+https://github.com/bluesky/event-model@18e1168c7518401ca691d86f6c5cf5030413251f",
     "pyyaml",
     "colorlog",
     "pydantic>=2.0",

src/ophyd_async/core/_hdf_dataset.py

@@ -23,7 +23,7 @@ class HDFDatasetDescription(BaseModel):
     """The dataset name within the HDF file,
     e.g. /entry/data/data or /entry/instrument/NDAttributes/sum"""
 
-    shape: tuple[int, ...] = Field(default_factory=tuple)
+    shape: tuple[int | None, ...] = Field(default_factory=tuple)
     """The shape of a single event's data in the HDF file,
     e.g. (1, 768, 1024) for arrays or () for scalars"""
 

src/ophyd_async/core/_providers.py

@@ -228,5 +228,5 @@ async def np_datatype(self) -> str:
         """Return the numpy datatype for this dataset."""
 
     @abstractmethod
-    async def shape(self) -> tuple[int, ...]:
+    async def shape(self) -> tuple[int | None, ...]:
         """Get the shape of the data collection."""

src/ophyd_async/core/_signal_backend.py

@@ -176,7 +176,7 @@ def _datakey_dtype_numpy(
         raise TypeError(f"Can't make dtype_numpy for {datatype}")
 
 
-def _datakey_shape(value: SignalDatatype) -> list[int]:
+def _datakey_shape(value: SignalDatatype) -> list[int | None]:
     if type(value) in _primitive_dtype or isinstance(value, EnumTypes):
         return []
     elif isinstance(value, np.ndarray):

src/ophyd_async/epics/adcore/_hdf_writer.py

@@ -1,6 +1,7 @@
 import asyncio
 from collections.abc import AsyncIterator
 from pathlib import Path
+from typing import TypeGuard
 from xml.etree import ElementTree as ET
 
 from bluesky.protocols import StreamAsset
@@ -22,6 +23,10 @@
 )
 
 
+def _is_fully_described(shape: tuple[int | None, ...]) -> TypeGuard[tuple[int, ...]]:
+    return None not in shape
+
+
 class ADHDFWriter(ADWriter[NDFileHDFIO]):
     """Allow `NDFileHDFIO` to be used within `StandardDetector`."""
 
@@ -75,6 +80,16 @@ async def open(
         # Determine number of frames that will be saved per HDF chunk
         frames_per_chunk = await self.fileio.num_frames_chunks.get_value()
 
+        if not _is_fully_described(detector_shape):
+            # Questions:
+            # - Can AreaDetector support this?
+            # - How to deal with chunking?
+            # Don't support for now - leave option open to support it later
+            raise ValueError(
+                "Datasets with partially unknown dimensionality "
+                "are not currently supported by ADHDFWriter."
+            )
+
         # Add the main data
         self._datasets = [
             HDFDatasetDescription(

tests/epics/adcore/test_writers.py

@@ -296,3 +296,19 @@ async def test_nd_attributes_plan_stub_gives_correct_error(RE, detectors):
             == f"Invalid type for ndattributes: {type(invalidObjects[0])}. "
             + "Expected NDAttributePv or NDAttributeParam."
         )
+
+
+async def test_hdf_writer_descriptor_shape_contains_none(
+    hdf_writer: adcore.ADHDFWriter,
+):
+    async def shape_containing_none():
+        return 10, None, 10
+
+    hdf_writer._dataset_describer.shape = shape_containing_none
+
+    with pytest.raises(
+        ValueError,
+        match=r"Datasets with partially unknown dimensionality "
+        r"are not currently supported by ADHDFWriter.",
+    ):
+        await hdf_writer.open(DETECTOR_NAME)