Merge pull request #623 from European-XFEL/fix/as-single-value-strings

Support and automatically convert string properties in KeyData

Author: philsmt

extra_data/keydata.py

@@ -309,7 +309,7 @@ def data_counts(self, labelled=True):
 
             return res
 
-    def as_single_value(self, rtol=1e-5, atol=0.0, reduce_by='median'):
+    def as_single_value(self, rtol=1e-5, atol=0.0, reduce_by=None):
         """Retrieve a single reduced value if within tolerances.
 
         The relative and absolute tolerances *rtol* and *atol* work the
@@ -322,13 +322,33 @@ def as_single_value(self, rtol=1e-5, atol=0.0, reduce_by='median'):
         the first value encountered. By default, 'median' is used.
 
         If within tolerances, the reduced value is returned.
+
+        For non-numerical keys like strings, the method instead always
+        checks for uniqueness and returns such a value, if present.
         """
 
         data = self.ndarray()
 
         if len(data) == 0:
             raise NoDataError(self.source, self.key)
 
+        if not np.issubdtype(self.dtype, np.number):
+            # Handle non-numeric types first.
+
+            if reduce_by is not None:
+                raise TypeError('custom reduce method not supported for '
+                                'non-numeric type')
+
+            unique_values = np.unique(data, axis=None)
+
+            if len(unique_values) > 1:
+                raise ValueError(f'str values are not unique: {unique_values}')
+
+            return unique_values[0]
+
+        elif reduce_by is None:
+            reduce_by = 'median'
+
         if callable(reduce_by):
             value = reduce_by(data)
         elif isinstance(reduce_by, str) and hasattr(np, reduce_by):
@@ -378,6 +398,16 @@ def ndarray(self, roi=(), out=None):
             )
             dest_cursor = dest_chunk_end
 
+        if out.dtype.hasobject:
+            # Can current only occur for string properties, convert from
+            # object array of bytes to to object array of strings.
+            # This will fail for structured dtypes containing strings,
+            # but such are not known to us yet.
+            out = np.array(
+                [x.decode('utf8', 'surrogateescape') for x in out.flat],
+                dtype=object
+            ).reshape(out.shape)
+
         return out
 
     def train_id_coordinates(self):

extra_data/tests/mockdata/xgm.py

@@ -1,9 +1,11 @@
 import numpy as np
+import h5py
 
 from .base import DeviceBase
 
 class XGM(DeviceBase):
     control_keys = [
+        ('state', h5py.string_dtype(), ()),
         ('beamPosition/ixPos', 'f4', ()),
         ('beamPosition/iyPos', 'f4', ()),
         ('current/bottom/output', 'f4', ()),
@@ -83,3 +85,7 @@ def write_instrument(self, f):
         ds = grp['value']
         ds.attrs['alias'] = b'IX.POS'
         ds.attrs['daqPolicy'] = np.array([1], dtype=np.int32)
+
+        grp = f[f'CONTROL/{self.device_id}/state']
+        grp['value'][:self.ntrains] = b'ON'
+        grp['value'][:5] = b'OFF'

extra_data/tests/test_aliases.py

@@ -319,6 +319,7 @@ def test_alias_deselect(mock_sa3_control_data, mock_sa3_control_aliases_yaml):
     ])
     assert subrun.all_sources == run.all_sources
     assert subrun.alias['sa3-xgm'].keys() == {
+        'state.value', 'state.timestamp',
         'beamPosition.ixPos.value', 'beamPosition.ixPos.timestamp',
         'beamPosition.iyPos.value', 'beamPosition.iyPos.timestamp',
         'pollingInterval.value', 'pollingInterval.timestamp'}

extra_data/tests/test_keydata.py

@@ -11,6 +11,7 @@
 from .mockdata import write_file
 from .mockdata.xgm import XGM
 
+
 def test_get_keydata(mock_spb_raw_run):
     run = RunDirectory(mock_spb_raw_run)
     print(run.instrument_sources)
@@ -43,6 +44,7 @@ def test_get_keydata(mock_spb_raw_run):
     with pytest.raises(TypeError):
         iter(xgm_beam_x)
 
+
 def test_select_trains(mock_spb_raw_run):
     run = RunDirectory(mock_spb_raw_run)
     xgm_beam_x = run['SPB_XTD9_XGM/DOOCS/MAIN', 'beamPosition.ixPos.value']
@@ -277,6 +279,7 @@ def test_single_value(mock_sa3_control_data, monkeypatch):
 
     imager = f['SA3_XTD10_IMGFEL/CAM/BEAMVIEW:daqOutput', 'data.image.pixels']
     flux = f['SA3_XTD10_XGM/XGM/DOOCS', 'pulseEnergy.photonFlux']
+    state = f['SA3_XTD10_XGM/XGM/DOOCS', 'state']
 
     # Try without data for a source and key.
     with pytest.raises(NoDataError):
@@ -309,6 +312,17 @@ def test_single_value(mock_sa3_control_data, monkeypatch):
     assert flux.as_single_value(rtol=1, reduce_by=np.mean) == np.mean(data)
     assert flux.as_single_value(atol=len(data)-1, reduce_by='first') == 0
 
+    # Try strings.
+    assert state[5:].as_single_value() == 'ON'
+
+    with pytest.raises(ValueError):
+        # Contains two unique values.
+        state.as_single_value()
+
+    with pytest.raises(TypeError):
+        # Does not accept reduce_by
+        state.as_single_value(reduce_by='mean')
+
     # Try vector data.
     intensity = f['SA3_XTD10_XGM/XGM/DOOCS:output', 'data.intensityTD']
     data = np.repeat(data, intensity.shape[1]).reshape(-1, intensity.shape[-1])
@@ -332,6 +346,15 @@ def test_ndarray_out(mock_spb_raw_run):
     assert buf_in is buf_out
 
 
+def test_string_arrays(mock_spb_raw_run):
+    f = RunDirectory(mock_spb_raw_run)
+    state = f['SPB_XTD9_XGM/DOOCS/MAIN', 'state']
+
+    for data in [state.ndarray(), state.xarray(), state.series()]:
+        assert data.dtype.hasobject
+        assert (data[3:8] == ['OFF', 'OFF', 'ON', 'ON', 'ON']).all()
+
+
 def test_xarray_structured_data(mock_remi_run):
     run = RunDirectory(mock_remi_run)
     dset = run['SQS_REMI_DLD6/DET/TOP:output', 'rec.hits'].xarray()
@@ -396,6 +419,6 @@ def test_units(mock_sa3_control_data):
 
     # Check that it still works after selecting 0 trains
     assert xgm_intensity.select_trains(np.s_[:0]).units == 'μJ'
-    
+
     # units are added to xarray's attributes
     assert xgm_intensity.xarray().attrs['units'] == 'μJ'