Fix bug in ChargeDistributionFilter for pedestal maker (cta-observatory#213)

* Only fill charges container if waveforms are available

* Cleanup commented stuff from some previous PR

* Pin `persistent` (dependency for `ZODB`) version to fix CI

---------

Co-authored-by: Jean-Philippe Lenain <[email protected]>
# Conflicts:
#	src/nectarchain/makers/component/pedestal_component.py

Author: tibaldo

environment.yml

@@ -24,6 +24,7 @@ dependencies:
   - pip:
     - zeo
     - zodb
+    - persistent<6.2
     - mechanize
     - browser-cookie3
     - pyqtgraph

pyproject.toml

@@ -26,6 +26,7 @@ dependencies = [
     "iminuit",
     "matplotlib",
     "pandas",
+    "persistent<6.2",
     "scipy",
     "zodb",
     "zeo",

src/nectarchain/dqm/bokeh_app/tests/test_app_hooks.py

@@ -1,10 +1,12 @@
 import numpy as np
-from bokeh.io import output_file, save
 
 # bokeh imports
+from bokeh.io import output_file, save
 from bokeh.layouts import layout
 from bokeh.models import Select
 from bokeh.plotting import curdoc
+
+# ctapipe imports
 from ctapipe.coordinates import EngineeringCameraFrame
 from ctapipe.instrument import CameraGeometry
 from ZODB import DB

src/nectarchain/makers/component/pedestal_component.py

@@ -202,50 +202,38 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs):
         )
 
     @staticmethod
-    def calculate_stats(container, mask, statistics):
+    def calculate_stats(waveformsContainers, wfs_mask, statistics):
         """Calculate statistics for the pedestals from a waveforms container.
 
         Parameters
         ----------
-        containers : `~nectarchain.data.container`
-            Waveforms or Charges container
-        mask : `numpy.ndarray`
-            Mask to apply to exclude outliers with shape (n_events,n_pixels,n_samples)
+        waveformsContainers : `~nectarchain.data.container.WaveformsContainer`
+            Waveforms container
+        wfs_mask : `numpy.ndarray`
+            Mask to apply to exclude outliers with shape (n_pixels,n_samples)
         statistics : `list`
             Names of the statistics (numpy.ma attributes) to compute
 
         Returns
         -------
         ped_stats : `dict`
-            A dictionary containing 3D (n_chan,n_pixels,n_samples)
-            or 2D (n_chan,n_pixels) arrays for each statistic.
+            A dictionary containing 3D (n_chan,n_pixels,n_samples) arrays for each
+            statistic.
         """
 
-        # Detect container type by attribute name
-        if hasattr(container, "wfs_hg"):
-            data_hg = container.wfs_hg
-            data_lg = container.wfs_lg
-            is_waveform = True
-        else:
-            data_hg = container.charges_hg
-            data_lg = container.charges_lg
-            is_waveform = False
-
-        # Adapt mask shape
-        # Waveforms: (n_events, n_pixels, n_samples)
-        # Charges: (n_events, n_pixels)
-        if not is_waveform:
-            mask = mask[:, :, 0]
-
         ped_stats = {}
 
         for stat in statistics:
             # Calculate the statistic along axis = 0, that is over events
-            ped_stat_hg = getattr(ma, stat)(ma.masked_array(data_hg, mask), axis=0)
-            ped_stat_lg = getattr(ma, stat)(ma.masked_array(data_lg, mask), axis=0)
+            ped_stat_hg = getattr(ma, stat)(
+                ma.masked_array(waveformsContainers.wfs_hg, wfs_mask), axis=0
+            )
+            ped_stat_lg = getattr(ma, stat)(
+                ma.masked_array(waveformsContainers.wfs_lg, wfs_mask), axis=0
+            )
 
             # Create a 3D array for the statistic
-            array_shape = np.append([N_GAINS], np.shape(data_hg[0]))
+            array_shape = np.append([N_GAINS], np.shape(waveformsContainers.wfs_hg[0]))
             ped_stat = np.zeros(array_shape)
             ped_stat[HIGH_GAIN] = ped_stat_hg
             ped_stat[LOW_GAIN] = ped_stat_lg
@@ -318,7 +306,7 @@ def flag_bad_pixels(self, ped_stats, nevents):
         # Flag on standard deviation per pixel
         # Standard deviation of pedestal in channel/pixel above threshold
         log.info(
-            f"Flag pixels with pedestal standard deviation in a channel/pixel above "
+            f"Flag pixels with pedestal standard deviation in a chennel/pixel above "
             f"the maximum acceptable value {self.pixel_mask_std_pixel_max}"
         )
         flag_pixel_std = np.int8(
@@ -508,8 +496,6 @@ def finish(self, *args, **kwargs):
         self._waveformsContainers = waveformsContainers.containers[
             EventType.SKY_PEDESTAL
         ]
-        # log.info('JPL: waveformsContainers=',waveformsContainers.containers[
-        # EventType.SKY_PEDESTAL].nsamples)
 
         # Check if waveforms container is empty
         if self._waveformsContainers is None:
@@ -524,8 +510,12 @@ def finish(self, *args, **kwargs):
             # container with no results
             return None
         else:
-            # Make sure that the charge distribution container is filled
-            if self._chargesContainers is None:
+            # If we want to filter based on charges distribution
+            # make sure that the charge distribution container is filled
+            if (
+                self.filter_method == "ChargeDistributionFilter"
+                and self._chargesContainers is None
+            ):
                 log.debug("Compute charges from waveforms")
                 chargesComponent_kwargs = {}
                 chargesComponent_configurable_traits = (
@@ -585,9 +575,6 @@ def finish(self, *args, **kwargs):
             self._ped_stats = self.calculate_stats(
                 self._waveformsContainers, self._wfs_mask, statistics
             )
-            self._charge_stats = self.calculate_stats(
-                self._chargesContainers, self._wfs_mask, statistics
-            )
 
             # calculate the number of events per pixel used to compute the quantitites
             # start wit total number of events
@@ -612,10 +599,6 @@ def finish(self, *args, **kwargs):
                 pedestal_mean_lg=self._ped_stats["mean"][LOW_GAIN],
                 pedestal_std_hg=self._ped_stats["std"][HIGH_GAIN],
                 pedestal_std_lg=self._ped_stats["std"][LOW_GAIN],
-                pedestal_charge_mean_hg=self._charge_stats["mean"][HIGH_GAIN],
-                pedestal_charge_mean_lg=self._charge_stats["mean"][LOW_GAIN],
-                pedestal_charge_std_hg=self._charge_stats["std"][HIGH_GAIN],
-                pedestal_charge_std_lg=self._charge_stats["std"][LOW_GAIN],
                 pixel_mask=pixel_mask,
             )