Support several NectarCAM cameras

notebooks/Access NectarCAM data using DIRAC API.py

@@ -7,59 +7,61 @@
 #       format_version: '1.3'
 #       jupytext_version: 1.14.6
 #   kernelspec:
-#     display_name: nectarchain-ctapipe0.19
+#     display_name: nectar-dev
 #     language: python
-#     name: nectarchain-ctapipe0.19
+#     name: nectar-dev
 # ---
 
 # %% [markdown]
-# # How to access NectarCAM data from the EGI grid directly using the DIRAC API ?
+# # How to access NectarCAM data from the EGI grid directly using through DIRAC within `nectarchain` ?
 #
-# In this short notebook, we will see how to access data stored on the grid using the
-# DIRAC API, without the hassle of first manually downloading them locally.
+# In this short notebook, we will see how to access data stored on the grid through DIRAC within `nectarchain` itself, without the hassle of first manually downloading them locally.
 #
-# In order to achieve this, you will obviously need a `conda` environment in which
-# all relevant code is installed, such as `ctapipe`, `nectarchain`, but also
-# `CTADIRAC` itself. Please refer to the `nectarchain` installation procedure at
-# <https://github.com/cta-observatory/nectarchain> and follow the instructions to
-# enable DIRAC support.
+# In order to achieve this, you will obviously need a `conda` environment in which all relevant code is installed, such as `ctapipe`, `nectarchain`, but also `CTADIRAC` itself. Please refer to the `nectarchain` installation procedure at <https://github.com/cta-observatory/nectarchain> and follow the instructions to enable DIRAC support.
 #
 # You will also need to have an active proxy for EGI, initialized e.g. with:
 #
 # ```
 # dirac-proxy-init -M -g cta_nectarcam
 # ```
 #
-# You can also check whether you currently have an active proxy with the command
-# `dirac-proxy-info`.
+# You can also check whether you currently have an active proxy with the command `dirac-proxy-info`.
 
 # %%
-import os
-from glob import glob
+from nectarchain.data import DataManagement
 
-from ctapipe.coordinates import EngineeringCameraFrame
-from ctapipe.instrument import CameraGeometry
-from ctapipe.io import EventSeeker, EventSource
-from ctapipe.visualization import CameraDisplay
+# %%
+dm = DataManagement()
+dm.findrun(6881)
+
+# %% [markdown]
+# Once the files are downloaded, the same command will *not* fetch them again, but will detect that the data are already locally present:
+
+# %%
+dm.findrun(6881)
+
+# %% [markdown]
+# It is also possible to fetch data using the DIRAC API directly. Under the hood, this is exactly what is done within `nectarchain` in the example above.
 
 # %%
-# %matplotlib inline
 from DIRAC.Interfaces.API.Dirac import Dirac
 
 # %%
 dirac = Dirac()
 
 # %%
-# dirac.getFile?
+# ?dirac.getFile
 
 # %%
 lfns = [
-    "/vo.cta.in2p3.fr/nectarcam/2022/20220411/NectarCAM.Run3169.0000.fits.fz",
-    "/vo.cta.in2p3.fr/nectarcam/2022/20220411/NectarCAM.Run3169.0001.fits.fz",
+    "/ctao/nectarcam/NectarCAMQM/2025/20250722/NectarCAM.Run6881.0000.fits.fz",
+    "/ctao/nectarcam/NectarCAMQM/2025/20250722/NectarCAM.Run6881.0001.fits.fz",
 ]
 
 # %%
-tmpdir = f"/tmp/{os.environ['USER']}/scratch"
+import os
+
+tmpdir = f"{os.environ['NECTARCAMDATA']}/runs"
 if not os.path.isdir(tmpdir):
     print(f"{tmpdir} does not exist yet, I will create it for you")
     os.makedirs(tmpdir)
@@ -71,16 +73,42 @@
 # **You are now ready to work with `ctapipe` as usual!**
 
 # %%
+# %matplotlib inline
+import numpy as np
+from matplotlib import pyplot as plt
+from traitlets.config import Config
+
+from ctapipe.coordinates import EngineeringCameraFrame
+from ctapipe.io import EventSource, EventSeeker
+from ctapipe.instrument import CameraGeometry
+from ctapipe.visualization import CameraDisplay
+
+# %%
+from glob import glob
+
 path = glob(f"{tmpdir}/NectarCAM.*.fits.fz")
 path.sort()
-reader = EventSource(input_url=path[0])
-seeker = EventSeeker(reader)
-
-# Get some event, and display camera charges for the high gain channel
-# (no time window optimization)
-evt = seeker.get_event_index(10)
-image = evt.r0.tel[0].waveform.sum(axis=2)
-camera = CameraGeometry.from_name("NectarCam-003").transform_to(
+
+config = Config(
+    dict(
+        NectarCAMEventSource=dict(
+            NectarCAMR0Corrections=dict(
+                calibration_path=None,
+                apply_flatfield=False,
+                select_gain=False,
+            )
+        )
+    )
+)
+
+reader = EventSource(input_url=path[0], config=config, max_events=100)
+
+tel_id = reader.subarray.tel_ids[0]
+
+# Get some event, and display camera charges for the high gain channel (no time window optimization)
+evt = next(iter(reader))
+image = evt.r0.tel[tel_id].waveform.sum(axis=2)
+camera = reader.subarray.tel[tel_id].camera.geometry.transform_to(
     EngineeringCameraFrame()
 )
 disp = CameraDisplay(geometry=camera, image=image[0])

notebooks/tool_implementation/tuto_photostat.py

@@ -74,6 +74,7 @@
     progress_bar=True,
     run_number=FF_run_number,
     Ped_run_number=Ped_run_number,
+    camera="NectarCAMQM",
     SPE_result=path[0],
     method="LocalPeakWindowSum",
     extractor_kwargs={"window_width": 12, "window_shift": 4},

notebooks/tool_implementation/tuto_tools.py

@@ -19,22 +19,33 @@
 # ## Context
 
 # %% [markdown]
-# Tool and Component are 2 modules of ctapipe, Tool is a high level module to analyse raw data (fits.fz files). This module use Component to perform computation on the raw data. Basically, we can create a class (MyTool) which inherits of Tool, where we can define 2 Component (Comp_A and Comp_B). Thus, with an instance of MyTool, we can loop over event within raw data, and for each event apply sucessively Comp_A, Comp_B.
+# Tool and Component are 2 modules of ctapipe, Tool is a high level module to analyse
+# raw data (fits.fz files).
+# This module use Component to perform computation on the raw data. Basically, we can
+# create a class (MyTool) which inherits of Tool, where we can define two Component
+# (Comp_A and Comp_B). Thus, with an instance of MyTool, we can loop over event within
+# raw data, and for each event apply successively Comp_A, Comp_B.
 #
-# A ctapipe tutorial is accessible here : https://ctapipe.readthedocs.io/en/stable/auto_examples/core/command_line_tools.html#sphx-glr-auto-examples-core-command-line-tools-py
+# A ctapipe tutorial is accessible here:
+# https://ctapipe.readthedocs.io/en/stable/auto_examples/core/command_line_tools.html
+# #sphx-glr-auto-examples-core-command-line-tools-py
 #
-# You can find documentation of ctapipe Tool and Component :
+# You can find documentation of ctapipe Tool and Component:
 #
 # https://ctapipe.readthedocs.io/en/stable/api-reference/tools/index.html
 #
 #
 # https://ctapipe.readthedocs.io/en/stable/api/ctapipe.core.Component.html
 #
-# Within nectarchain, we implemented within the nectarchain.makers module both a top level Tool and Component from which all the nectarchain Component and Tool should inherit.
+# Within nectarchain, we implemented within the nectarchain.makers module both a top
+# level Tool and Component from which all the nectarchain Component and Tool should
+# inherit.
 #
-# In this tutorial, we explain quickly how we can use Tool and Component to develop the nectarchain software, as an example there is the implementation of a PedestalTool which extract pedestal
+# In this tutorial, we explain quickly how we can use Tool and Component to develop the
+# nectarchain software, as an example there is the implementation of a PedestalTool
+# which extract pedestal
 
-# %% [markdown]
+# %%
 # ### Imports
 
 import os
@@ -45,20 +56,14 @@
 # %%
 import numpy as np
 from ctapipe.containers import Field
-from ctapipe.core import Component
 from ctapipe.core.traits import ComponentNameList, Integer
 from ctapipe.io import HDF5TableReader
 from ctapipe_io_nectarcam import constants
 from ctapipe_io_nectarcam.containers import NectarCAMDataContainer
 
-from nectarchain.data.container import (
-    ArrayDataContainer,
-    NectarCAMContainer,
-    TriggerMapContainer,
-)
+from nectarchain.data.container import NectarCAMContainer
 from nectarchain.makers import EventsLoopNectarCAMCalibrationTool
-from nectarchain.makers.component import ArrayDataComponent, NectarCAMComponent
-from nectarchain.utils import ComponentUtils
+from nectarchain.makers.component import NectarCAMComponent
 
 # %%
 tools = EventsLoopNectarCAMCalibrationTool()
@@ -69,9 +74,10 @@
 
 
 # %% [markdown]
-# The only thing to add to to fill the componentList field, which contains the names of the component to be apply on events.
+# The only thing to add is to fill the componentList field, which contains
+# the names of the component to be applied on events.
 #
-# Then we will define a very simple component to compute the pedestal of each events.
+# Then we will define a very simple component to compute the pedestal of each event.
 
 # %% [markdown]
 # ### Definition of container to store extracted data on disk
@@ -124,7 +130,8 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs):
         super().__init__(
             subarray=subarray, config=config, parent=parent, *args, **kwargs
         )
-        ## If you want you can add here members of MyComp, they will contain interesting quantity during the event loop process
+        # If you want you can add here members of MyComp, they will contain interesting
+        # quantity during the event loop process
 
         self.__ucts_timestamp = []
         self.__event_type = []
@@ -133,17 +140,23 @@ def __init__(self, subarray, config=None, parent=None, *args, **kwargs):
         self.__pedestal_hg = []
         self.__pedestal_lg = []
 
-    ##This method need to be defined !
+    # This method need to be defined !
     def __call__(self, event: NectarCAMDataContainer, *args, **kwargs):
         self.__event_id.append(np.uint32(event.index.event_id))
         self.__event_type.append(event.trigger.event_type.value)
-        self.__ucts_timestamp.append(event.nectarcam.tel[0].evt.ucts_timestamp)
+        self.__ucts_timestamp.append(
+            event.nectarcam.tel[self.tel_id].evt.ucts_timestamp
+        )
 
         wfs = []
-        wfs.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id])
-        wfs.append(event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id])
+        wfs.append(
+            event.r0.tel[self.tel_id].waveform[constants.HIGH_GAIN][self.pixels_id]
+        )
+        wfs.append(
+            event.r0.tel[self.tel_id].waveform[constants.LOW_GAIN][self.pixels_id]
+        )
 
-        #####THE JOB IS HERE######
+        # The main work is here !
         for i, pedestal in enumerate([self.__pedestal_hg, self.__pedestal_lg]):
             index_peak = np.argmax(wfs[i])
             signal_start = index_peak - self.window_shift
@@ -162,7 +175,7 @@ def __call__(self, event: NectarCAMDataContainer, *args, **kwargs):
                 / (constants.N_SAMPLES - self.window_width)
             )
 
-    ##This method need to be defined !
+    # This method need to be defined !
     def finish(self):
         output = MyContainer(
             run_number=MyContainer.fields["run_number"].type(self._run_number),
@@ -187,7 +200,8 @@ def finish(self):
 # ### Definition of our Tool
 
 # %% [markdown]
-# Now we can define out Tool, we have just to add our component "MyComp" in the ComponentList :
+# Now we can define out Tool, we have just to add our component "MyComp"
+# in the ComponentList:
 
 
 # %%
@@ -243,13 +257,15 @@ def _init_output_path(self):
 tool.setup()
 
 # %% [markdown]
-# The following command will just start the tool and apply components looping over events
+# The following command will just start the tool and apply components
+# looping over events.
 
 # %%
 tool.start()
 
 # %% [markdown]
-# Then, we finish the tool, behind thius command the component will be finilized and will create an output container whiich will be written on disk and can be returned
+# Then, we finish the tool. Behind this command the component will be finalized and will
+# create an output container which will be written on disk and can be returned
 
 # %%
 output = tool.finish(return_output_component=True)[0]
@@ -290,7 +306,8 @@ def _init_output_path(self):
 # %%
 container_loaded = next(
     MyContainer._container_from_hdf5(
-        f"{os.environ.get('NECTARCAMDATA','/tmp')}/tutorials/PedestalTutoNectarCAM_run4943_maxevents500.h5",
+        f"{os.environ.get('NECTARCAMDATA','/tmp')}/tutorials/PedestalTutoNectarCAM_"
+        f"run4943_maxevents500.h5",
         MyContainer,
     )
 )
@@ -301,8 +318,13 @@ def _init_output_path(self):
 # ## Going further
 
 # %% [markdown]
-# An argument that are implemented in EventsLoopNectarCAMCalibrationTool is the 'event_per_slice', this argument allows to split all the events within the raw data fits.fz file in slices. It allows to, for each slice, loop over events and write container on disk. This mechanism allows to save RAM.
-# The resulting hdf5 file that is written on disk , can be easily loaded thereafter. There is only one hdf5 file for the whole run, which is a mapping between slices and containers filled by computed quantity from components.
+# An argument that are implemented in EventsLoopNectarCAMCalibrationTool is
+# `event_per_slice`. This argument allows to split all the events within the raw
+# data fits.fz file in slices. It allows, for each slice, to loop over events and
+# write container on disk. This mechanism allows to save RAM.
+# The resulting HDF5 file that is written on disk can be easily loaded thereafter.
+# There is only one HDF5 file for the whole run, which is a mapping between slices
+# and containers filled by computed quantity from components.
 
 # %%
 tool = MyTool(
@@ -331,7 +353,9 @@ def _init_output_path(self):
 # !h5ls -r $NECTARCAMDATA/tutorials/PedestalTutoNectarCAM_run4943_maxevents2000.h5
 
 # %%
-# container_loaded = ArrayDataContainer._container_from_hdf5(f"{os.environ.get('NECTARCAMDATA','/tmp')}/tutorials/PedestalTutoNectarCAM_run4943_maxevents2000.h5",MyContainer)
+# container_loaded = ArrayDataContainer._container_from_hdf5(
+#     f"{os.environ.get('NECTARCAMDATA','/tmp')}/tutorials/PedestalTutoNectarCAM_"
+#     f"run4943_maxevents2000.h5",MyContainer)
 # container_loaded
 
 
@@ -354,11 +378,13 @@ def read_hdf5_sliced(path):
 
 # %%
 container_loaded = read_hdf5_sliced(
-    f"{os.environ.get('NECTARCAMDATA','/tmp')}/tutorials/PedestalTutoNectarCAM_run4943_maxevents2000.h5"
+    f"{os.environ.get('NECTARCAMDATA','/tmp')}/tutorials/PedestalTutoNectarCAM_"
+    f"run4943_maxevents2000.h5"
 )
 for i, container in enumerate(container_loaded):
     print(
-        f"Container {i} is filled by events from {container.event_id[0]} to {container.event_id[-1]}"
+        f"Container {i} is filled by events from {container.event_id[0]} to"
+        f"{container.event_id[-1]}"
     )
 
 # %%