DQM Bokeh time series (#227)

* New attempt to add timelines to Bokeh app

* Follow Bokeh API recent changes for tab panels

* Play with layouts.
Timelines still not updated on run change, added runid in timelines titles to help to debug it

* Play with update timelines (not yet working)

* Include timelines in Bokeh DQM app, now working as expected !

Author: jlenain

src/nectarchain/dqm/bokeh_app/app_hooks.py

@@ -2,6 +2,13 @@
 import re
 
 import numpy as np
+
+# bokeh imports
+from bokeh.layouts import gridplot
+from bokeh.models import TabPanel
+from bokeh.plotting import figure
+
+# ctapipe imports
 from ctapipe.coordinates import EngineeringCameraFrame
 from ctapipe.instrument import CameraGeometry
 
@@ -27,18 +34,87 @@ def get_rundata(src, runid):
     return run_data
 
 
-def make_camera_displays(db, source, runid):
+def make_timelines(source, runid=None):
+    timelines = collections.defaultdict(dict)
+    for parentkey in source.keys():
+        if re.match("(?:.*PIXTIMELINE-.*)", parentkey):
+            for childkey in source[parentkey].keys():
+                print(f"Run id {runid} Preparing plot for {parentkey}, {childkey}")
+                timelines[parentkey][childkey] = figure(title=childkey)
+                evts = np.arange(len(source[parentkey][childkey]))
+                timelines[parentkey][childkey].line(
+                    x=evts,
+                    y=source[parentkey][childkey],
+                    line_width=3,
+                )
+    return dict(timelines)
+
+
+def update_timelines(data, timelines, runid=None):
+    # Reset each timeline
+    for k in timelines.keys():
+        for kk in timelines[k].keys():
+            timelines[k][kk].line(x=0, y=0)
+
+    timelines = make_timelines(data, runid)
+
+    list_timelines = [
+        timelines[parentkey][childkey]
+        for parentkey in timelines.keys()
+        for childkey in timelines[parentkey].keys()
+    ]
+
+    layout_timelines = gridplot(
+        list_timelines,
+        ncols=2,
+    )
+
+    tab_timelines = TabPanel(child=layout_timelines, title="Timelines")
+
+    return tab_timelines
+
+
+def make_camera_displays(source, runid):
     displays = collections.defaultdict(dict)
-    for parentkey in db[runid].keys():
+    for parentkey in source.keys():
         if not re.match(TEST_PATTERN, parentkey):
-            for childkey in db[runid][parentkey].keys():
+            for childkey in source[parentkey].keys():
                 print(f"Run id {runid} Preparing plot for {parentkey}, {childkey}")
                 displays[parentkey][childkey] = make_camera_display(
                     source, parent_key=parentkey, child_key=childkey
                 )
     return dict(displays)
 
 
+def update_camera_displays(data, displays, runid=None):
+    ncols = 3
+
+    # Reset each display
+    for k in displays.keys():
+        for kk in displays[k].keys():
+            displays[k][kk].image = np.zeros(shape=constants.N_PIXELS)
+
+    displays = make_camera_displays(data, runid)
+
+    camera_displays = [
+        displays[parentkey][childkey].figure
+        for parentkey in displays.keys()
+        for childkey in displays[parentkey].keys()
+    ]
+
+    layout_camera_displays = gridplot(
+        camera_displays,
+        sizing_mode="scale_width",
+        ncols=ncols,
+    )
+
+    tab_camera_displays = TabPanel(
+        child=layout_camera_displays, title="Camera displays"
+    )
+
+    return tab_camera_displays
+
+
 def make_camera_display(source, parent_key, child_key):
     # Example camera display
     image = source[parent_key][child_key]

src/nectarchain/dqm/bokeh_app/main.py

@@ -1,59 +1,40 @@
-import re
-
-import numpy as np
-from app_hooks import TEST_PATTERN, get_rundata, make_camera_displays
+from app_hooks import (
+    get_rundata,
+    make_camera_displays,
+    make_timelines,
+    update_camera_displays,
+    update_timelines,
+)
 
 # bokeh imports
-from bokeh.layouts import layout, row
-from bokeh.models import Select  # , NumericInput
+from bokeh.layouts import column, gridplot, row
+from bokeh.models import Select, TabPanel, Tabs
 from bokeh.plotting import curdoc
 
 # ctapipe imports
 from ctapipe.coordinates import EngineeringCameraFrame
 from ctapipe.instrument import CameraGeometry
-from ctapipe_io_nectarcam import constants
 
 from nectarchain.dqm.db_utils import DQMDB
 
 geom = CameraGeometry.from_name("NectarCam-003")
 geom = geom.transform_to(EngineeringCameraFrame())
 
 
-def update_camera_displays(attr, old, new):
+def update(attr, old, new):
     runid = run_select.value
-    new_rundata = get_rundata(db, runid)
-
-    # Reset each display
-    for k in displays.keys():
-        for kk in displays[k].keys():
-            displays[k][kk].image = np.zeros(shape=constants.N_PIXELS)
-
-    for parentkey in db[runid].keys():
-        if not re.match(TEST_PATTERN, parentkey):
-            for childkey in db[runid][parentkey].keys():
-                print(f"Run id {runid} Updating plot for {parentkey}, {childkey}")
-
-                image = new_rundata[parentkey][childkey]
-                image = np.nan_to_num(image, nan=0.0)
-                try:
-                    displays[parentkey][childkey].image = image
-                except ValueError as e:
-                    print(
-                        f"Caught {type(e).__name__} for {childkey}, filling display"
-                        f"with zeros. Details: {e}"
-                    )
-                    image = np.zeros(shape=displays[parentkey][childkey].image.shape)
-                    displays[parentkey][childkey].image = image
-                except KeyError as e:
-                    print(
-                        f"Caught {type(e).__name__} for {childkey}, filling display"
-                        f"with zeros. Details: {e}"
-                    )
-                    image = np.zeros(shape=constants.N_PIXELS)
-                    displays[parentkey][childkey].image = image
-                # TODO: TRY TO USE `stream` INSTEAD, ON UPDATES:
-                # display.datasource.stream(new_data)
-                # displays[parentkey][childkey].datasource.stream(image)
+    source = get_rundata(db, runid)
+
+    tab_camera_displays = update_camera_displays(source, displays, runid)
+    tab_timelines = update_timelines(source, timelines, runid)
+
+    # Combine panels into tabs
+    tabs = Tabs(
+        tabs=[tab_camera_displays, tab_timelines],
+        sizing_mode="scale_width",
+    )
+
+    page_layout.children[1] = tabs
 
 
 print("Opening connection to ZODB")
@@ -71,13 +52,13 @@ def update_camera_displays(attr, old, new):
 
 print("Defining Select")
 # runid_input = NumericInput(value=db.root.keys()[-1], title="NectarCAM run number")
+# run_select = Select(value=runid, title="NectarCAM run number", options=runids)
 run_select = Select(value=runid, title="NectarCAM run number", options=runids)
 
 print(f"Getting data for run {run_select.value}")
 source = get_rundata(db, run_select.value)
-displays = make_camera_displays(db, source, runid)
-
-run_select.on_change("value", update_camera_displays)
+displays = make_camera_displays(source, runid)
+timelines = make_timelines(source, runid)
 
 controls = row(run_select)
 
@@ -88,15 +69,40 @@ def update_camera_displays(attr, old, new):
 # update_camera_displays(attr, old, new)
 
 ncols = 3
-plots = [
+camera_displays = [
     displays[parentkey][childkey].figure
     for parentkey in displays.keys()
     for childkey in displays[parentkey].keys()
 ]
-curdoc().add_root(
-    layout(
-        [[controls], [[plots[x : x + ncols] for x in range(0, len(plots), ncols)]]],
-        sizing_mode="scale_width",
-    )
+list_timelines = [
+    timelines[parentkey][childkey]
+    for parentkey in timelines.keys()
+    for childkey in timelines[parentkey].keys()
+]
+
+layout_camera_displays = gridplot(
+    camera_displays,
+    ncols=ncols,
+)
+
+layout_timelines = gridplot(
+    list_timelines,
+    ncols=2,
+)
+
+# Create different tabs
+tab_camera_displays = TabPanel(child=layout_camera_displays, title="Camera displays")
+tab_timelines = TabPanel(child=layout_timelines, title="Timelines")
+
+# Combine panels into tabs
+tabs = Tabs(
+    tabs=[tab_camera_displays, tab_timelines],
 )
+
+page_layout = column([controls, tabs], sizing_mode="scale_width")
+
+run_select.on_change("value", update)
+
+# Add to the Bokeh document
+curdoc().add_root(page_layout)
 curdoc().title = "NectarCAM Data Quality Monitoring web app"

src/nectarchain/dqm/bokeh_app/tests/test_app_hooks.py

@@ -2,8 +2,8 @@
 
 # bokeh imports
 from bokeh.io import output_file, save
-from bokeh.layouts import layout
-from bokeh.models import Select
+from bokeh.layouts import column, gridplot, row
+from bokeh.models import Select, TabPanel, Tabs
 from bokeh.plotting import curdoc
 
 # ctapipe imports
@@ -19,10 +19,12 @@
         "mykey1": {
             "mysubkey1": np.random.normal(size=geom.n_pixels),
             "mysubkey2": np.random.normal(size=geom.n_pixels),
+            "FOOPIXTIMELINE-HIGH": np.random.normal(size=1000),
         },
         "mykey2": {
             "mysubkey1": np.random.normal(size=geom.n_pixels),
             "mysubkey2": np.random.normal(size=geom.n_pixels),
+            "FOOPIXTIMELINE-HIGH": np.random.normal(size=1000),
         },
     }
 }
@@ -34,11 +36,22 @@ def test_make_camera_displays():
     from nectarchain.dqm.bokeh_app.app_hooks import make_camera_displays
 
     for runid in list(test_dict.keys()):
-        make_camera_displays(test_dict, test_dict[runid], runid)
+        make_camera_displays(source=test_dict[runid], runid=runid)
+
+
+def test_make_timelines():
+    from nectarchain.dqm.bokeh_app.app_hooks import make_timelines
+
+    for runid in list(test_dict.keys()):
+        make_timelines(source=test_dict[runid], runid=runid)
 
 
 def test_bokeh(tmp_path):
-    from nectarchain.dqm.bokeh_app.app_hooks import get_rundata, make_camera_displays
+    from nectarchain.dqm.bokeh_app.app_hooks import (
+        get_rundata,
+        make_camera_displays,
+        make_timelines,
+    )
 
     db = DB(None)
     conn = db.open()
@@ -53,20 +66,46 @@ def test_bokeh(tmp_path):
     run_select = Select(value=runid, title="NectarCAM run number", options=runids)
 
     source = get_rundata(root, run_select.value)
-    displays = make_camera_displays(root, source, runid)
+    displays = make_camera_displays(source=source, runid=runid)
+    timelines = make_timelines(source, runid)
 
     ncols = 3
-    plots = [
+    camera_displays = [
         displays[parentkey][childkey].figure
         for parentkey in displays.keys()
         for childkey in displays[parentkey].keys()
     ]
-    curdoc().add_root(
-        layout(
-            [[[plots[x : x + ncols] for x in range(0, len(plots), ncols)]]],
-            sizing_mode="scale_width",
-        )
+    list_timelines = [
+        timelines[parentkey][childkey]
+        for parentkey in timelines.keys()
+        for childkey in timelines[parentkey].keys()
+    ]
+
+    layout_camera_displays = gridplot(
+        camera_displays,
+        ncols=ncols,
     )
+
+    layout_timelines = gridplot(
+        list_timelines,
+        ncols=2,
+    )
+    # Create different tabs
+    tab_camera_displays = TabPanel(
+        child=layout_camera_displays, title="Camera displays"
+    )
+    tab_timelines = TabPanel(child=layout_timelines, title="Timelines")
+
+    # Combine panels into tabs
+    tabs = Tabs(
+        tabs=[tab_camera_displays, tab_timelines],
+    )
+
+    controls = row(run_select)
+
+    page_layout = column([controls, tabs], sizing_mode="scale_width")
+
+    curdoc().add_root(page_layout)
     curdoc().title = "NectarCAM Data Quality Monitoring web app"
 
     output_path = tmp_path / "test.html"