DQM Bokeh app (#150)

* Test pattern on parent key, not child one, for rejected camera displays

* Adapt pattern

* More clever way to find the run number with most information to initialize the DQM Bokeh app

* Use OOBTree to write into ZODB. This allows more efficient write access to the DB, and reduces the needed storage by a factor 4!

* Fix first run to be loaded, otherwise the Bokeh app crashes with an Out-of-Memory signal to search the most populated run

---------

Co-authored-by: Jean-Philippe Lenain <[email protected]>

Author: jlenain

src/nectarchain/dqm/bokeh_app/app_hooks.py

@@ -14,7 +14,7 @@
     "PED-INTEGRATION-.*",
     "START-TIMES",
     "WF-.*",
-    ".*PixTimeline-.*",
+    ".*PIXTIMELINE-.*",
 ]
 TEST_PATTERN = "(?:% s)" % "|".join(NOTINDISPLAY)
 
@@ -30,8 +30,8 @@ def get_rundata(src, runid):
 def make_camera_displays(db, source, runid):
     displays = collections.defaultdict(dict)
     for parentkey in db[runid].keys():
-        for childkey in db[runid][parentkey].keys():
-            if not re.match(TEST_PATTERN, childkey):
+        if not re.match(TEST_PATTERN, parentkey):
+            for childkey in db[runid][parentkey].keys():
                 print(f"Run id {runid} Preparing plot for {parentkey}, {childkey}")
                 displays[parentkey][childkey] = make_camera_display(
                     source, parent_key=parentkey, child_key=childkey

src/nectarchain/dqm/bokeh_app/main.py

@@ -29,8 +29,8 @@ def update_camera_displays(attr, old, new):
             displays[k][kk].image = np.zeros(shape=constants.N_PIXELS)
 
     for parentkey in db[runid].keys():
-        for childkey in db[runid][parentkey].keys():
-            if not re.match(TEST_PATTERN, childkey):
+        if not re.match(TEST_PATTERN, parentkey):
+            for childkey in db[runid][parentkey].keys():
                 print(f"Run id {runid} Updating plot for {parentkey}, {childkey}")
 
                 image = new_rundata[parentkey][childkey]
@@ -56,26 +56,24 @@ def update_camera_displays(attr, old, new):
                 # displays[parentkey][childkey].datasource.stream(image)
 
 
+print("Opening connection to ZODB")
 db = DQMDB(read_only=True).root
+print("Getting list of run numbers")
 runids = sorted(list(db.keys()))
 
 # First, get the run id with the most populated result dictionary
-runid_max = runids[-1]
-largest = 0
-for runid in runids:
-    larger = 0
-    for k in db[runid].keys():
-        length = len(db[runid][k])
-        if length > larger:
-            larger = length
-    if larger > largest:
-        largest = larger
-        runid_max = runid
-runid = runid_max
-
+# On the full DB, this takes an awful lot of time, and saturates the RAM on the host
+# VM (gets OoM killed)
+# run_dict_lengths = [len(db[r]) for r in runids]
+# runid = runids[np.argmax(run_dict_lengths)]
+runid = "NectarCAM_Run0008"
+print(f"We will start with run {runid}")
+
+print("Defining Select")
 # runid_input = NumericInput(value=db.root.keys()[-1], title="NectarCAM run number")
 run_select = Select(value=runid, title="NectarCAM run number", options=runids)
 
+print(f"Getting data for run {run_select.value}")
 source = get_rundata(db, run_select.value)
 displays = make_camera_displays(db, source, runid)
 

src/nectarchain/dqm/db_utils.py

@@ -1,4 +1,5 @@
 import transaction
+from BTrees.OOBTree import OOBTree
 from ZEO import ClientStorage
 from ZODB import DB
 
@@ -17,7 +18,7 @@ def __init__(self, read_only=True):
     def insert(self, key=None, value=None):
         if key is not None and value is not None:
             try:
-                self.root[key] = value
+                self.root[key] = OOBTree(value)
                 return True
             except AttributeError:
                 return False