jupyter investigation to close any open h5 file handles so not to reuse old data

Author: PhilipDeegan

pyphare/pyphare/core/ipython.py

@@ -0,0 +1,24 @@
+
+# https://stackoverflow.com/a/40222538/795574
+
+# exit_register runs at the end of ipython %run or the end of the python interpreter
+try:
+    def exit_register(fun, *args, **kwargs):
+        """ Decorator that registers at post_execute. After its execution it
+        unregisters itself for subsequent runs. """
+        def callback():
+            fun()
+            ip.events.unregister('post_execute', callback)
+        ip.events.register('post_execute', callback)
+
+
+    ip = get_ipython()
+except NameError:
+    from atexit import register as exit_register
+
+
+# @exit_register
+# def callback():
+#     print('I\'m done!')
+
+

pyphare/pyphare/core/phare_utilities.py

@@ -115,7 +115,7 @@ def decode_bytes(input, errors="ignore"):
     return input.decode("ascii", errors=errors)
 
 
-def run_cli_cmd(cmd, shell=True, capture_output=True, check=False, print_cmd=True):
+def run_cli_cmd(cmd, shell=True, capture_output=True, check=False, print_cmd=False):
     """
     https://docs.python.org/3/library/subprocess.html
     """

pyphare/pyphare/pharesee/hierarchy.py

@@ -5,12 +5,42 @@
 
 from .particles import Particles
 
+from ..core.ipython import exit_register
 from ..core import box as boxm
 from ..core.box import Box
 from ..core.gridlayout import GridLayout
 import matplotlib.pyplot as plt
 from ..core.phare_utilities import np_array_ify, is_scalar, listify, refinement_ratio
 
+import weakref
+h5_weak_refs = {}
+
+def add_h5_weak_ref(h5File):
+    # tmp = h5_weak_refs.copy()
+    # for object_id, h5_weak_ref in h5_weak_refs:
+    #     if not h5_weak_ref.alive():
+    #         del tmp[object_id]
+    # h5_weak_refs = tmp
+
+    object_id = id(h5File)
+    #h5_weak_refs[object_id] = weakref.ref(h5File)
+    h5_weak_refs[object_id] = h5File
+
+@exit_register
+def close_h5_files():
+    for h5File in list(h5_weak_refs.values()):
+        if h5File.__bool__():
+            # print("closing", h5File)
+            h5File.close()
+
+# @exit_register
+def close_h5_files_refs():
+    for object_id, h5_weak_ref in h5_weak_refs.items():
+        if h5_weak_ref() is not None and h5_weak_ref().__bool__():
+            # print("closing", h5_weak_ref())
+            h5_weak_ref().close()
+
+
 
 class PatchData:
     """
@@ -859,6 +889,7 @@ def patch_has_datasets(h5_patch_grp):
 def hierarchy_fromh5(h5_filename, time, hier, silent=True):
     import h5py
     data_file = h5py.File(h5_filename, "r")
+    add_h5_weak_ref(data_file)
     basename = os.path.basename(h5_filename)
 
     root_cell_width = np.asarray(data_file.attrs["cell_width"])