Merge pull request #84 from kif/83_close_files

83 close files

Author: kif

plugins/bm29/common.py

@@ -11,7 +11,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "03/12/2024"
+__date__ = "20/02/2025"
 __status__ = "development"
 version = "0.0.2"
 
@@ -71,7 +71,9 @@ def get_integrator(keycache):
         ai = pyFAI.load(keycache.poni)
         ai.wavelength = 1e-10 * pyFAI.units.hc / keycache.energy
         if keycache.mask:
-            mask = numpy.logical_or(fabio.open(keycache.mask).data, ai.detector.mask).astype("int8")
+            with fabio.open(keycache.mask) as fimg:
+                fabio_mask =  fimg.data
+            mask = numpy.logical_or(fabio_mask, ai.detector.mask).astype("int8")
             ai.detector.mask = mask
         shared_cache[keycache] = ai
     return ai

plugins/bm29/ispyb.py

@@ -105,16 +105,15 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
         :param path: directory name where processed data are staying
         :param raw: directory name of the raw data (not the processed ones)
         :param data: dict with all data sent to ISpyB
-        
         """
         tmp = self.gallery.strip("/").split("/")
         idx_process = [i for i,j in enumerate(tmp) if j.lower().startswith("process")][-1]
-        if tmp[idx_process] == "process":
-            assert idx_process>5
+        if tmp[idx_process] == "processed":
+            assert idx_process>=6
             if proposal is None:
-                proposal = tmp[idx_process-5]
+                proposal = tmp[idx_process-6]
             if beamline is None:
-                beamline = tmp[idx_process-4]
+                beamline = tmp[idx_process-5]
             if sample is None:
                 sample = tmp[idx_process-2]
             if dataset is None:

plugins/bm29/nexus.py

@@ -1,10 +1,10 @@
 """Module for writing HDF5 in the Nexus style"""
 
-__author__ = "Jerome Kieffer"
+__author__ = "Jérôme Kieffer"
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "21/04/2022"
+__date__ = "20/02/2025"
 __status__ = "production"
 __docformat__ = 'restructuredtext'
 
@@ -13,6 +13,7 @@
 import time
 import logging
 import h5py
+import atexit
 logger = logging.getLogger(__name__)
 
 
@@ -45,7 +46,7 @@ def from_isotime(text, use_tz=False):
         text = str(text)
     if len(text) < 19:
         logger.warning("Not a iso-time string: %s", text)
-        return None
+        return
     base = text[:19]
     if use_tz and len(text) == 25:
         sgn = 1 if text[:19] == "+" else -1
@@ -85,14 +86,18 @@ class Nexus:
     TODO: make it thread-safe !!!
     """
 
-    def __init__(self, filename, mode=None, creator=None, timeout=None):
+    def __init__(self, filename, mode=None, 
+                 creator=None, 
+                 timeout=None,
+                 start_time=None):
         """
         Constructor
 
         :param filename: name of the hdf5 file containing the nexus
         :param mode: can be 'r', 'a', 'w', '+' ....
         :param creator: set as attr of the NXroot
         :param timeout: retry for that amount of time (in seconds)
+        :param start_time: set as attr of the NXroot
         """
         self.filename = os.path.abspath(filename)
         self.mode = mode
@@ -132,10 +137,10 @@ def __init__(self, filename, mode=None, creator=None, timeout=None):
                 self.file_handle = None
                 self.h5 = h5py.File(self.filename, mode=self.mode)
         self.to_close = []
-
+        atexit.register(self.close)
         if not pre_existing:
             self.h5.attrs["NX_class"] = "NXroot"
-            self.h5.attrs["file_time"] = get_isotime()
+            self.h5.attrs["file_time"] = get_isotime(start_time)
             self.h5.attrs["file_name"] = self.filename
             self.h5.attrs["HDF5_Version"] = h5py.version.hdf5_version
             self.h5.attrs["creator"] = creator or self.__class__.__name__
@@ -178,8 +183,7 @@ def get_entry(self, name):
                 if isinstance(grp, h5py.Group) and \
                    ("start_time" in grp) and  \
                    self.get_attr(grp, "NX_class") == "NXentry":
-                    return grp
-        return None
+                        return grp
 
     def get_entries(self):
         """
@@ -212,7 +216,8 @@ def find_detector(self, all=False):
         return result
 
     def new_entry(self, entry="entry", program_name="pyFAI",
-                  title=None, force_time=None, force_name=False):
+                  title="description of experiment",
+                  force_time=None, force_name=False):
         """
         Create a new entry
 

plugins/id02/nexus.py

@@ -4,7 +4,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "29/01/2024"
+__date__ = "20/02/2025"
 __status__ = "production"
 __docformat__ = 'restructuredtext'
 
@@ -13,6 +13,7 @@
 import time
 import logging
 import h5py
+import atexit
 logger = logging.getLogger(__name__)
 
 
@@ -70,7 +71,7 @@ def is_hdf5(filename):
     return sig == signature
 
 
-class Nexus(object):
+class Nexus:
     """
     Writer class to handle Nexus/HDF5 data
 
@@ -85,13 +86,17 @@ class Nexus(object):
     TODO: make it thread-safe !!!
     """
 
-    def __init__(self, filename, mode=None, creator=None, start_time=None):
+    def __init__(self, filename, mode=None, 
+                 creator=None, 
+                 timeout=None,
+                 start_time=None):
         """
         Constructor
 
         :param filename: name of the hdf5 file containing the nexus
         :param mode: can be 'r', 'a', 'w', '+' ....
         :param creator: set as attr of the NXroot
+        :param timeout: retry for that amount of time (in seconds)
         :param start_time: set as attr of the NXroot
         """
         self.filename = os.path.abspath(filename)
@@ -107,15 +112,33 @@ def __init__(self, filename, mode=None, creator=None, start_time=None):
             else:
                 self.mode = "a"
 
-        if self.mode == "r" and h5py.version.version_tuple >= (2, 9):
-            self.file_handle = open(self.filename, mode=self.mode + "b")
-            self.h5 = h5py.File(self.file_handle, mode=self.mode)
+        if timeout:
+            end = time.perf_counter() + timeout
+            while time.perf_counter() < end :
+                try:
+                    if self.mode == "r":
+                        self.file_handle = open(self.filename, mode="rb")
+                        self.h5 = h5py.File(self.file_handle, mode="r")
+                    else:
+                        self.file_handle = None
+                        self.h5 = h5py.File(self.filename, mode=self.mode)
+                except OSError:
+                    os.stat(os.path.dirname(self.filename))
+                    time.sleep(1)
+                else:
+                    break
+            else:
+                raise OSError(f"Unable to open HDF5 file {self.filename}")
         else:
-            self.file_handle = None
-            self.h5 = h5py.File(self.filename, mode=self.mode)
+            if self.mode == "r":
+                self.file_handle = open(self.filename, mode=self.mode + "b")
+                self.h5 = h5py.File(self.file_handle, mode=self.mode)
+            else:
+                self.file_handle = None
+                self.h5 = h5py.File(self.filename, mode=self.mode)
         self.to_close = []
-
-        if not pre_existing or "w" in mode:
+        atexit.register(self.close)
+        if not pre_existing:
             self.h5.attrs["NX_class"] = "NXroot"
             self.h5.attrs["file_time"] = get_isotime(start_time)
             self.h5.attrs["file_name"] = self.filename
@@ -143,6 +166,7 @@ def __exit__(self, *arg, **kwarg):
         self.close()
 
     def flush(self):
+        "write to disk"
         if self.h5:
             self.h5.flush()
 
@@ -199,7 +223,7 @@ def new_entry(self, entry="entry", program_name="pyFAI",
 
         :param entry: name of the entry
         :param program_name: value of the field as string
-        :param title: value of the field as string
+        :param title: description of experiment as str
         :param force_time: enforce the start_time (as string!)
         :param force_name: force the entry name as such, without numerical suffix.
         :return: the corresponding HDF5 group
@@ -211,7 +235,7 @@ def new_entry(self, entry="entry", program_name="pyFAI",
         entry_grp = self.h5.require_group(entry)
         self.h5.attrs["default"] = entry
         entry_grp.attrs["NX_class"] = "NXentry"
-        entry_grp["title"] = title
+        entry_grp["title"] = str(title)
         entry_grp["program_name"] = program_name
         if isinstance(force_time, str):
             entry_grp["start_time"] = force_time
@@ -230,16 +254,18 @@ def new_instrument(self, entry="entry", instrument_name="id00",):
 #        howto external link
         # myfile['ext link'] = h5py.ExternalLink("otherfile.hdf5", "/path/to/resource")
 
-    def new_class(self, grp, name, class_type="NXcollection"):
+    @staticmethod
+    def new_class(grp, name, class_type="NXcollection"):
         """
         create a new sub-group with  type class_type
+        
         :param grp: parent group
         :param name: name of the sub-group
         :param class_type: NeXus class name
         :return: subgroup created
         """
         sub = grp.require_group(name)
-        sub.attrs["NX_class"] = class_type
+        sub.attrs["NX_class"] = str(class_type)
         return sub
 
     def new_detector(self, name="detector", entry="entry", subentry="pyFAI"):
@@ -253,8 +279,8 @@ def new_detector(self, name="detector", entry="entry", subentry="pyFAI"):
         from . import __version__ as version
         entry_grp = self.new_entry(entry)
         pyFAI_grp = self.new_class(entry_grp, subentry, "NXsubentry")
-        pyFAI_grp["definition_local"] = "pyFAI"
-        pyFAI_grp["definition_local"].attrs["version"] = version
+        pyFAI_grp["definition_local"] = str("pyFAI")
+        pyFAI_grp["definition_local"].attrs["version"] = str(version)
         det_grp = self.new_class(pyFAI_grp, name, "NXdetector")
         return det_grp
 
@@ -282,6 +308,21 @@ def get_data(self, grp, attr=None, value=None):
                 self.get_attr(grp[name], attr) == value]
         return coll
 
+    def get_default_NXdata(self):
+        """Return the default plot configured in the nexus structure.
+
+        :return: the group with the default plot or None if not found
+        """
+        entry_name = self.h5.attrs.get("default")
+        if entry_name:
+            entry_grp = self.h5.get(entry_name)
+            nxdata_name = entry_grp.attrs.get("default")
+            if nxdata_name:
+                if nxdata_name.startswith("/"):
+                    return self.h5.get(nxdata_name)
+                return entry_grp.get(nxdata_name)
+        return None
+
     def deep_copy(self, name, obj, where="/", toplevel=None, excluded=None, overwrite=False):
         """
         perform a deep copy:
@@ -300,7 +341,7 @@ def deep_copy(self, name, obj, where="/", toplevel=None, excluded=None, overwrit
             if name not in toplevel:
                 grp = toplevel.require_group(name)
                 for k, v in obj.attrs.items():
-                        grp.attrs[k] = v
+                    grp.attrs[k] = v
         elif isinstance(obj, h5py.Dataset):
             if name in toplevel:
                 if overwrite:

plugins/id02/single_detector.py

@@ -8,7 +8,7 @@
 __contact__ = "Jerome.Kieffer@ESRF.eu"
 __license__ = "MIT"
 __copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
-__date__ = "14/02/2024"
+__date__ = "20/02/2025"
 __status__ = "development"
 __version__ = "0.9.3"
 
@@ -371,20 +371,20 @@ def process(self):
             if self.flat_filename.endswith(".h5") or self.flat_filename.endswith(".nxs") or self.flat_filename.endswith(".hdf5"):
                 flat = self.read_data(self.flat_filename)
             else:
-                flat_fabio = fabio.open(self.flat_filename)
-                flat = flat_fabio.data
-                dummy = flat_fabio.header.get("Dummy")
-                try:
-                    dummy = float(dummy)
-                except:
-                    self.log_error("Dummy value in mask is unconsistent %s" % dummy)
-                    dummy = None
-                ddummy = flat_fabio.header.get("DDummy")
-                try:
-                    ddummy = float(ddummy)
-                except:
-                    self.log_error("DDummy value in mask is unconsitent %s" % ddummy)
-                    ddummy = 0
+                with fabio.open(self.flat_filename) as flat_fabio:
+                    flat = flat_fabio.data
+                    dummy = flat_fabio.header.get("Dummy")
+                    try:
+                        dummy = float(dummy)
+                    except:
+                        self.log_error("Dummy value in mask is unconsistent %s" % dummy)
+                        dummy = None
+                    ddummy = flat_fabio.header.get("DDummy")
+                    try:
+                        ddummy = float(ddummy)
+                    except:
+                        self.log_error("DDummy value in mask is unconsitent %s" % ddummy)
+                        ddummy = 0
 
             if flat.ndim == 3:
                 self.flat = pyFAI.average.average_dark(flat, center_method="median")
@@ -415,28 +415,27 @@ def process(self):
         self.mask_filename = self.input.get("regrouping_mask_filename")
         if isinstance(self.mask_filename, StringTypes) and os.path.exists(self.mask_filename):
             try:
-                mask_fabio = fabio.open(self.mask_filename)
+                with fabio.open(self.mask_filename) as mask_fabio:
+                    dummy = mask_fabio.header.get("Dummy")
+                    try:
+                        dummy = float(dummy)
+                    except:
+                        self.log_error("Dummy value in mask is unconsitent %s" % dummy)
+                        dummy = None
+                    ddummy = mask_fabio.header.get("DDummy")
+                    try:
+                        ddummy = float(ddummy)
+                    except:
+                        self.log_error("DDummy value in mask is unconsitent %s" % ddummy)
+                        ddummy = 0
+                    if ddummy:
+                        local_mask = abs(mask_fabio.data - dummy) <= ddummy
+                    else:
+                        local_mask = mask_fabio.data == dummy
+                    self.dummy = dummy
+                    self.delta_dummy = ddummy
             except:
                 local_mask = self.read_data(self.mask_filename) != 0
-            else:  # this is very ID02 specific !!!!
-                dummy = mask_fabio.header.get("Dummy")
-                try:
-                    dummy = float(dummy)
-                except:
-                    self.log_error("Dummy value in mask is unconsitent %s" % dummy)
-                    dummy = None
-                ddummy = mask_fabio.header.get("DDummy")
-                try:
-                    ddummy = float(ddummy)
-                except:
-                    self.log_error("DDummy value in mask is unconsitent %s" % ddummy)
-                    ddummy = 0
-                if ddummy:
-                    local_mask = abs(mask_fabio.data - dummy) <= ddummy
-                else:
-                    local_mask = mask_fabio.data == dummy
-                self.dummy = dummy
-                self.delta_dummy = ddummy
             if local_mask.ndim == 3:
                 local_mask = pyFAI.average.average_dark(local_mask, center_method="median")
             if (local_mask is not None) and (local_mask.shape != shape):