Merge branch 'develop' into 'master'

version 1.2.1

See merge request eco/eo/lime_tbx!84

Author: javgat

CHANGELOG.md

@@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 [//]: # "## [unreleased] - yyyy-mm-dd"
 
+## [1.2.1] - 2025-07-04
+
+### Added
+
+- Added a disclaimer during comparison processing to inform users that,
+  if the operation is taking too long, they can improve performance by
+  disabling uncertainty computation.
+
+
+### Changed
+
+- Optimised observation file loading on the comparison page, achieving 1.25x to 1.5x faster performance.
+
 ## [1.2.0] - 2025-06-20
 
 ### Added
@@ -85,8 +98,8 @@ calculations compared to v1.0.3. (**NFR306**)
 
 Initial version that serves as the baseline for tracking changes in the change log.
 
-
-[unreleased]: https://github.com/LIME-ESA/lime_tbx/compare/v1.2.0...HEAD
+[unreleased]: https://github.com/LIME-ESA/lime_tbx/compare/v1.2.1...HEAD
+[1.2.1]: https://github.com/LIME-ESA/lime_tbx/compare/v1.2.0...1.2.1
 [1.2.0]: https://github.com/LIME-ESA/lime_tbx/compare/v1.1.0...v1.2.0
 [1.1.0]: https://github.com/LIME-ESA/lime_tbx/compare/v1.0.3...v1.1.0
 [1.0.3]: https://github.com/LIME-ESA/lime_tbx/releases/tag/v1.0.3

README.md

@@ -1,4 +1,4 @@
-[![Version 1.2.0](https://img.shields.io/badge/version-1.2.0-informational?style=for-the-badge)](https://github.com/LIME-ESA/lime_tbx/tags)
+[![Version 1.2.1](https://img.shields.io/badge/version-1.2.1-informational?style=for-the-badge)](https://github.com/LIME-ESA/lime_tbx/tags)
 [![License: LGPL 3.0][license-shield]][license-url]
 [![Issues][issues-shield]][issues-url]
 [![Stargazers][stars-shield]][stars-url]

deployment/installer/build_mac_installer.sh

@@ -5,5 +5,5 @@ cp -r ../../dist/LimeTBX.app mac/bundle/LimeTBX.app
 cp -r ../../kernels mac/bundle/LimeTBX.app/Contents/Resources/kernels
 cp -r ../../eocfi_data mac/bundle/LimeTBX.app/Contents/Resources/eocfi_data
 cp -r ../../coeff_data mac/bundle/LimeTBX.app/Contents/Resources/coeff_data
-version='1.2.0'
+version='1.2.1'
 pkgbuild --install-location /Applications --root mac/bundle --identifier 'int.esa.LimeTBX' --scripts mac/scripts --version $version --component-plist mac/Components.plist mac/lime.pkg

deployment/installer/debian/build_deb.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 echo "Building lime .deb package."
-# 1.2.0 Comment so the search for the version a.b.c appears (here it is a.b-c)
-version="1.2-0"
-# 1.2.0 Change it in the control file too!
+# 1.2.1 Comment so the search for the version a.b.c appears (here it is a.b-c)
+version="1.2-1"
+# 1.2.1 Change it in the control file too!
 name="lime_"$version
 [ -d $name ] && rm -rf $name
 [ -f $name".deb" ] && rm -f $name".deb"

deployment/installer/debian/control

@@ -1,5 +1,5 @@
 Package: lime
-Version: 1.2-0
+Version: 1.2-1
 Section: base
 Priority: optional
 Architecture: amd64

deployment/installer/windows/inno_installer_builder.iss

@@ -2,7 +2,7 @@
 ; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
 
 #define MyAppName "LimeTBX"
-#define MyAppVersion "1.2.0"
+#define MyAppVersion "1.2.1"
 #define MyAppPublisher "European Space Agency"
 #define MyAppExeName "LimeTBX.exe"
 #define MyAppDevPath "C:\repo\"

lime_tbx/application/filedata/moon.py

@@ -214,53 +214,51 @@ def read_moon_obs(
         Generated MoonObservation from the given datafile
     """
     try:
-        ds = xr_open_dataset(path, mask_limits={"sat_pos": False})
+        ds = xr_open_dataset(path, mask_limits={"irr_obs": True, "__default__": False})
         _validate_schema_regular_moonobs(ds)
-        n_channels = len(ds["channel_name"])
-        ch_names = []
-        ch_irrs = {}
-        for i in range(n_channels):
-            ch_name = str(ds["channel_name"][i].values, "utf-8").rstrip("\x00")
-            ch_names.append(ch_name)
+        ch_names = np.char.rstrip(
+            np.char.decode(ds["channel_name"].to_numpy(), "utf-8"), "\x00"
+        )
         dt = ds["date"].values[0]
         if np.issubdtype(dt.dtype, np.datetime64):
-            dt = dt.astype("datetime64[us]").astype(datetime)
-            dt = dt.replace(tzinfo=timezone.utc)
+            dt = (
+                dt.astype("datetime64[us]")
+                .astype(datetime)
+                .replace(tzinfo=timezone.utc)
+            )
         else:
             dt = datetime.fromtimestamp(dt, tz=timezone.utc)
-        sat_pos_ref = ds["sat_pos_ref"].values
-        if str(sat_pos_ref.dtype.str).startswith("|S"):
-            sat_pos_ref = str(sat_pos_ref, "utf-8")
+        sat_pos_ref = ds["sat_pos_ref"].to_numpy()
+        if isinstance(sat_pos_ref, np.ndarray):
+            sat_pos_ref = sat_pos_ref.item()
+        if isinstance(sat_pos_ref, (bytes, bytearray)):
+            sat_pos_ref = sat_pos_ref.decode("utf-8", errors="replace")
         else:
-            sat_pos_ref = sat_pos_ref[0]
-        sat_pos_units: str = ds["sat_pos"].units
-        d_to_m = get_length_conversion_factor(sat_pos_units, "m")
-        to_correct_distance = False
-        if not np.isnan(ds["sat_pos"].values).any():
-            sat_pos = SatellitePosition(*list(ds["sat_pos"].values * d_to_m))
+            sat_pos_ref = str(sat_pos_ref)
+        sat_pos_vals = ds["sat_pos"].to_numpy()
+        has_sat_pos = ~np.isnan(sat_pos_vals).any()
+        if has_sat_pos:
+            d_to_m = get_length_conversion_factor(ds["sat_pos"].units, "m")
+            sat_pos = SatellitePosition(*sat_pos_vals * d_to_m)
             md = None
+            corr_dist = False
         else:
             sat_pos = None
             md = _get_moondata_from_moon_obs(ds, dt, kernels_path, eocfi_path)
-            if "to_correct_distance" in ds.attrs and int(ds.to_correct_distance) == 1:
-                to_correct_distance = True
-        irr_obs = ds["irr_obs"].values
-        irr_obs_units: str = ds["irr_obs"].units
-        d_to_nm = _calc_divisor_to_nm(irr_obs_units)
-        if to_correct_distance:
-            irr_obs = (
-                irr_obs
-                * ((1 / md.distance_sun_moon) ** 2)
-                * (DIST_EARTH_MOON_KM / md.distance_observer_moon) ** 2
-            )
-        irr_obs = irr_obs / d_to_nm
-        for i, ch_irr in enumerate(irr_obs):
-            if not np.isnan(ch_irr):
-                ch_irrs[ch_names[i]] = float(ch_irr)
-        data_source = ds.data_source
+            corr_dist = bool(int(ds.attrs.get("to_correct_distance", 0)))
+        irr_obs = ds["irr_obs"].to_numpy()
+        d_to_nm = _calc_divisor_to_nm(ds["irr_obs"].units)
+        data_source = ds.attrs.get("data_source")
         ds.close()
+        if corr_dist:
+            irr_obs *= (1 / md.distance_sun_moon) ** 2 * (
+                DIST_EARTH_MOON_KM / md.distance_observer_moon
+            ) ** 2
+        irr_obs /= d_to_nm
+        valid = ~np.isnan(irr_obs)
+        ch_irrs = dict(zip(ch_names[valid], irr_obs[valid].astype(float)))
         return LunarObservation(
-            ch_names, sat_pos_ref, ch_irrs, dt, sat_pos, data_source, md
+            list(ch_names), sat_pos_ref, ch_irrs, dt, sat_pos, data_source, md
         )
     except SchemaError as e:
         raise e

lime_tbx/application/filedata/netcdfcommon.py

@@ -139,29 +139,33 @@ def xr_open_dataset(
     mask_limits: bool | Mapping[str, bool]
         If True, replace array values lower than `valid_min` or higher than `valid_max` with `NA`, if present.
         Pass a mapping, e.g. {"my_variable": False}, to toggle this feature per-variable individually.
-        If a mapping is passed and a variable is missing, it's understood that the variable is mapped to True by default.
+        If a mapping is passed and a variable is missing, it's understood that the variable is mapped to the default value.
+        The default value is the one defined for the "__default__" mapping, or True if not specified.
 
     Returns
     -------
     ds: xr.Dataset
         Dataset with the information of the netCDF file.
     """
-    ds = xr.open_dataset(filepath, mask_and_scale=mask_fillvalue)
-    for vname in list(ds.data_vars.keys()):
-        mask_limits_var = False
-        if isinstance(mask_limits, bool):
-            mask_limits_var = mask_limits
-        else:
-            mask_limits_var = vname not in mask_limits or mask_limits[vname]
-        if mask_limits_var:
-            values = ds[vname].values
-            if hasattr(ds[vname], "valid_min"):
-                valid_min = ds[vname].valid_min
-                values = np.where(values >= valid_min, values, np.nan)
-            if hasattr(ds[vname], "valid_max"):
-                valid_max = ds[vname].valid_max
-                values = np.where(values <= valid_max, values, np.nan)
-            ds[vname].values = values
+    ds = xr.open_dataset(filepath, engine="netcdf4", mask_and_scale=mask_fillvalue)
+    if mask_limits is not False:
+        default = True
+        if not isinstance(mask_limits, bool):
+            default = mask_limits.get("__default__", True)
+        for vname, var in ds.data_vars.items():
+            mask_limits_var = False
+            if isinstance(mask_limits, bool):
+                mask_limits_var = mask_limits
+            else:
+                mask_limits_var = mask_limits.get(vname, default)
+            if mask_limits_var:
+                values = var.values
+                mask = np.ones(values.shape, dtype=bool)
+                if hasattr(var, "valid_min"):
+                    mask &= values >= var.valid_min
+                if hasattr(var, "valid_max"):
+                    mask &= values <= var.valid_max
+                var.values = np.where(mask, values, np.nan)
     return ds
 
 

lime_tbx/presentation/gui/maingui.py

@@ -721,14 +721,19 @@ def compare(self):
         )
 
     def compare_info(self, data: str):
+        is_skip = self.settings_manager.is_skip_uncertainties()
         if (
             self.quant_mos_simulated < self.quant_mos - 1
         ):  # -1 So it gives time to the last message to be shown
-            self.spinner.set_text(f"{self.quant_mos_simulated}/{self.quant_mos}")
-            self.quant_mos_simulated += 1
+            text = f"{self.quant_mos_simulated}/{self.quant_mos}"
+            if not is_skip:
+                text += (
+                    "\nIf this is too slow, try deactivating uncertainty computation"
+                )
         else:
-            self.spinner.set_text(f"Finishing comparisons\nand drawing graphs...")
-            self.quant_mos_simulated += 1
+            text = f"Finishing comparisons\nand drawing graphs..."
+        self.spinner.set_text(text)
+        self.quant_mos_simulated += 1
 
     def set_show_comparison_input(self, show: bool):
         self.top_precomp.setVisible(show)
@@ -768,6 +773,11 @@ def clear_comparison_accepted(self):
 
     def _clear_comparison_finished(self, data):
         self.srf = None
+        self.comps = None
+        self.data_source = None
+        self.skipped_uncs = None
+        self.mpa_comps = None
+        self.version = None
         self.set_show_comparison_input(True)
         if not self.compare_by_field.currentText() == CompFields.COMP_DATE:
             self.compare_by_field.setCurrentText(CompFields.COMP_DATE)

lime_tbx/presentation/gui/spinner.py

@@ -25,6 +25,7 @@ def _build_layout(self):
         self.label_spinner = QtWidgets.QLabel()
         self.label_spinner.setMovie(self.movie)
         self.label_text = QtWidgets.QLabel()
+        self.label_text.setAlignment(QtCore.Qt.AlignmentFlag.AlignCenter)
         self.main_layout.addWidget(QtWidgets.QLabel())
         self.main_layout.addWidget(self.label_spinner)
         self.main_layout.addWidget(self.label_text)