Basetime offset needed for members between normal basetimes (#52)

* Basetime offset needed for members between normal basetimes

* Linting

* Linting

* Adapt to basetime offset

* Basetime must be set

* Linting

Author: trygveasp

pysurfex/file.py

@@ -1186,7 +1186,7 @@ def read(self, var, times):
                                     and self.geo.mask[j] == iii
                                 ):
                                     field2d[i] = np.nan
-                                    if field[tstep, xxx, yyy, patch] != np.nan:
+                                    if not np.isnan(field[tstep, xxx, yyy, patch]):
                                         field2d[i] = field[tstep, xxx, yyy, patch]
                                     i = i + 1
                                     j = j + 1
@@ -1195,7 +1195,7 @@ def read(self, var, times):
                         for yyy in range(field.shape[2]):
                             for xxx in range(field.shape[1]):
                                 field2d[i] = np.nan
-                                if field[tstep, xxx, yyy, patch] != np.nan:
+                                if not np.isnan(field[tstep, xxx, yyy, patch]):
                                     field2d[i] = field[tstep, xxx, yyy, patch]
                                 i = i + 1
                 if i != npoints:

pysurfex/obs.py

@@ -971,11 +971,11 @@ def __init__(self, fname, validtime, varname=None, label="vobs", sigmao=None):
                 lat = odata["lat"]
                 elev = odata["hgt"]
                 obs = np.nan
-                if obname in obsx[stid]:
-                    obs = obsx[stid][obname]
+                if obname in odata:
+                    obs = odata[obname]
                 if obs == -99:
                     obs = np.nan
-                value = obsx[stid][obname]
+                value = odata[obname]
                 if varname is None or varname == obname:
                     observations.append(
                         Observation(

pysurfex/util.py

@@ -24,7 +24,7 @@ def deep_update(source, overrides):
             returned = deep_update(source.get(key, {}), value)
             source[key] = returned
         else:
-            override = overrides[key]
+            override = value
             source[key] = override
     return source
 

pysurfex/variable.py

@@ -31,6 +31,7 @@ def __init__(self, var_type, var_dict, initial_basetime, prefer_forecast=True):
                                               Defaults to True.
 
         Raises:
+            RuntimeError: Initial base time not set
             RuntimeError: No filepattern provided
             RuntimeError: variable must have attribute
             RuntimeError: You can not have larger offset than the frequency of forecasts
@@ -45,6 +46,8 @@ def __init__(self, var_type, var_dict, initial_basetime, prefer_forecast=True):
 
         self.filepattern = self.substitute_macros(var_dict["filepattern"])
         self.initial_basetime = initial_basetime
+        if self.initial_basetime is None:
+            raise RuntimeError("Initial base time not set")
         try:
             self.fcint = int(self.var_dict["fcint"])
         except KeyError:
@@ -70,6 +73,16 @@ def __init__(self, var_type, var_dict, initial_basetime, prefer_forecast=True):
                 + " > "
                 + str(self.fcint)
             )
+        # Calculate basetime offset based on initial basetime and fcint
+        seconds_since_midnight = int(
+            (
+                self.initial_basetime
+                - self.initial_basetime.replace(hour=0, minute=0, second=0, microsecond=0)
+            ).total_seconds()
+        )
+        if seconds_since_midnight == 86400:
+            seconds_since_midnight = 0
+        self.basetime_offset = seconds_since_midnight % self.fcint
         accumulated, instant, file_var = self.set_var(validtime=self.initial_basetime)
         self.file_var = file_var
         self.instant = instant
@@ -559,11 +572,18 @@ def get_basetime(self, validtime, previoustime=None, allow_different_basetime=Fa
         logging.debug("                offset: %s", self.offset)
         logging.debug("      initial_basetime: %s", self.initial_basetime)
         logging.debug("  Basetime with offset: %s", basetime)
+        logging.info("       Basetime offset: %s", self.basetime_offset)
 
         # Modify based on fcint
         seconds_since_midnight = int(
             (
-                basetime - basetime.replace(hour=0, minute=0, second=0, microsecond=0)
+                basetime
+                - basetime.replace(
+                    hour=int(self.basetime_offset / 3600),
+                    minute=0,
+                    second=0,
+                    microsecond=0,
+                )
             ).total_seconds()
         )
         if seconds_since_midnight == 86400:
@@ -586,7 +606,9 @@ def get_basetime(self, validtime, previoustime=None, allow_different_basetime=Fa
 
         fcint = as_timedelta(seconds=self.fcint)
         basetime = (
-            basetime.replace(hour=0, minute=0, second=0, microsecond=0)
+            basetime.replace(
+                hour=int(self.basetime_offset / 3600), minute=0, second=0, microsecond=0
+            )
             + (basetime_inc * fcint)
             - prefer_forecast
         )

tests/smoke/test_cli_arguments.py

@@ -25,7 +25,16 @@ def parse_args(argv):
 
 
 def test_single_parameter(system_file_paths):
-    argv = ["--inputfile", "fil_rh", "--inputtype", "surfex", "--variable", "rh2m"]
+    argv = [
+        "--basetime",
+        "2019010100",
+        "--inputfile",
+        "fil_rh",
+        "--inputtype",
+        "surfex",
+        "--variable",
+        "rh2m",
+    ]
     parent_parser = argparse.ArgumentParser(add_help=False)
     variable_parse_options(parent_parser)
 

tests/unit/test_interpolation.py

@@ -297,7 +297,7 @@ def test_plot_wind_barbs(data_thredds_nc_file_aa):
             assert pytest.approx(new_field_y[i][j]) == target_new_field_y[i][j]
 
     if plot:
-        import matplotlib.pyplot as plt
+        import matplotlib.pyplot as plt  # noqa: PLC0415
 
         plt.clf()
         plt.barbs(lon, lat, old_field_x, old_field_y, barbcolor=["red"])

tests/unit/test_variable.py

@@ -279,6 +279,7 @@ def test_open_new_file_grib1(fixture):
 
 def test_open_new_file_grib2(fixture):
     """Test to open a grib2 file."""
+    basetime = as_datetime_args(year=2019, month=11, day=13, hour=0)
     initialtime = as_datetime_args(year=2019, month=11, day=13, hour=2)
     intervall = 3600
     case = "grib2"
@@ -288,7 +289,7 @@ def test_open_new_file_grib2(fixture):
     for i in range(11):
         validtime = initialtime + as_timedelta(seconds=intervall * i)
         previoustime = validtime - as_timedelta(seconds=intervall)
-        variable = Variable(var_type, var_dict, initialtime)
+        variable = Variable(var_type, var_dict, basetime)
         previous_filename = variable.get_filename(validtime, previoustime=previoustime)
         filename = variable.get_filename(validtime)
         assert filename == var_dict["blueprint"][str(i)]