Bugfix/harp sqlite generation

docs/auto_sphinx.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Create auto documentation."""
+
 import logging
 import os
 import time

examples/find_subset.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Example."""
+
 import json
 import logging
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
   name = "pysurfex"
-  version = "0.1.2"
+  version = "0.1.3"
   description = "Python API to SURFEX"
   authors = [{name = "Trygve Aspelien"}]
   license = "MIT"
@@ -77,7 +77,8 @@
   verification = [
     "verif>=1.2.3",
     "xarray>=2025.3.1",
-    "pandas>=2.0.0"
+    "pandas>=2.0.0",
+    "grib2sqlite@git+https://github.com/destination-earth-digital-twins/grib2sqlite.git"
   ]
 
   test = [

pysurfex/__init__.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """pysurfex module."""
+
 from importlib.metadata import version
 from pathlib import Path
 

pysurfex/datetime_utils.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 """Implement helper routines to deal with dates and times."""
+
 from datetime import date, datetime, timedelta, timezone
 
 

pysurfex/obsoul.py

@@ -113,8 +113,7 @@ def __init__(
                             add_obs = False
                     else:
                         logging.debug(
-                            "Not checking time window.",
-                            "neg_dt=%s and/or pos_dt=%s are None",
+                            "Not checking time window. neg_dt=%s and/or pos_dt=%s None",
                             neg_dt,
                             pos_dt,
                         )

pysurfex/verification.py

@@ -281,24 +281,6 @@ def merge(self, ds_model, ds_obs):
         leadtime = ds_model["leadtime"].data
         model_reference_time = ds_model["time"].data
         location = ds_model["location"].data
-        model_valid_time = []
-        for tim in model_reference_time:
-            for lt in leadtime:
-                model_valid_time.append(tim + lt)
-        unique_model_valid_time = np.sort(np.unique(model_valid_time))
-        ntimes = unique_model_valid_time.shape[0]
-
-        # dummy observations
-        nlocations = location.shape[0]
-        dummy_obs = np.nan * np.random.randn(ntimes, nlocations)
-        ds3 = xr.Dataset(
-            data_vars={"obs": (["obstime", "location"], dummy_obs)},
-            coords={
-                "obstime": unique_model_valid_time,
-                "location": location,
-            },
-        )
-        ds_obs = ds3.update(ds_obs)
 
         fcst = ds_model.fcst.data
         obs = ds_obs.obs.sel(obstime=ds_model.time + ds_model.leadtime).data
@@ -1051,7 +1033,7 @@ def converter2harp(converter, **kwargs):
     if sqlite_template is None:
         sqlite_template = "{MODEL}/{YYYY}/{MM}/FCTABLE_{PP}_{YYYY}{MM}.sqlite"
 
-    leadtime = int((validtime - basetime).total_seconds() / 3600.0)
+    leadtime = int((validtime - basetime).total_seconds())
     cache = None
     param = {
         "varname": variable,
@@ -1065,6 +1047,12 @@ def converter2harp(converter, **kwargs):
         validtime, cache
     )
 
+    if harp_param_unit == "%" and max(data_vector) <= 2:
+        logging.warning("Harp exepcts percentage but data seems to be fraction")
+        logging.warning("Scaling fractional data to percent")
+        data_vector = data_vector * 100
+        data_vector[data_vector > 100] = 100.0
+
     lons, lats, elevs = stationlist.get_pos_from_stid(stationlist.stids)
     harp_station_list = pandas.DataFrame(
         {

tests/conftest.py

@@ -606,8 +606,7 @@ def bufr_bad_file(tmp_path_factory):
 def obsoul_cryoclim_cy43(tmp_path_factory):
     fname = f"{tmp_path_factory.getbasetemp().as_posix()}/cryoclim.obsoul"
     with open(fname, mode="w", encoding="utf-8") as fhandler:
-        fhandler.write(
-            """
+        fhandler.write("""
             1           2
   15  1        17  80.47041   24.15402  'CRYO    '  20200220  60000 -2.14748e+09     1   1111   0
          92  999999.00  -2147483647.00  0.100  2048
@@ -617,8 +616,7 @@ def obsoul_cryoclim_cy43(tmp_path_factory):
          92  999999.00  -2147483647.00  0.100  2048
   15  1        17  80.21594   25.50849  'CRYO    '  20200220  60000 -2.14748e+09     1   1111   0
          92  999999.00  -2147483647.00  0.000  2048
-"""
-        )
+""")
     return fname
 
 
@@ -627,8 +625,7 @@ def data_thredds_nc_file(tmp_path_factory):
     fname = f"{tmp_path_factory.getbasetemp().as_posix()}/data_thredds_nc.nc"
     cdlfname = f"{tmp_path_factory.getbasetemp().as_posix()}/data_thredds_nc.cdl"
     with open(cdlfname, mode="w", encoding="utf-8") as fhandler:
-        fhandler.write(
-            """
+        fhandler.write("""
 netcdf meps_thredds {
 dimensions:
         time = UNLIMITED ;
@@ -746,8 +743,7 @@ def data_thredds_nc_file(tmp_path_factory):
 200, 0, 230, 20, 0, 1000,
 300, 0, 330, 30, 0, 3000;
 }
-"""
-        )
+""")
     Dataset(fname, mode="w").fromcdl(
         cdlfname, ncfilename=fname, mode="a", format="NETCDF3_CLASSIC"
     )
@@ -759,8 +755,7 @@ def data_thredds_nc_file_aa(tmp_path_factory):
     fname = f"{tmp_path_factory.getbasetemp().as_posix()}/data_thredds_nc.nc"
     cdlfname = f"{tmp_path_factory.getbasetemp().as_posix()}/data_thredds_nc.cdl"
     with open(cdlfname, mode="w", encoding="utf-8") as fhandler:
-        fhandler.write(
-            """
+        fhandler.write("""
 netcdf aa_thredds {
 dimensions:
         time = UNLIMITED ;
@@ -827,8 +822,7 @@ def data_thredds_nc_file_aa(tmp_path_factory):
 -20.0, -20.0, -20.0, -20.0, -20.0, -20.0,-20.0, -20.0, -20.0, -20.0, -20.0, -20.0,-20.0, -20.0, -20.0, -20.0, -20.0, -20.0,
 -20.0, -20.0, -20.0, -20.0, -20.0, -20.0,-20.0, -20.0, -20.0, -20.0, -20.0, -20.0,-20.0, -20.0, -20.0, -20.0, -20.0, -20.0;
 }
-"""
-        )
+""")
     Dataset(fname, mode="w").fromcdl(
         cdlfname, ncfilename=fname, mode="a", format="NETCDF3_CLASSIC"
     )
@@ -857,8 +851,7 @@ def data_surfex_pgd_nc_file(tmp_path_factory):
     fname = f"{tmp_path_factory.getbasetemp().as_posix()}/data_surfex_pgd_nc.nc"
     cdlfname = f"{tmp_path_factory.getbasetemp().as_posix()}/data_surfex_pgd_nc.cdl"
     with open(cdlfname, mode="w", encoding="utf-8") as fhandler:
-        fhandler.write(
-            """
+        fhandler.write("""
 netcdf PGD {
 dimensions:
         xx = 2 ;
@@ -1013,8 +1006,7 @@ def data_surfex_pgd_nc_file(tmp_path_factory):
 0.0, 1.0, 0.8, 0.25, 0.75, 0.1;
 
 }
-"""
-        )
+""")
     Dataset(fname, mode="w").fromcdl(
         cdlfname, ncfilename=fname, mode="a", format="NETCDF3_CLASSIC"
     )
@@ -1038,8 +1030,7 @@ def firstguess4gridpp(tmp_path_factory):
     fname = f"{tmp_path_factory.getbasetemp().as_posix()}/FirstGuess4gridpp.nc"
     cdlfname = f"{tmp_path_factory.getbasetemp().as_posix()}/FirstGuess4gridpp.cdl"
     with open(cdlfname, mode="w", encoding="utf-8") as fhandler:
-        fhandler.write(
-            """
+        fhandler.write("""
 netcdf FirstGuess4gridpp {
 dimensions:
         y = 3 ;
@@ -1129,8 +1120,7 @@ def firstguess4gridpp(tmp_path_factory):
 land_area_fraction =
    1, 1, 0, 1, 0.1664643, 0.1266151;
 }
-"""
-        )
+""")
     if not os.path.exists(fname):
         Dataset(fname, mode="w").fromcdl(cdlfname, ncfilename=fname, mode="a")
     return fname
@@ -1141,8 +1131,7 @@ def data_cryoclim_nc_file(tmp_path_factory):
     fname = f"{tmp_path_factory.getbasetemp().as_posix()}/cryoclim_nc.nc"
     cdlfname = f"{tmp_path_factory.getbasetemp().as_posix()}/cryoclim_nc.cdl"
     with open(cdlfname, mode="w", encoding="utf-8") as fhandler:
-        fhandler.write(
-            """
+        fhandler.write("""
 netcdf cryoclim {
 dimensions:
         time = 1 ;
@@ -1206,8 +1195,7 @@ def data_cryoclim_nc_file(tmp_path_factory):
 classed_value_c = 1, 1, 2, 3, 0, 4;
 classed_product = 1, 1, 2, 3, 0, 4;
 }
-"""
-        )
+""")
     Dataset(fname, mode="w").fromcdl(
         cdlfname, ncfilename=fname, mode="a", format="NETCDF3_CLASSIC"
     )

tests/smoke/test_cli_misc.py

@@ -173,8 +173,7 @@ def data_sentinel_nc_file(tmp_path_factory):
     fname = f"{tmp_path_factory.getbasetemp().as_posix()}/sentinel_nc.nc"
     cdlfname = f"{tmp_path_factory.getbasetemp().as_posix()}/sentinel_nc.cdl"
     with open(cdlfname, mode="w", encoding="utf-8") as fhandler:
-        fhandler.write(
-            """
+        fhandler.write("""
 netcdf sentinel {
 dimensions:
         xc = 2 ;
@@ -194,8 +193,7 @@ def data_sentinel_nc_file(tmp_path_factory):
 
 surface_soil_moisture = 0.01, 0.01, 0.01, 0.03, 0.001, 0.001;
 }
-"""
-        )
+""")
     Dataset(fname, mode="w").fromcdl(
         cdlfname, ncfilename=fname, mode="a", format="NETCDF3_CLASSIC"
     )

tests/unit/test_bufr.py

@@ -13,7 +13,7 @@ def settings(bufr_file):
         "label": {
             "filetype": "bufr",
             "filepattern": bufr_file,
-            "varname": "airTemperatureAt2m",
+            "varname": ["airTemperatureAt2M"],
             "lonrange": [0, 20],
             "latrange": [55, 65],
             "dt": 1800,