added CIOFS3, fixed some date issues

* CIOFS3 has been added as an option
* some issues were found in `make_ciofs_kerchunk` which should now be fixed

Author: kthyng

particle_tracking_manager/config_ocean_model.py

@@ -105,6 +105,14 @@ def open_dataset(self, drop_vars: list) -> xr.Dataset:
                         drop_variables=drop_vars,
                         decode_times=False,
                     )
+                elif ".parquet" in self.ocean_model_config.loc_remote:
+                    ds = xr.open_dataset(
+                        self.ocean_model_config.loc_remote,
+                        engine="kerchunk",
+                        # chunks={},  # Looks like it is faster not to include this for kerchunk
+                        drop_variables=drop_vars,
+                        decode_times=False,
+                    )
                 else:
                     ds = xr.open_zarr(
                         self.ocean_model_config.loc_remote,
@@ -185,6 +193,8 @@ def get_file_date_string(name: str, date: datetime) -> str:
         return f"{date.year}_{str(date.timetuple().tm_yday - 1).zfill(4)}"
     elif name == "CIOFSFRESH":
         return f"{date.year}_{str(date.timetuple().tm_yday - 1).zfill(4)}"
+    elif name == "CIOFS3":
+        return f"{date.year}_{str(date.timetuple().tm_yday - 1).zfill(4)}"
     else:
         raise ValueError(f"get_file_date_string not implemented for {name}.")
 

particle_tracking_manager/models/opendrift/utils.py

@@ -114,6 +114,8 @@ def make_ciofs_kerchunk(start: str, end: str, name: str) -> dict:
 
     if name == "CIOFS":
         output_dir_single_files = "/mnt/vault/ciofs/HINDCAST/.kerchunk_json"
+    elif name == "CIOFS3":
+        output_dir_single_files = "/mnt/vault/ciofs/CIOFS3/HINDCAST/.kerchunk_json"
     elif name == "CIOFSFRESH":
         output_dir_single_files = "/mnt/vault/ciofs/HINDCAST_FRESHWATER/.kerchunk_json"
     elif name == "CIOFSOP":
@@ -123,21 +125,24 @@ def make_ciofs_kerchunk(start: str, end: str, name: str) -> dict:
 
     fs2 = fsspec.filesystem("")  # local file system to save final jsons to
 
-    if name in ["CIOFS", "CIOFSFRESH"]:
+    if name in ["CIOFS", "CIOFSFRESH", "CIOFS3"]:
 
         # base for matching
         def base_str(a_time: str) -> str:
             return f"{output_dir_single_files}/{a_time}_*.json"
 
+        file_date_format = "%Y_0%j"
         date_format = "%Y_0%j"
 
     elif name == "CIOFSOP":
 
         # base for matching
         def base_str(a_time: str) -> str:
-            return f"{output_dir_single_files}/ciofs_{a_time}-*.json"
+            return f"{output_dir_single_files}/ciofs_{a_time}*.json"
+
+        file_date_format = "ciofs_%Y-%m-%d"
+        date_format = "%Y-%m-%d"
 
-        date_format = "ciofs_%Y-%m-%d"
     else:
         raise ValueError(f"Name {name} not recognized")
 
@@ -146,21 +151,24 @@ def base_str(a_time: str) -> str:
     if end[:4] != start[:4]:
         json_list += fs2.glob(base_str(end[:4]))
 
+    start_date = datetime.strptime(start, date_format)
+    end_date = datetime.strptime(end, date_format)
+
     # forward in time
-    if end > start:
+    if end_date >= start_date:
         json_list = [
             j
             for j in json_list
-            if datetime.strptime(Path(j).stem, date_format).isoformat() >= start
-            and datetime.strptime(Path(j).stem, date_format).isoformat() <= end
+            if datetime.strptime(Path(j).stem, file_date_format) >= start_date
+            and datetime.strptime(Path(j).stem, file_date_format) <= end_date
         ]
     # backward in time
-    elif end < start:
+    elif end_date < start_date:
         json_list = [
             j
             for j in json_list
-            if datetime.strptime(Path(j).stem, date_format).isoformat() <= start
-            and datetime.strptime(Path(j).stem, date_format).isoformat() >= end
+            if datetime.strptime(Path(j).stem, file_date_format) <= start_date
+            and datetime.strptime(Path(j).stem, file_date_format) >= end_date
         ]
 
     if json_list == []:

particle_tracking_manager/ocean_models/CIOFS3.yaml

@@ -0,0 +1,26 @@
+standard_name_mapping_CIOFS:
+  mask_rho: "land_binary_mask"
+  wetdry_mask_rho: "land_binary_mask"
+  u_eastward: "x_sea_water_velocity"
+  v_northward: "y_sea_water_velocity"
+
+CIOFS3:
+  name: "CIOFS3"
+  loc_remote: null
+  temporal_resolution_str: "PT1H"
+  lon_min: -156.485291
+  lon_max: -148.925125
+  lat_min: 56.7004919
+  lat_max: 61.5247774
+  start_time_model: "1999-01-01T00:00:00"
+  end_time_fixed: "2025-01-01T00:00:00"
+  oceanmodel_lon0_360: false
+  standard_name_mapping:
+    mask_rho: "land_binary_mask"
+    wetdry_mask_rho: "land_binary_mask"
+    u_eastward: "x_sea_water_velocity"
+    v_northward: "y_sea_water_velocity"
+  model_drop_vars:
+    - "wetdry_mask_psi"
+  dx: 100
+  kerchunk_func_str: "make_ciofs_kerchunk"

tests/test_config_ocean_model.py

@@ -44,6 +44,12 @@
         "start_time": "2014-01-01T00:00",
         "end_time": "2014-01-01T05:00",
     },
+    "CIOFS3": {
+        "lon": -153,
+        "lat": 59,
+        "start_time": "2023-01-01T00:00",
+        "end_time": "2023-01-01T05:00",
+    },
 }
 
 # Invalid values (except start_times are valid since not testing those here)
@@ -76,6 +82,13 @@
         "end_time": "2022-01-01T05:00",
         "ocean_model_config": ocean_model_registry.get("CIOFSFRESH"),
     },
+    "CIOFS3": {
+        "lon": -145,
+        "lat": 40,
+        "start_time": "2026-01-01",
+        "end_time": "2026-01-01T05:00",
+        "ocean_model_config": ocean_model_registry.get("CIOFS3"),
+    },
 }
 
 
@@ -151,6 +164,16 @@ def test_oceanmodel_lon0_360():
     assert m.ocean_model_config.oceanmodel_lon0_360 == False
     assert m.lon == lon_in
 
+    m = ocean_model_simulation_mapper["CIOFS3"](
+        start_time="2004-01-01",
+        lon=lon_in,
+        lat=57,
+        end_time="2004-01-01T05:00",
+        ocean_model_local=True,
+    )
+    assert m.ocean_model_config.oceanmodel_lon0_360 == False
+    assert m.lon == lon_in
+
     m = ocean_model_simulation_mapper["NWGOA"](
         start_time="2007-01-01",
         lon=lon_in,