Merge pull request #177 from nasa/feature/issue-176-grouping-using-day-in-US-central

Issue 176: Modifies grouping algorithm to use day in US Central as the first part of unique grouping identifier.

Author: ank1m

CHANGELOG.md

@@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+- update grouping algorithm to use day in US Central as the first part of unique grouping identifier ([#177](https://github.com/nasa/batchee/pull/177))([**@ank1m**](https://github.com/ank1m))
+
 ## [1.3.0] - 2024-11-19
 
 ### Changed

batcher/tempo_filename_parser.py

@@ -29,6 +29,8 @@
 import logging
 import re
 from argparse import ArgumentParser
+from datetime import datetime
+from zoneinfo import ZoneInfo
 
 default_logger = logging.getLogger(__name__)
 
@@ -45,6 +47,36 @@
 )
 
 
+def get_day_in_us_central(
+    day_in_granule: str, time_in_granule: str, assume_tz=ZoneInfo("UTC")
+) -> str:
+    """
+    Convert a datetime to US Central time (US/Central) and return
+    a timezone-aware datetime.
+
+    Parameters
+    ----------
+    day_in_granule: str
+        The day from granule filename
+    time_in_granule: str
+        The time from granule filename
+    assume_tz : timezone, optional (default: UTC)
+        this is the timezone in which `day_in_granule` and `time_in_granule`
+        should be interpreted before converting to Central.
+
+    Returns
+    -------
+    str
+        The day for datetime converted to US/Central
+    """
+
+    dt = datetime.strptime(day_in_granule + time_in_granule, "%Y%m%d%H%M%S")
+    dt = dt.replace(tzinfo=assume_tz)
+
+    dt_central = dt.astimezone(ZoneInfo("America/Chicago"))
+    return dt_central.strftime("%Y%m%d")
+
+
 def get_batch_indices(filenames: list, logger: logging.Logger = default_logger) -> list[int]:
     """
     Returns
@@ -60,7 +92,10 @@ def get_batch_indices(filenames: list, logger: logging.Logger = default_logger)
         matches = tempo_granule_filename_pattern.match(name)
         if matches:
             match_dict = matches.groupdict()
-            day_and_scans.append((match_dict["day_in_granule"], match_dict["daily_scan_id"]))
+            day_in_central = get_day_in_us_central(
+                match_dict["day_in_granule"], match_dict["time_in_granule"]
+            )
+            day_and_scans.append((day_in_central, match_dict["daily_scan_id"]))
 
     # Unique day-scans are determined (while keeping the same order). Each will be its own batch.
     unique_day_scans: list[tuple[str, str]] = sorted(set(day_and_scans), key=day_and_scans.index)

tests/test_filename_grouping.py

@@ -2,22 +2,41 @@
 from unittest.mock import patch
 
 import batcher.tempo_filename_parser
-from batcher.tempo_filename_parser import get_batch_indices
+from batcher.tempo_filename_parser import get_batch_indices, get_day_in_us_central
 
 example_filenames = [
-    "TEMPO_HCHO_L2_V01_20130701T212354Z_S009G05.nc",
-    "TEMPO_HCHO_L2_V01_20130701T212953Z_S009G06.nc",
-    "TEMPO_HCHO_L2_V01_20130701T213553Z_S009G07.nc",
-    "TEMPO_HCHO_L2_V01_20130701T215955Z_S010G01.nc",
-    "TEMPO_HCHO_L2_V01_20130701T220554Z_S010G02.nc",
-    "TEMPO_HCHO_L2_V01_20130701T221154Z_S010G03.nc",
+    "TEMPO_NO2_L2_V03_20240731T235252Z_S016G04.nc",
+    "TEMPO_NO2_L2_V03_20240731T235929Z_S016G05.nc",
+    "TEMPO_NO2_L2_V03_20240801T000606Z_S016G06.nc",
+    "TEMPO_NO2_L2_V03_20240801T001302Z_S017G01.nc",
+    "TEMPO_NO2_L2_V03_20240801T001942Z_S017G02.nc",
+    "TEMPO_NO2_L2_V03_20240801T002619Z_S017G03.nc",
+    "TEMPO_NO2_L2_V03_20240801T233313Z_S016G01.nc",
+    "TEMPO_NO2_L2_V03_20240801T233953Z_S016G02.nc",
+    "TEMPO_NO2_L2_V03_20240801T234630Z_S016G03.nc",
 ]
 
 
+def test_timezone_conversion():
+    utcdates = [filename.split("_")[4] for filename in example_filenames]
+    days_in_central = [get_day_in_us_central(utcdt[0:8], utcdt[9:15]) for utcdt in utcdates]
+    assert days_in_central == [
+        "20240731",
+        "20240731",
+        "20240731",
+        "20240731",
+        "20240731",
+        "20240731",
+        "20240801",
+        "20240801",
+        "20240801",
+    ]
+
+
 def test_grouping():
     results = get_batch_indices(example_filenames)
 
-    assert results == [0, 0, 0, 1, 1, 1]
+    assert results == [0, 0, 0, 1, 1, 1, 2, 2, 2]
 
 
 def test_main_cli():
@@ -27,4 +46,4 @@ def test_main_cli():
     with patch.object(sys, "argv", test_args):
         grouped_names = batcher.tempo_filename_parser.main()
 
-    assert grouped_names == [example_filenames[0:3], example_filenames[3:6]]
+    assert grouped_names == [example_filenames[0:3], example_filenames[3:6], example_filenames[6:9]]