ENH: webpoda: Add an instrument-specific buffer for downloading packet files

Each instrument can specify they buffer they want to apply to include
packets +/- X minutes on either side of the midnight boundary.

Author: greglucas

imap_data_access/webpoda.py

@@ -127,6 +127,13 @@
     "ialirt": [478],
 }
 
+_INSTRUMENT_BUFFER_MINUTES = {
+    "hit": 20,
+    "mag": 30,
+    "swapi": 1,
+    "swe": 1,
+}
+
 
 def _get_webpoda_headers() -> dict:
     """Get the necessary headers for webpoda requests."""
@@ -299,7 +306,7 @@ def download_daily_data(
 
     # Iterate over the packet dates to make a query for each individual spacecraft day
     # packet_date 00:00:00 -> packet_date 23:59:59
-    for date in unique_dates:
+    for date in sorted(unique_dates):
         science_file = imap_data_access.ScienceFilePath.generate_from_inputs(
             instrument=instrument,
             data_level="l0",
@@ -315,6 +322,13 @@ def download_daily_data(
         daily_start_time = datetime.datetime.combine(date, datetime.time.min)
         daily_end_time = datetime.datetime.combine(date, datetime.time.max)
 
+        # Some instruments request a buffer of packets on either side of the midnight
+        # boundary to ensure their packet groupings work together.
+        buffer_minutes = _INSTRUMENT_BUFFER_MINUTES.get(instrument, 0)
+        buffer_timedelta = datetime.timedelta(minutes=buffer_minutes)
+        daily_start_time -= buffer_timedelta
+        daily_end_time += buffer_timedelta
+
         # Iterate over all apids, downloading the content for this time period
         # concatenating all the binary returns into a single binary file
         daily_packet_content = b"".join(

tests/test_webpoda.py

@@ -108,6 +108,15 @@ def test_download_daily_data(
         instrument, start_time, end_time, upload_to_server=upload_to_server
     )
 
+    # Make sure swapi was called with a buffer of 1 minute on either side of midnight
+    call = mock_get_packet_binary_data_sctime.call_args_list[0][0]
+    assert call == (
+        1184,
+        start_time - datetime.timedelta(minutes=1),
+        datetime.datetime.combine(start_time, datetime.time.max)
+        + datetime.timedelta(minutes=1),
+    )
+
     # We expect two daily files to be created because we have packets
     # across two separate days
     for day in mock_get_packet_times_ert.return_value: