BUG: Fix IDEX add_l1a_partition sensor (#1527)

* fix idex partition

* add test

* add comments to tests

Author: lacoak21

sds_data_manager/orchestration/custom_partitions.py

@@ -138,8 +138,9 @@ def add_daily_partitions(context: SensorEvaluationContext):
 @sensor(minimum_interval_seconds=86400)
 def add_idex_10_day_partitions(context: SensorEvaluationContext):
     """Alert Dagster when new IDEX 10-day partitions should be made."""
-    start_date = context.cursor or config.MISSION_START_TIME
-    start_dt = datetime.datetime.fromisoformat(start_date).replace(
+    # These partitions come from a static cadence CSV, so we always evaluate from
+    # mission start to avoid cursor drift shrinking the effective window.
+    start_dt = datetime.datetime.fromisoformat(config.MISSION_START_TIME).replace(
         tzinfo=datetime.timezone.utc
     )
     end_dt = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
@@ -187,9 +188,7 @@ def add_idex_10_day_partitions(context: SensorEvaluationContext):
         partition_requests.append(idex10_partitions.build_add_request(partition_names))
         context.log.info(f"Registered new dynamic partitions: {partition_names}")
 
-    return SensorResult(
-        dynamic_partitions_requests=partition_requests, cursor=end_dt.isoformat()
-    )
+    return SensorResult(dynamic_partitions_requests=partition_requests)
 
 
 ##### THIS TELLS DAGSTER THAT SOME FILES ARE DIVIDED UP BY 30-day
@@ -199,8 +198,9 @@ def add_idex_10_day_partitions(context: SensorEvaluationContext):
 @sensor(minimum_interval_seconds=86400)
 def add_idex_30_day_partitions(context: SensorEvaluationContext):
     """Alert Dagster when new IDEX 30-day partitions should be made."""
-    start_date = context.cursor or config.MISSION_START_TIME
-    start_dt = datetime.datetime.fromisoformat(start_date).replace(
+    # These partitions come from a static cadence CSV, so we always evaluate from
+    # mission start to avoid cursor drift shrinking the effective window.
+    start_dt = datetime.datetime.fromisoformat(config.MISSION_START_TIME).replace(
         tzinfo=datetime.timezone.utc
     )
     end_dt = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(
@@ -252,9 +252,7 @@ def add_idex_30_day_partitions(context: SensorEvaluationContext):
         partition_requests.append(idex30_partitions.build_add_request(partition_names))
         context.log.info(f"Registered new dynamic partitions: {partition_names}")
 
-    return SensorResult(
-        dynamic_partitions_requests=partition_requests, cursor=end_dt.isoformat()
-    )
+    return SensorResult(dynamic_partitions_requests=partition_requests)
 
 
 # Run daily (24 hours = 86400 seconds)

tests/orchestration/test_custom_partitions.py

@@ -0,0 +1,56 @@
+"""Tests for dynamic partition sensors in custom_partitions."""
+
+import datetime
+from unittest.mock import patch
+
+from dagster import build_sensor_context, instance_for_test
+
+from sds_data_manager.orchestration import custom_partitions
+
+
+# wrap the real datetime *module*; override only datetime.datetime.now
+class _FrozenDatetime(datetime.datetime):
+    @classmethod
+    def now(cls, tz=None):
+        return datetime.datetime(2025, 9, 29, tzinfo=datetime.timezone.utc)
+
+
+@patch.object(custom_partitions.datetime, "datetime", _FrozenDatetime)
+def test_add_idex_10_day_partitions():
+    """Check that add_idex_10_day_partitions adds the correct partitions."""
+    with instance_for_test() as instance:
+        # Mock existing partitions
+        instance.add_dynamic_partitions(
+            "idex_10_day_partitions",
+            ["idex10_2025-09-27T00:00:00_to_2025-10-07T00:00:00"],
+        )
+        context = build_sensor_context(instance=instance)
+        # Trigger the sensor. This should add more partitions.
+        sensor_result = custom_partitions.add_idex_10_day_partitions(context)
+
+    new_partitions = sensor_result.dynamic_partitions_requests[0].partition_keys
+    assert new_partitions == [
+        "idex10_2025-10-07T00:00:00_to_2025-10-17T00:00:00",
+        "idex10_2025-10-17T00:00:00_to_2025-10-27T00:00:00",
+        "idex10_2025-10-27T00:00:00_to_2025-11-06T00:00:00",
+    ]
+
+
+@patch.object(custom_partitions.datetime, "datetime", _FrozenDatetime)
+def test_add_idex_30_day_partitions():
+    """Check that add_idex_30_day_partitions adds the correct partitions."""
+    with instance_for_test() as instance:
+        # Mock existing partitions
+        instance.add_dynamic_partitions(
+            "idex_30_day_partitions",
+            ["idex30_2025-09-27T00:00:00_to_2025-10-07T00:00:00"],
+        )
+        context = build_sensor_context(instance=instance)
+        # Trigger the sensor. This should add more partitions.
+        sensor_result = custom_partitions.add_idex_30_day_partitions(context)
+
+    new_partitions = sensor_result.dynamic_partitions_requests[0].partition_keys
+    assert new_partitions == [
+        "idex30_2025-09-24T00:00:00_to_2025-09-27T00:00:00",
+        "idex30_2025-09-27T00:00:00_to_2025-10-27T00:00:00",
+    ]