From bd8092d0295e9df7f1b5203a3db9bad228bae56d Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Wed, 24 Sep 2025 11:46:38 -0600 Subject: [PATCH 1/5] MNT: add timeout on batch jobs --- sds_data_manager/constructs/processing_construct.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sds_data_manager/constructs/processing_construct.py b/sds_data_manager/constructs/processing_construct.py index fcb830ed3..0b07b36a7 100644 --- a/sds_data_manager/constructs/processing_construct.py +++ b/sds_data_manager/constructs/processing_construct.py @@ -137,4 +137,5 @@ def add_job(self, job_name: str, data_access_url: str = ""): f"ProcessingJob-{job_name}", job_definition_name=f"ProcessingJob-{job_name}", container=container_definition, + timeout=cdk.Duration.hours(2), ) From bc35390efe6bf3cded6f523df59c87f640315e70 Mon Sep 17 00:00:00 2001 From: Sean Hoyt Date: Fri, 10 Oct 2025 13:43:37 -0600 Subject: [PATCH 2/5] added pset deps --- .../SDSCode/pipeline_lambdas/dependency_config.csv | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sds_data_manager/lambda_code/SDSCode/pipeline_lambdas/dependency_config.csv b/sds_data_manager/lambda_code/SDSCode/pipeline_lambdas/dependency_config.csv index 786551548..66bf574d9 100644 --- a/sds_data_manager/lambda_code/SDSCode/pipeline_lambdas/dependency_config.csv +++ b/sds_data_manager/lambda_code/SDSCode/pipeline_lambdas/dependency_config.csv @@ -600,10 +600,15 @@ lo, l1b, de, lo, l1c, pset, HARD, DOWNSTREAM lo, ancillary, good-times, lo, l1c, pset, HARD, DOWNSTREAM lo, ancillary, hydrogen-background, lo, l1c, pset, HARD, DOWNSTREAM lo, ancillary, oxygen-background, lo, l1c, pset, HARD, DOWNSTREAM -spin, spin, historical, lo, l1b, de, HARD, DOWNSTREAM -repoint, repoint, historical, lo, l1b, de, HARD, DOWNSTREAM +spin, spin, historical, lo, l1c, pset, HARD, DOWNSTREAM +repoint, repoint, historical, lo, l1c, pset, HARD, DOWNSTREAM leapseconds, spice, historical, lo, l1c, pset, HARD_NO_TRIGGER, DOWNSTREAM spacecraft_clock, spice, historical, lo, l1c, pset, HARD_NO_TRIGGER, DOWNSTREAM +attitude_history, spice, historical, lo, l1c, pset, HARD_NO_TRIGGER, DOWNSTREAM +imap_frames, spice, historical, lo, l1c, pset, HARD_NO_TRIGGER, DOWNSTREAM +planetary_ephemeris, spice, historical, lo, l1c, pset, HARD_NO_TRIGGER, DOWNSTREAM +pointing_attitude, spice, historical, lo, l1c, pset, HARD_NO_TRIGGER, DOWNSTREAM +science_frames, spice, historical, lo, l1c, pset, HARD_NO_TRIGGER, DOWNSTREAM # TODO: Maps for 5-7 days need to be defined. deg resolution for those maps are currently N/A in Lo mapping product sheet. From f4da6f4e2da08fa9c7956348e19e73d935ad83bc Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Mon, 13 Oct 2025 14:15:47 -0600 Subject: [PATCH 3/5] ENH: increase batch memory size --- sds_data_manager/constructs/processing_construct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sds_data_manager/constructs/processing_construct.py b/sds_data_manager/constructs/processing_construct.py index 0b07b36a7..013fd5bd5 100644 --- a/sds_data_manager/constructs/processing_construct.py +++ b/sds_data_manager/constructs/processing_construct.py @@ -116,7 +116,7 @@ def add_job(self, job_name: str, data_access_url: str = ""): image=ecs.ContainerImage.from_ecr_repository( repository=container_repo, tag="latest" ), - memory=cdk.Size.mebibytes(4096), + memory=cdk.Size.gibibytes(16), cpu=1, environment={ # Useful for switching APIs between dev / prod endpoints From 2a02f62912355bd2f96234455422d85d7da0ba70 Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Mon, 13 Oct 2025 15:05:33 -0600 Subject: [PATCH 4/5] feedback --- sds_data_manager/constructs/processing_construct.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sds_data_manager/constructs/processing_construct.py b/sds_data_manager/constructs/processing_construct.py index 013fd5bd5..52e186664 100644 --- a/sds_data_manager/constructs/processing_construct.py +++ b/sds_data_manager/constructs/processing_construct.py @@ -117,7 +117,7 @@ def add_job(self, job_name: str, data_access_url: str = ""): repository=container_repo, tag="latest" ), memory=cdk.Size.gibibytes(16), - cpu=1, + cpu=4, environment={ # Useful for switching APIs between dev / prod endpoints "IMAP_DATA_ACCESS_URL": data_access_url, @@ -137,5 +137,4 @@ def add_job(self, job_name: str, data_access_url: str = ""): f"ProcessingJob-{job_name}", job_definition_name=f"ProcessingJob-{job_name}", container=container_definition, - timeout=cdk.Duration.hours(2), ) From aa6d57f6cbdcaa93bd822cc8c57d9a4cbe90077d Mon Sep 17 00:00:00 2001 From: Tenzin Choedon Date: Mon, 13 Oct 2025 15:07:49 -0600 Subject: [PATCH 5/5] rebase fix --- sds_data_manager/constructs/processing_construct.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sds_data_manager/constructs/processing_construct.py b/sds_data_manager/constructs/processing_construct.py index 52e186664..d299af965 100644 --- a/sds_data_manager/constructs/processing_construct.py +++ b/sds_data_manager/constructs/processing_construct.py @@ -137,4 +137,5 @@ def add_job(self, job_name: str, data_access_url: str = ""): f"ProcessingJob-{job_name}", job_definition_name=f"ProcessingJob-{job_name}", container=container_definition, + timeout=cdk.Duration.hours(2), )