Path fix superdark

examples/dark_pipeline_run.py

@@ -45,7 +45,7 @@
     dark_pipeline = DarkPipeline("WFI01")
     dark_pipeline.run_pipeline(file_list)
 
-rfp_tvac_list_run_prep = 1
+rfp_tvac_list_run_prep = 0
 if rfp_tvac_list_run_prep == 1:
 
     files = glob.glob("/grp/roman/GROUND_TESTS/TVAC2/ASDF/NOM_OPS/OTP00644_Darks_TV2a_R2_MCEB/Activity_1/*_WFI01*.asdf")
@@ -59,3 +59,9 @@
     file_list = list(prepped_asdf_files)
     dark_pipe.prep_superdark_file(short_file_list=file_list, short_dark_num_reads=350, outfile="validate_superdark_TVAC_test_prepped_superdark_short.asdf")
     print(f"superdark prepped!!")
+
+rfp_superdark_only = 1
+if rfp_superdark_only:
+    file_list = glob.glob("/grp/roman/GROUND_TESTS/TVAC2_IRRCcorr/ASDF/NOM_OPS/OTP00644_Darks_TV2a_R2_MCEB/Activity_1/*_WFI01*.asdf")
+    dark_pipe = DarkPipeline("WFI01")
+    dark_pipe.prep_superdark_file(full_file_list=file_list, full_file_num_reads=350, outfile="THIS_DEFINITELY_WORKED_FIRST_TRY.asdf")

src/wfi_reference_pipeline/pipelines/dark_pipeline.py

@@ -130,6 +130,7 @@ def prep_superdark_file(
         full_file_list=[],
         short_file_list=[],
         long_file_list=[],
+        full_file_num_reads = 0,
         short_dark_num_reads=DARK_SHORT_NUM_READS,
         long_dark_num_reads=DARK_LONG_NUM_READS,
         sig_clip_sd_low=DARK_SIGMA_CLIP_SD_LOW,
@@ -189,6 +190,12 @@ def prep_superdark_file(
         else:
             if full_file_list:
                 file_list = full_file_list
+                if full_file_num_reads == 0:
+                    raise ValueError(
+                        "full_file_num_reads parameter must be greater than 0"
+                    )
+                # full_file_list will eventually become short list
+                short_dark_num_reads = full_file_num_reads
             else:
                 # Standard case: use the pipeline prepped files
                 if len(self.prepped_files):
@@ -199,7 +206,7 @@ def prep_superdark_file(
                     )  # TODO - once we have documentation add link here
 
             # Filter list for detector
-            file_list = [file for file in file_list if self.detector in file.stem]
+            file_list = [Path(file) for file in file_list if self.detector in Path(file).stem]
             short_dark_file_list, long_dark_file_list = self.extract_short_and_long_file_lists(file_list)
 
         if len(short_dark_file_list) == 0:

src/wfi_reference_pipeline/pipelines/pipeline.py

@@ -2,9 +2,8 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from constants import REF_TYPE_DARK, WFI_DETECTORS
-
 from wfi_reference_pipeline.config.config_access import get_data_files_config
+from wfi_reference_pipeline.constants import REF_TYPE_DARK, WFI_DETECTORS
 from wfi_reference_pipeline.utilities.file_handler import FileHandler
 from wfi_reference_pipeline.utilities.logging_functions import configure_logging
 from wfi_reference_pipeline.utilities.quality_control.dark_quality_control import (

src/wfi_reference_pipeline/reference_types/dark/superdark_dynamic.py

@@ -133,6 +133,9 @@ def generate_superdark(
             f"Calculated Max Additional Processes:              {max_num_processes} "
         )
 
+        if max_num_processes < 1:
+            raise MemoryError("Not enough memory to effectively run Superdark Dynamic Processing. Set config/pipelines_config.yml `multiprocess_superdark: false`")
+
         print(f"Begin Multiprocessing with {max_num_processes} processes")
 
         try: