Merge pull request #34 from Loxeris/fix-use-postprocess-value

Check post-process returned value for job success

Author: aldbr

src/dirac_cwl_proto/execution_hooks/plugins/lhcb.py

@@ -325,7 +325,12 @@ def post_process(self, job_path: Path, **kwargs: Any) -> bool:
         catalog_files = glob.glob(str(job_path / "pool_xml_catalog.xml"))
         for catalog_file in catalog_files:
             try:
-                self.store_output("pool_xml_catalog", catalog_file)
+                self.store_output(
+                    "pool_xml_catalog",
+                    catalog_file,
+                    task_id=self.task_id,
+                    run_id=self.run_id,
+                )
             except Exception as e:
                 print(f"Failed to store catalog {catalog_file}: {e}")
                 success = False
@@ -407,7 +412,9 @@ def post_process(self, job_path: Path, **kwargs: Any) -> bool:
         dst_files = glob.glob(str(job_path / "*.dst"))
         for dst_file in dst_files:
             try:
-                self.store_output("dst", dst_file)
+                self.store_output(
+                    "dst", dst_file, task_id=self.task_id, run_id=self.run_id
+                )
             except Exception as e:
                 print(f"Failed to store DST output {dst_file}: {e}")
                 success = False
@@ -416,7 +423,9 @@ def post_process(self, job_path: Path, **kwargs: Any) -> bool:
         log_files = glob.glob(str(job_path / "*.log"))
         for log_file in log_files:
             try:
-                self.store_output("log", log_file)
+                self.store_output(
+                    "log", log_file, task_id=self.task_id, run_id=self.run_id
+                )
             except Exception as e:
                 print(f"Failed to store log {log_file}: {e}")
                 success = False

src/dirac_cwl_proto/job/__init__.py

@@ -396,15 +396,17 @@ def run_job(job: JobSubmissionModel) -> bool:
 
         # Post-process the job
         logger.info("Post-processing Task...")
-        _post_process(
+        if _post_process(
             result.returncode,
             result.stdout,
             result.stderr,
             job_path,
             runtime_metadata,
-        )
-        logger.info("Task post-processed successfully!")
-        return True
+        ):
+            logger.info("Task post-processed successfully!")
+            return True
+        logger.error("Failed to post-process Task")
+        return False
 
     except Exception:
         logger.exception("JobWrapper: Failed to execute workflow")

test/test_workflows.py

@@ -316,7 +316,7 @@ def test_run_job_validation_failure(
         # --- Gaussian fit example ---
         # Data generation workflow
         (
-            "test/workflows/gaussian_fit/data_generation/data-generation.cwl",
+            "test/workflows/gaussian_fit/data_generation/data-generation-workflow.cwl",
             "test/workflows/gaussian_fit/type_dependencies/transformation/inputs-data-generation.yaml",
         ),
     ],

test/workflows/gaussian_fit/data_generation/data-generation-workflow.cwl

@@ -7,10 +7,10 @@ doc: >
 inputs:
   output_file_name_1:
     type: string
-    default: "data1.txt"
+    default: "data-gen1.txt"
   output_file_name_2:
     type: string
-    default: "data2.txt"
+    default: "data-gen2.txt"
 
 outputs:
   data1: