dev: added nested_wf case and created additional functions to reduce cognitive complexity

Author: Stellatsuu

src/dirac_cwl_proto/execution_hooks/requirement_validator.py

@@ -25,11 +25,6 @@ def get_requirement(self, cwl_object: Workflow | CommandLineTool | WorkflowStep)
                 return requirement
         return None
 
-    # TODO : get all nested workflows of a workflow,
-    # for each nested workflow, validate its requirements
-    def get_nested_workflows(self):
-        pass
-
     @abstractmethod
     def validate_requirement(
         self, requirement, context: str = "", global_requirement=None
@@ -55,54 +50,84 @@ def validate_production_requirement(self, requirement, is_global: bool = False):
         """
         pass
 
-    def validate_requirements(self, production: bool = False) -> None:
+    def validate_requirements(self, cwl_object=None, production: bool = False) -> None:
         """
-        Validate all requirements in Workflow, WorkflowStep and WorkflowStep.run.
+        Validate all requirements in a Workflow.
 
+        :param cwl_object: The cwl_object to validate the requirements for.
+        If None, use the cwl_object from the class. This is used for nested workflows validation.
         :param production: True if the requirements are for a production workflow, False otherwise.
-        Maybe we could add a transformation parameter later.
+        -- Maybe we could add a transformation parameter later.
         """
-        global_requirement = self.get_requirement(self.cwl_object)
+        cwl_object = cwl_object if cwl_object else self.cwl_object
+        global_requirement = self.get_requirement(cwl_object)
 
         # Validate Workflow/CommandLineTool global requirements.
         if global_requirement:
+            # Production-workflow-specific validation.
             if production:
                 self.validate_production_requirement(global_requirement, is_global=True)
             self.validate_requirement(global_requirement, context="global requirements")
 
         # Validate WorkflowStep requirements, if any.
-        if self.cwl_object.class_ != "CommandLineTool" and self.cwl_object.steps:
-            self.validate_steps_requirements(global_requirement, production)
+        if cwl_object.class_ != "CommandLineTool" and cwl_object.steps:
+            self.validate_steps_requirements(
+                cwl_object.steps, global_requirement, production
+            )
 
-    def validate_steps_requirements(self, global_requirement, production: bool = False):
+    def validate_steps_requirements(
+        self, steps, global_requirement, production: bool = False
+    ):
         """
-        Validate steps requirements in WorkflowStep and WorkflowStep.run.
+        Validate steps requirements in WorkflowStep.
 
+        :param steps: The WorkflowStep to validate the requirements for.
         :param global_requirement: The global Workflow requirement, if any.
         :param production: True if the requirements are for a production workflow, False otherwise.
+        -- Maybe we could add a transformation parameter later.
         """
-        # Validate WorkflowStep requirements, if any.
-        for step in self.cwl_object.steps:
+        for step in steps:
             step_requirement = self.get_requirement(step)
+            # Validate WorkflowStep requirements, if any.
             if step_requirement:
+                # Production-workflow-specific validation.
                 if production:
                     self.validate_production_requirement(step_requirement)
                 self.validate_requirement(
                     step_requirement,
                     context="step requirements",
                     global_requirement=global_requirement,
                 )
+
             # Validate WorkflowStep.run, if any.
             if step.run:
-                step_run_requirement = self.get_requirement(step.run)
-                if step_run_requirement:
-                    if production:
-                        self.validate_production_requirement(step_run_requirement)
-                    self.validate_requirement(
-                        step_run_requirement,
-                        context="step run requirements",
-                        global_requirement=global_requirement,
-                    )
+                self.validate_run_requirements(step.run, global_requirement, production)
+
+    def validate_run_requirements(
+        self, run, global_requirement, production: bool = False
+    ):
+        """
+        Validate WorkflowStep.run requirements.
+
+        :param run: The WorkflowStep.run to validate the requirements for.
+        :param global_requirement: The global Workflow requirement, if any.
+        :param production: True if the requirements are for a production workflow, False otherwise.
+        -- Maybe we could add a transformation parameter later.
+        """
+        if run.class_ == "Workflow":
+            # Validate nested Workflow requirements, if any.
+            self.validate_requirements(cwl_object=run)
+
+        step_run_requirement = self.get_requirement(run)
+        if step_run_requirement:
+            # Production-workflow-specific validation.
+            if production:
+                self.validate_production_requirement(step_run_requirement)
+            self.validate_requirement(
+                step_run_requirement,
+                context="step run requirements",
+                global_requirement=global_requirement,
+            )
 
 
 class RequirementError(Exception):
@@ -148,7 +173,8 @@ def validate_conflict(min_value, global_max_value, resource, context):
     def validate_production_requirement(self, requirement, is_global: bool = False):
         """
         Raise an error if there's a global ResourceRequirement in a production workflow.
-        Otherwise, add some logic for WorkflowSteps, CommandLineTools and WorkflowStep.run in production workflows.
+        Otherwise, add some logic for WorkflowSteps, CommandLineTools
+        and WorkflowStep.run, etc. in production workflows.
 
         :param requirement: The current requirement to validate.
         :param is_global: True if there's a global ResourceRequirement, False otherwise.

test/test_workflows.py

@@ -279,6 +279,16 @@ def test_run_job_success(cli_runner, cleanup, cwl_file, inputs):
             [],
             "ResourceRequirementisinvalid",
         ),
+        (
+            "test/workflows/resource_requirements/resource_conflicts/nested_wf/external_conflict_nested_wf.cwl",
+            [],
+            "ResourceRequirementisinvalid",
+        ),
+        (
+            "test/workflows/resource_requirements/resource_conflicts/nested_wf/internal_conflict_nested_wf.cwl",
+            [],
+            "ResourceRequirementisinvalid",
+        ),
     ],
 )
 def test_run_job_validation_failure(
@@ -699,6 +709,11 @@ def test_run_simple_production_success(cli_runner, cleanup, cwl_file, metadata):
             "test/workflows/mandelbrot/type_dependencies/production/malformed-nonexisting-type_metadata-mandelbrot_complete.yaml",
             "Unknownexecutionhooksplugin:'MandelBrotDoesNotExist'",
         ),
+        (
+            "test/workflows/resource_requirements/bad_merge_production.cwl",
+            "test/workflows/merge/type_dependencies/production/metadata-merge_complete.yaml",
+            "ResourceRequirementisinvalid",
+        ),
     ],
 )
 def test_run_production_validation_failure(

test/workflows/resource_requirements/resource_conflicts/nested_wf/external_conflict_nested_wf.cwl

@@ -0,0 +1,23 @@
+cwlVersion: v1.2
+class: Workflow
+label: "Higher NestedWorkflow resourceMin than resourceMax"
+doc: The NestedWorkflow ResourceRequirement has higher resourceMin than global resourceMax = failure, both NestedWorflows
+      here are wrong but only the first one will trigger the exception.
+
+inputs: []
+outputs: []
+
+requirements:
+  ResourceRequirement:
+    ramMax: 512
+    coresMax: 4
+
+steps:
+  too_high_ram:
+    run: ./nested_wf_high_ram.cwl
+    in: []
+    out: []
+  too_high_cores:
+    run: ./nested_wf_high_cores.cwl
+    in: []
+    out: []

test/workflows/resource_requirements/resource_conflicts/nested_wf/internal_conflict_nested_wf.cwl

@@ -0,0 +1,18 @@
+cwlVersion: v1.2
+class: Workflow
+label: "Higher NestedWorkflow resourceMin than resourceMax"
+doc: The NestedWorkflow ResourceRequirement has higher resourceMin than resourceMax = failure, both NestedWorflows
+      here are wrong but only the first one will trigger the exception.
+
+inputs: []
+outputs: []
+
+steps:
+  too_high_ram:
+    run: ../ram_conflict_wf_step.cwl
+    in: [ ]
+    out: [ ]
+  too_high_cores:
+    run: ../cores_conflict_wf_step.cwl
+    in: []
+    out: []

test/workflows/resource_requirements/resource_conflicts/nested_wf/nested_wf_high_cores.cwl

@@ -0,0 +1,21 @@
+cwlVersion: v1.2
+class: Workflow
+label: "Higher NestedWorkflow resourceMin than resourceMax"
+doc: This NestedWorkflow ResourceRequirement has higher coresMin than global coresMax = failure
+
+inputs: []
+outputs: []
+
+requirements:
+  ResourceRequirement:
+    coresMin: 10
+
+steps:
+  step:
+    run:
+      class: CommandLineTool
+      baseCommand: ["echo", "Hello World"]
+      inputs: []
+      outputs: []
+    in: []
+    out: []

test/workflows/resource_requirements/resource_conflicts/nested_wf/nested_wf_high_ram.cwl

@@ -0,0 +1,22 @@
+cwlVersion: v1.2
+class: Workflow
+label: "Higher NestedWorkflow resourceMin than resourceMax"
+doc: This NestedWorkflow ResourceRequirement has higher ramMin than global ramMax = failure
+
+
+inputs: []
+outputs: []
+
+requirements:
+  ResourceRequirement:
+    ramMin: 1028
+
+steps:
+  step:
+    run:
+      class: CommandLineTool
+      baseCommand: ["echo", "Hello World"]
+      inputs: []
+      outputs: []
+    in: []
+    out: []