Merge pull request #471 from neurorepro/input_spec_changes

Changes to Input/Output Specs

Author: djarecka

docs/input_spec.rst

@@ -162,11 +162,17 @@ In the example we used multiple keys in the metadata dictionary including `help_
    A flag that specifies if the file extension should be removed from the field value.
    Used in order to create an output specification.
 
-
 `readonly` (`bool`, default: `False`):
    If `True` the input field can't be provided by the user but it aggregates other input fields
    (for example the fields with `argstr: -o {fldA} {fldB}`).
 
+`formatter` (`function`):
+   If provided the `argstr` of the field is created using the function. This function can for example
+   be used to combine several inputs into one command argument.
+   The function can take `field` (this input field will be passed to the function),
+   `inputs` (entire `inputs` will be passed) or any input field name
+   (a specific input field will be sent).
+
 
 Validators
 ----------

docs/output_spec.rst

@@ -61,6 +61,9 @@ The metadata dictionary for `output_spec` can include:
 `help_string` (`str`, mandatory):
    A short description of the input field. The same as in `input_spec`.
 
+`mandatory` (`bool`, default: `False`):
+   If `True` the output file has to exist, otherwise an error will be raised.
+
 `output_file_template` (`str`):
    If provided the output file name (or list of file names) is created using the template.
    The template can use other fields, e.g. `{file1}`. The same as in `input_spec`.
@@ -69,11 +72,14 @@ The metadata dictionary for `output_spec` can include:
    If provided the field is added to the output spec with changed name.
    The same as in `input_spec`.
 
+`absolute_path` (`bool` default: `False`):
+   A flag that specifies if the `output_file_template` is an absolute path. If the flag is not set,
+   then the `output_file_template` is searched in the nodes output directory.
+
 `keep_extension` (`bool`, default: `True`):
    A flag that specifies if the file extension should be removed from the field value.
    The same as in `input_spec`.
 
-
 `requires` (`list`):
    List of field names that are required to create a specific output.
    The fields do not have to be a part of the `output_file_template` and

pydra/engine/helpers_file.py

@@ -610,8 +610,15 @@ def template_update_single(
         return attr.NOTHING
     else:  # inputs_dict[field.name] is True or spec_type is output
         value = _template_formatting(field, inputs, inputs_dict_st)
-        # changing path so it is in the output_dir
-        if output_dir and value is not attr.NOTHING:
+        # changing path so it is in the output_dir, but only if absolute_path is not set
+        if (
+            output_dir
+            and value is not attr.NOTHING
+            and (
+                "absolute_path" not in field.metadata
+                or not field.metadata["absolute_path"]
+            )
+        ):
             # should be converted to str, it is also used for input fields that should be str
             if type(value) is list:
                 return [str(output_dir / Path(val).name) for val in value]

pydra/engine/specs.py

@@ -6,7 +6,7 @@
 import re
 from glob import glob
 
-from .helpers_file import template_update_single
+from .helpers_file import template_update_single, is_existing_file
 
 
 def attr_fields(spec, exclude_names=()):
@@ -400,6 +400,8 @@ def check_metadata(self):
             "keep_extension",
             "xor",
             "sep",
+            "absolute_path",
+            "formatter",
         }
         for fld in attr_fields(self, exclude_names=("_func", "_graph_checksums")):
             mdata = fld.metadata
@@ -496,7 +498,9 @@ def generated_output_names(self, inputs, output_dir):
                 output_names.append(fld.name)
             elif (
                 fld.metadata
-                and self._field_metadata(fld, inputs, output_dir, outputs=None)
+                and self._field_metadata(
+                    fld, inputs, output_dir, outputs=None, check_existance=False
+                )
                 != attr.NOTHING
             ):
                 output_names.append(fld.name)
@@ -529,7 +533,9 @@ def _field_defaultvalue(self, fld, output_dir):
             else:
                 raise AttributeError(f"no file matches {default.name}")
 
-    def _field_metadata(self, fld, inputs, output_dir, outputs=None):
+    def _field_metadata(
+        self, fld, inputs, output_dir, outputs=None, check_existance=True
+    ):
         """Collect output file if metadata specified."""
         if self._check_requires(fld, inputs) is False:
             return attr.NOTHING
@@ -543,10 +549,38 @@ def _field_metadata(self, fld, inputs, output_dir, outputs=None):
             value = template_update_single(
                 fld, inputs=inputs, output_dir=output_dir, spec_type="output"
             )
+
             if fld.type is MultiOutputFile and type(value) is list:
-                return [Path(val) for val in value]
+                # TODO: how to deal with mandatory list outputs
+                ret = []
+                for val in value:
+                    val = Path(val)
+                    if check_existance and not val.exists():
+                        ret.append(attr.NOTHING)
+                    elif check_existance and (
+                        fld.metadata.get("absolute_path", False)
+                        and not val.is_absolute()
+                    ):
+                        ret.append(attr.NOTHING)
+                    else:
+                        ret.append(val)
+                return ret
             else:
-                return Path(value)
+                val = Path(value)
+                # checking if the file exists
+                if check_existance and not val.exists():
+                    # if mandatory raise exception
+                    if "mandatory" in fld.metadata:
+                        if fld.metadata["mandatory"]:
+                            raise Exception(
+                                f"mandatory output for variable {fld.name} does not exit"
+                            )
+                    return attr.NOTHING
+                if check_existance and (
+                    fld.metadata.get("absolute_path", False) and not val.is_absolute()
+                ):
+                    return attr.NOTHING
+                return val
         elif "callable" in fld.metadata:
             call_args = inspect.getargspec(fld.metadata["callable"])
             call_args_val = {}

pydra/engine/task.py

@@ -345,7 +345,11 @@ def _command_args_single(self, state_ind=None, index=None):
             exclude_names=("container", "image", "container_xargs", "bindings"),
         ):
             name, meta = field.name, field.metadata
-            if getattr(self.inputs, name) is attr.NOTHING and not meta.get("readonly"):
+            if (
+                getattr(self.inputs, name) is attr.NOTHING
+                and not meta.get("readonly")
+                and not meta.get("formatter")
+            ):
                 continue
             if name == "executable":
                 pos_args.append(
@@ -404,8 +408,9 @@ def _command_pos_args(self, field, state_ind, index):
         the specific field.
         """
         argstr = field.metadata.get("argstr", None)
-        if argstr is None:
-            # assuming that input that has no arstr is not used in the command
+        formatter = field.metadata.get("formatter", None)
+        if argstr is None and formatter is None:
+            # assuming that input that has no arstr is not used in the command, or a formatter is not provided too.
             return None
         pos = field.metadata.get("position", None)
         if pos is not None:
@@ -426,11 +431,45 @@ def _command_pos_args(self, field, state_ind, index):
         value = self._field_value(field, state_ind, index, check_file=True)
         if field.metadata.get("readonly", False) and value is not None:
             raise Exception(f"{field.name} is read only, the value can't be provided")
-        elif value is None and not field.metadata.get("readonly", False):
+        elif (
+            value is None
+            and not field.metadata.get("readonly", False)
+            and formatter is None
+        ):
             return None
 
+        # getting stated inputs
+        inputs_dict_st = attr.asdict(self.inputs)
+        if state_ind is not None:
+            for k, v in state_ind.items():
+                k = k.split(".")[1]
+                inputs_dict_st[k] = inputs_dict_st[k][v]
+
         cmd_add = []
-        if field.type is bool:
+        # formatter that creates a custom command argument
+        # it can thake the value of the filed, all inputs, or the value of other fields.
+        if "formatter" in field.metadata:
+            call_args = inspect.getargspec(field.metadata["formatter"])
+            call_args_val = {}
+            for argnm in call_args.args:
+                if argnm == "field":
+                    call_args_val[argnm] = value
+                elif argnm == "inputs":
+                    call_args_val[argnm] = inputs_dict_st
+                else:
+                    if argnm in inputs_dict_st:
+                        call_args_val[argnm] = inputs_dict_st[argnm]
+                    else:
+                        raise AttributeError(
+                            f"arguments of the formatter function from {field.name} "
+                            f"has to be in inputs or be field or output_dir, "
+                            f"but {argnm} is used"
+                        )
+            cmd_el_str = field.metadata["formatter"](**call_args_val)
+            cmd_el_str = cmd_el_str.strip().replace("  ", " ")
+            if cmd_el_str != "":
+                cmd_add += cmd_el_str.split(" ")
+        elif field.type is bool:
             # if value is simply True the original argstr is used,
             # if False, nothing is added to the command
             if value is True:

pydra/engine/tests/test_boutiques.py

@@ -1,13 +1,15 @@
 import os, shutil
 import subprocess as sp
 from pathlib import Path
+import attr
 import pytest
 
 from ..core import Workflow
 from ..task import ShellCommandTask
 from ..submitter import Submitter
 from ..boutiques import BoshTask
 from .utils import result_no_submitter, result_submitter, no_win
+from ...engine.specs import File
 
 need_bosh_docker = pytest.mark.skipif(
     shutil.which("docker") is None
@@ -36,12 +38,11 @@ def test_boutiques_1(maskfile, plugin, results_function, tmpdir):
 
     assert res.output.return_code == 0
 
-    # checking if the outfile exists and if it has proper name
+    # checking if the outfile exists and if it has a proper name
     assert res.output.outfile.name == "test_brain.nii.gz"
     assert res.output.outfile.exists()
-    # other files should also have proper names, but they do not exist
-    assert res.output.out_outskin_off.name == "test_brain_outskin_mesh.off"
-    assert not res.output.out_outskin_off.exists()
+    # files that do not exist were set to NOTHING
+    assert res.output.out_outskin_off == attr.NOTHING
 
 
 @no_win