feat: add helper functions to parse input files (#2918)

<!--Add a description of your PR here-->

Add helper functions to parse input files. See example in
snakemake/snakemake-wrappers#2725.

### QC
<!-- Make sure that you can tick the boxes below. -->

* [x] The PR contains a test case for the changes or the changes are
already covered by an existing test case.
* [x] The documentation (`docs/`) is updated to reflect the changes or
this is not necessary (e.g. if the change does neither modify the
language nor the behavior or functionalities of Snakemake).


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Release Notes

- **New Features**
  - Added integration of Xonsh scripts into Snakemake workflows.
- Introduced a new rule for executing Python scripts within a Conda
environment.
- Enhanced report generation capabilities, including self-contained HTML
and ZIP archives.
- Expanded input handling capabilities with new methods for parsing
inputs and extracting checksums.

- **Bug Fixes**
- Improved checksum extraction and validation processes in existing
workflows.

- **Documentation**
- Expanded documentation to include details on Xonsh integration and
report generation features.
- Clarified usage of conda environments and apptainer integration within
Snakemake.

- **Tests**
- Added new test cases for Xonsh script execution and Conda deployment
scenarios.
- Updated existing tests to validate checksum functionality and improve
input handling.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Johannes Köster <[email protected]>

Author: fgvieira

docs/snakefiles/rules.rst

@@ -547,6 +547,49 @@ It can for example be used to condition some behavior in the workflow on the exi
         shell:
             "cp {input} {output}"
 
+.. _snakefiles-semantic-helpers-parse-input:
+
+The parse_input function
+""""""""""""""""""""""""
+
+The ``parse_input`` function allows to parse an input file and return a value.
+It has the signature ``parse_input(input_item, parser, kwargs)``, with ``input_item`` being the key of an input file, ``parser`` being a callable to extract the desired information, and ``kwargs`` extra arguments passed to the parser.
+The function will return the extracted value.
+It can for example be used to extract a value from inside an input file.
+
+.. code-block:: python
+
+    rule a:
+	input:
+	    samples="samples.tsv",
+        output:
+            "samples.id",
+        params:
+            id=parse_input(input.samples, parser=extract_id)
+        shell:
+            "echo {params.id} > {output}"
+
+
+.. _snakefiles-semantic-helpers-extract-checksum:
+
+The extract_checksum function
+"""""""""""""""""""""""""""""
+
+The ``extract_checksum`` function parses an input file and returns the checksum of the given file.
+It has the signature ``extract_checksum(infile, file)``, with ``infile`` being the input file, and ``file`` the filename to search for.
+The function will return the checksum of ``file`` present in ``infile``.
+
+.. code-block:: python
+
+    rule a:
+	input:
+	    checksum="samples.md5",
+        output:
+            tsv="{a}.tsv",
+        params:
+            checksum=parse_input(input.checksum, parser=extract_checksum, file=output.tsv)
+        shell:
+            "echo {params.checksum} > {output}"
 
 .. _snakefiles-rule-item-access:
 
@@ -3146,4 +3189,4 @@ The name is optional and can be left out, creating an anonymous rule. It can als
     Note that any placeholders in the shell command (like ``{input}``) are always evaluated and replaced
     when the corresponding job is executed, even if they are occurring inside a comment.
     To avoid evaluation and replacement, you have to mask the braces by doubling them,
-    i.e. ``{{input}}``.
+    i.e. ``{{input}}``.

snakemake/ioutils/__init__.py

@@ -5,6 +5,7 @@
 from snakemake.ioutils.lookup import lookup
 from snakemake.ioutils.rule_items_proxy import rule_item_factory
 from snakemake.ioutils.subpath import subpath
+from snakemake.ioutils.input import parse_input, extract_checksum
 
 
 def register_in_globals(_globals):
@@ -21,5 +22,7 @@ def register_in_globals(_globals):
             "resources": rule_item_factory("resources"),
             "threads": rule_item_factory("threads"),
             "subpath": subpath,
+            "parse_input": parse_input,
+            "extract_checksum": extract_checksum,
         }
     )

snakemake/ioutils/input.py

@@ -0,0 +1,30 @@
+def parse_input(infile, parser, **kwargs):
+    def inner(wildcards, input, output):
+        with open(infile, "r") as fh:
+            if parser is None:
+                return fh.read().strip()
+            else:
+                return parser(fh, **kwargs)
+
+    return inner
+
+
+def extract_checksum(infile, **kwargs):
+    try:
+        import pandas as pd
+
+        fix_file_name = lambda x: x.removeprefix("./")
+        return (
+            pd.read_csv(
+                infile,
+                sep="  ",
+                header=None,
+                engine="python",
+                converters={1: fix_file_name},
+            )
+            .set_index(1)
+            .loc[fix_file_name(kwargs.get("file"))]
+            .item()
+        )
+    except ImportError:
+        raise WorkflowError("Pandas is required to extract checksum from file.")

tests/test_ioutils/Snakefile

@@ -8,6 +8,9 @@ configfile: "config.yaml"
 
 assert lookup(dpath="does/not/exist", within=config, default=None) is None
 assert lookup(dpath="does/not/exist", within=config, default=5) == 5
+assert (
+    extract_checksum("samples.md5", file="1.tsv") == "9695eb6f38992d796551f4cb20d7d138"
+)
 
 
 rule all:
@@ -16,10 +19,16 @@ rule all:
 
 
 rule a:
+    input:
+        checksum="samples.md5",
     output:
         "a/{sample}.txt",
+    params:
+        checksum=lambda w, input, output: parse_input(
+            input.checksum, parser=extract_checksum, file=f"{w.sample}.tsv"
+        ),
     shell:
-        "echo a > {output}"
+        "echo {params.checksum} > {output}"
 
 
 rule b:
@@ -46,7 +55,7 @@ rule c:
 
 rule item_access:
     input:
-        txt="in.txt"
+        txt="in.txt",
     output:
         txt="test.txt",
     params:
@@ -73,4 +82,4 @@ rule e:
     output:
         "results/switch~{switch}.column~{col}.txt",
     shell:
-        "cat {input} > {output}"
+        "cat {input} > {output}"

tests/test_ioutils/expected-results/c/1.txt

@@ -1 +1 @@
-a
+9695eb6f38992d796551f4cb20d7d138

tests/test_ioutils/expected-results/results/switch~someswitch.column~sample.txt

@@ -1,3 +1,3 @@
-a
+9695eb6f38992d796551f4cb20d7d138
 b
 d

tests/test_ioutils/samples.md5

@@ -0,0 +1,5 @@
+5695eb6f38992d796551f4cb20d7d138  2.tsv
+6695eb6f38992d796551f4cb20d7d138  3.tsv
+7695eb6f38992d796551f4cb20d7d138  prefix_1.tsv
+8695eb6f38992d796551f4cb20d7d138  1_suffix.tsv
+9695eb6f38992d796551f4cb20d7d138  1.tsv