test: reorganize pipeline scenario tests

Author: edmundmiller

tests/AGENTS.md

@@ -0,0 +1,27 @@
+# tests/
+
+## Layout
+Each pipeline-level scenario lives in its own directory:
+
+- `pipeline_api_only/`
+- `pipeline_mixed_no_benchmark/`
+- `pipeline_benchmark_tarball/`
+- `pipeline_benchmark_directory/`
+
+Each scenario directory contains:
+- `main.nf.test` — the nf-test scenario
+- `main.nf.test.snap` — the snapshot for that scenario
+- `AGENTS.md` — scenario-specific guidance for future edits
+
+## Conventions
+- One scenario per directory.
+- Keep assertions local and explicit rather than heavily abstracted.
+- Prefer stable fixture files under `workflows/nf_aggregate/assets/`.
+- If routing behavior changes, update the scenario-specific `AGENTS.md` along with the test.
+
+## Running tests
+Run all pipeline-level scenario tests with:
+
+```bash
+find tests -name 'main.nf.test' | sort | xargs nf-test test --profile=+docker --verbose
+```

tests/default.nf.test

@@ -0,0 +1,16 @@
+# pipeline_api_only
+
+## Purpose
+Covers the API-only input path when `generate_benchmark_report` is disabled.
+
+## Fixtures
+- `workflows/nf_aggregate/assets/test_run_ids.csv`
+
+## Expected behavior
+- No Nextflow processes should run.
+- A warning should explain that API runs produce no output without benchmark generation.
+- Only `pipeline_info/collated_software_versions.yml` should be emitted.
+- No `benchmark_report/` directory should exist.
+
+## Edit guidance
+If you change API-only routing, benchmark gating, or warning text, update this test first.

tests/pipeline_api_only/AGENTS.md

@@ -0,0 +1,49 @@
+def INPUT_PATH = new File('workflows/nf_aggregate/assets/test_run_ids.csv').canonicalPath
+
+def expectedOutputNames = [
+    'pipeline_info',
+    'pipeline_info/collated_software_versions.yml',
+]
+
+def collatedVersions = { outdir ->
+    new File("${outdir}/pipeline_info/collated_software_versions.yml").text
+}
+
+nextflow_pipeline {
+    name "Pipeline API-only without benchmark"
+    script "../../main.nf"
+    tag "pipeline"
+    tag "nf-aggregate"
+    tag "api-only"
+
+    test("api-only input skips benchmark work") {
+        tag("test")
+
+        when {
+            params {
+                input  = INPUT_PATH
+                outdir = "$outputDir"
+            }
+        }
+
+        then {
+            def versions = collatedVersions(params.outdir)
+            def stdout = workflow.stdout.join('\n')
+
+            assert workflow.success
+            assert workflow.trace.succeeded().size() == 0
+            assert !stdout.contains('Invalid method invocation')
+            assert stdout.contains('Found 3 API run(s) but --generate_benchmark_report is not enabled.')
+            assert versions.contains('Workflow:')
+            assert !versions.contains('EXTRACT_TARBALL:')
+            assert !versions.contains('BENCHMARK_REPORT:')
+            assert !new File("${params.outdir}/benchmark_report").exists()
+
+            assert snapshot(
+                workflow.trace.succeeded().size(),
+                removeFromYamlMap("$outputDir/pipeline_info/collated_software_versions.yml", "Workflow"),
+                expectedOutputNames
+            ).match()
+        }
+    }
+}

tests/pipeline_api_only/main.nf.test

@@ -0,0 +1,19 @@
+{
+    "api-only input skips benchmark work": {
+        "content": [
+            0,
+            {
+                
+            },
+            [
+                "pipeline_info",
+                "pipeline_info/collated_software_versions.yml"
+            ]
+        ],
+        "meta": {
+            "nf-test": "0.9.2",
+            "nextflow": "26.03.1"
+        },
+        "timestamp": "2026-04-06T21:16:01.182514"
+    }
+}

tests/pipeline_api_only/main.nf.test.snap

@@ -0,0 +1,17 @@
+# pipeline_benchmark_directory
+
+## Purpose
+Covers benchmark generation when external logs are provided as an already-extracted directory.
+
+## Fixtures
+- `workflows/nf_aggregate/assets/test_benchmark_directory.csv`
+- JSON fixture directory under `workflows/nf_aggregate/assets/log_dirs/`
+
+## Expected behavior
+- `EXTRACT_TARBALL` must not run.
+- `BENCHMARK_REPORT` should consume the directory directly.
+- Benchmark outputs should include both HTML and DuckDB artifacts.
+- The rendered HTML should mention the expected run ID and group.
+
+## Edit guidance
+If you change support for directory-based external logs, update this test first.

tests/pipeline_benchmark_directory/AGENTS.md

@@ -0,0 +1,61 @@
+def INPUT_PATH = new File('workflows/nf_aggregate/assets/test_benchmark_directory.csv').canonicalPath
+
+def expectedOutputNames = [
+    'benchmark_report',
+    'benchmark_report/benchmark.duckdb',
+    'benchmark_report/benchmark_report.html',
+    'pipeline_info',
+    'pipeline_info/collated_software_versions.yml',
+]
+
+def collatedVersions = { outdir ->
+    new File("${outdir}/pipeline_info/collated_software_versions.yml").text
+}
+
+nextflow_pipeline {
+    name "Pipeline benchmark from directory logs"
+    script "../../main.nf"
+    tag "pipeline"
+    tag "nf-aggregate"
+    tag "benchmark-directory"
+
+    test("benchmark report consumes external log directory directly") {
+        tag("benchmark")
+        tag("routing")
+
+        when {
+            params {
+                input                     = INPUT_PATH
+                outdir                    = "$outputDir"
+                generate_benchmark_report = true
+            }
+        }
+
+        then {
+            def versions = collatedVersions(params.outdir)
+            def stdout = workflow.stdout.join('\n')
+            def report = new File("${params.outdir}/benchmark_report/benchmark_report.html")
+            def database = new File("${params.outdir}/benchmark_report/benchmark.duckdb")
+            def reportText = report.text
+
+            assert workflow.success
+            assert workflow.trace.succeeded().size() == 1
+            assert !stdout.contains('Invalid method invocation')
+            assert !stdout.contains('SEQERALABS_NF_AGGREGATE:NF_AGGREGATE:EXTRACT_TARBALL')
+            assert stdout.contains('SEQERALABS_NF_AGGREGATE:NF_AGGREGATE:BENCHMARK_REPORT')
+            assert !versions.contains('EXTRACT_TARBALL:')
+            assert versions.contains('BENCHMARK_REPORT:')
+            assert report.isFile()
+            assert database.isFile()
+            assert reportText.contains('Benchmark overview')
+            assert reportText.contains('15dvxY1LnZYrYe')
+            assert reportText.contains('g5')
+
+            assert snapshot(
+                workflow.trace.succeeded().size(),
+                removeFromYamlMap("$outputDir/pipeline_info/collated_software_versions.yml", "Workflow"),
+                expectedOutputNames
+            ).match()
+        }
+    }
+}

tests/pipeline_benchmark_directory/main.nf.test

@@ -0,0 +1,25 @@
+{
+    "benchmark report consumes external log directory directly": {
+        "content": [
+            1,
+            {
+                "BENCHMARK_REPORT": {
+                    "duckdb": "1.3.2",
+                    "python": "3.12.13"
+                }
+            },
+            [
+                "benchmark_report",
+                "benchmark_report/benchmark.duckdb",
+                "benchmark_report/benchmark_report.html",
+                "pipeline_info",
+                "pipeline_info/collated_software_versions.yml"
+            ]
+        ],
+        "meta": {
+            "nf-test": "0.9.2",
+            "nextflow": "26.03.1"
+        },
+        "timestamp": "2026-04-06T21:16:23.609417"
+    }
+}

tests/pipeline_benchmark_directory/main.nf.test.snap

@@ -0,0 +1,17 @@
+# pipeline_benchmark_tarball
+
+## Purpose
+Covers benchmark generation when external logs are supplied as tarballs.
+
+## Fixtures
+- `workflows/nf_aggregate/assets/test_benchmark.csv`
+- tarballs under `workflows/nf_aggregate/assets/logs/`
+
+## Expected behavior
+- `EXTRACT_TARBALL` should run for each external input.
+- `BENCHMARK_REPORT` should run once after collection.
+- Benchmark outputs should include both HTML and DuckDB artifacts.
+- The rendered HTML should mention the expected run IDs and groups.
+
+## Edit guidance
+If you change tarball extraction, benchmark aggregation, or HTML report content, update this test.

tests/pipeline_benchmark_tarball/AGENTS.md

@@ -0,0 +1,65 @@
+def INPUT_PATH = new File('workflows/nf_aggregate/assets/test_benchmark.csv').canonicalPath
+
+def expectedOutputNames = [
+    'benchmark_report',
+    'benchmark_report/benchmark.duckdb',
+    'benchmark_report/benchmark_report.html',
+    'pipeline_info',
+    'pipeline_info/collated_software_versions.yml',
+]
+
+def collatedVersions = { outdir ->
+    new File("${outdir}/pipeline_info/collated_software_versions.yml").text
+}
+
+nextflow_pipeline {
+    name "Pipeline benchmark from tarballs"
+    script "../../main.nf"
+    tag "pipeline"
+    tag "nf-aggregate"
+    tag "benchmark-tarball"
+
+    test("benchmark report includes extracted tarball runs") {
+        tag("benchmark")
+
+        when {
+            params {
+                input                     = INPUT_PATH
+                outdir                    = "$outputDir"
+                generate_benchmark_report = true
+            }
+        }
+
+        then {
+            def versions = collatedVersions(params.outdir)
+            def stdout = workflow.stdout.join('\n')
+            def report = new File("${params.outdir}/benchmark_report/benchmark_report.html")
+            def database = new File("${params.outdir}/benchmark_report/benchmark.duckdb")
+            def reportText = report.text
+
+            assert workflow.success
+            assert workflow.trace.succeeded().size() == 3
+            assert !stdout.contains('Invalid method invocation')
+            assert stdout.contains('SEQERALABS_NF_AGGREGATE:NF_AGGREGATE:EXTRACT_TARBALL')
+            assert stdout.contains('SEQERALABS_NF_AGGREGATE:NF_AGGREGATE:BENCHMARK_REPORT')
+            assert versions.contains('EXTRACT_TARBALL:')
+            assert versions.contains('BENCHMARK_REPORT:')
+            assert report.isFile()
+            assert database.isFile()
+            assert reportText.contains('Benchmark overview')
+            assert reportText.contains('Run overview')
+            assert reportText.contains('Process overview')
+            assert reportText.contains('Task overview')
+            assert reportText.contains('15dvxY1LnZYrYe')
+            assert reportText.contains('5ymDk0hmgqv4L4')
+            assert reportText.contains('g5')
+            assert reportText.contains('cpu')
+
+            assert snapshot(
+                workflow.trace.succeeded().size(),
+                removeFromYamlMap("$outputDir/pipeline_info/collated_software_versions.yml", "Workflow"),
+                expectedOutputNames
+            ).match()
+        }
+    }
+}