try out codspeed runners and add more realistic tests (#4395)

Author: mashehu

.github/workflows/codspeed.yml

@@ -22,7 +22,7 @@ permissions:
 jobs:
   startup-benchmarks:
     name: Run start-up benchmarks
-    runs-on: ubuntu-latest
+    runs-on: codspeed-macro
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
         name: Check out source-code repository
@@ -32,6 +32,11 @@ jobs:
         with:
           python-version: "3.10"
 
+      - name: Install Nextflow
+        uses: nf-core/setup-nextflow@b4ec1bc7c16a94435159de94a05253542fddf6ef # v3
+        with:
+          version: ${{ github.event.inputs.nextflow-version || matrix.nextflow-version }}
+
       - name: Install uv
         uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2
         with:
@@ -40,6 +45,13 @@ jobs:
       - name: Install dependencies
         run: uv sync --all-extras
 
+      # Clone nf-core/modules into the local cache here, while network is
+      # available. The modules-install benchmark then runs entirely offline
+      # (no per-round fetch), so it stays deterministic even if the CodSpeed
+      # step sandboxes network during measurement.
+      - name: Warm nf-core/modules cache
+        run: uv run python -c "from nf_core.modules.modules_repo import ModulesRepo; ModulesRepo()"
+
       # These benchmarks spawn a fresh interpreter per round to measure
       # import/start-up cost, which CodSpeed's instruction-counting
       # "simulation" instrument can't see. Use "walltime" mode instead.

benchmarks/test_startup_benchmarks.py

@@ -10,6 +10,7 @@
 """
 
 import os
+import shutil
 import subprocess
 import sys
 
@@ -34,6 +35,39 @@ def _cli_script(setup, args):
     return f"{setup}\nimport sys\nsys.argv = {args!r}\nfrom nf_core.__main__ import run_nf_core\nrun_nf_core()"
 
 
+@pytest.fixture(scope="session")
+def generated_pipeline(tmp_path_factory):
+    """Render a real pipeline scaffold once, reused by the realistic benchmarks.
+
+    Generation is expensive and identical for every consumer, so it happens a
+    single time per session and outside any measured rounds. Consumers that
+    mutate the pipeline (e.g. installing a module) must work on a copy so the
+    shared scaffold stays pristine and the suite is order-independent.
+    """
+    from nf_core.pipelines.create.create import PipelineCreate
+
+    pipeline_dir = tmp_path_factory.mktemp("pipeline_fixture") / "pipeline"
+    PipelineCreate("benchmark", "benchmark pipeline", "nf-core", outdir=pipeline_dir, no_git=True).init_pipeline()
+    return pipeline_dir
+
+
+@pytest.fixture(scope="session")
+def modules_cache():
+    """Ensure a local nf-core/modules clone exists, without ever fetching.
+
+    Constructing ``ModulesRepo`` clones the remote into ``NFCORE_DIR`` only if it
+    is not already present; ``no_pull_global`` suppresses the ``git fetch`` on an
+    existing clone. In CI the clone is warmed by an earlier, network-enabled
+    workflow step, so this fixture (and every measured round) runs offline. When
+    run locally against a cold cache it performs the one-off clone here, outside
+    the measured rounds.
+    """
+    from nf_core.modules.modules_repo import ModulesRepo
+
+    ModulesRepo.no_pull_global = True
+    ModulesRepo()
+
+
 def test_import_main_startup(benchmark):
     """Wall-time cost of importing the CLI entry point in a fresh interpreter."""
     _benchmark_command(benchmark, IMPORT_CMD)
@@ -71,16 +105,32 @@ def test_pipelines_create_startup(benchmark, tmp_path):
     _benchmark_command(benchmark, [sys.executable, "-c", _cli_script(setup, args)])
 
 
-def test_modules_install_startup(benchmark, tmp_path):
-    """Start module installation, stubbing only the remote clone and resulting writes."""
-    args = ["nf-core", "modules", "install", "fastp", "-d", str(tmp_path)]
-    setup = """import nf_core.modules.install as install_module
-install_module.ModuleInstall = type(
-    "ModuleInstall",
-    (),
-    {
-        "__init__": lambda self, *args, **kwargs: None,
-        "install": lambda self, *args, **kwargs: True,
-    },
-)"""
+def test_modules_install_startup(benchmark, generated_pipeline, modules_cache, tmp_path):
+    """Install a real module into a real pipeline, offline.
+
+    The pipeline scaffold (``generated_pipeline``) and the nf-core/modules clone
+    (``modules_cache``) are both prepared once, outside the measured rounds. The
+    rounds then run the real install (git checkout + file copy + modules.json
+    write) against a private copy of the scaffold. ``no_pull_global`` suppresses
+    the per-round ``git fetch`` and ``--force`` avoids the re-install prompt, so
+    every round is deterministic and network-free.
+    """
+    target = tmp_path / "pipeline"
+    shutil.copytree(generated_pipeline, target)
+    args = ["nf-core", "modules", "install", "fastp", "-d", str(target), "--force"]
+    setup = "from nf_core.modules.modules_repo import ModulesRepo\nModulesRepo.no_pull_global = True"
+    _benchmark_command(benchmark, [sys.executable, "-c", _cli_script(setup, args)])
+
+
+def test_pipelines_schema_lint_startup(benchmark, generated_pipeline):
+    """Lint a real, freshly generated pipeline schema.
+
+    The schema fixture is rendered once by ``generated_pipeline`` (outside the
+    measured rounds); the benchmark then runs the real lint (JSON parse +
+    JSON-Schema validation + default-param checks). Only ``fetch_wf_config`` is
+    stubbed to skip the ``nextflow config`` JVM subprocess, which would
+    otherwise dominate wall time and require Nextflow on the runner.
+    """
+    args = ["nf-core", "pipelines", "schema", "lint", str(generated_pipeline)]
+    setup = "import nf_core.utils\nnf_core.utils.fetch_wf_config = lambda *args, **kwargs: {}"
     _benchmark_command(benchmark, [sys.executable, "-c", _cli_script(setup, args)])