Add ds000114 fixture infrastructure for longitudinal integration tests

Adds the download script, conftest fixtures, and CI cache that the
Stage 2 longitudinal integration test (deferred from PR #306) and the
upcoming Stages 3-6 tier-2 tests need. Pins OpenNeuro snapshot 1.0.2
via sha256 of task-fingerfootlips_bold.json so silent upstream drift
(including the old missing-TaskName regression) is caught at fetch
time. Also lands the deferred rbc longitudinal template integration
test asserting the ses-longitudinal BIDS tree shape (template T1w plus
from-test / from-retest ITK xfms).

Refs #301, follow-up to #306.

Author: nx10

.github/workflows/test_full.yaml

@@ -31,6 +31,12 @@ jobs:
         enable-cache: true # not automatic on self-hosted runners
     - run: uv sync
 
+    - name: Cache ds000114 longitudinal fixture
+      uses: actions/cache@v5
+      with:
+        path: tests/data/ds000114
+        key: ds000114-${{ hashFiles('scripts/download_ds000114.py') }}
+
     - name: Run all tests
       shell: bash
       run: |

.gitignore

@@ -172,5 +172,6 @@ cython_debug/
 # Full-pipeline test artifacts
 tests/full_pipeline/.last_run.json
 tests/data/rbc_run/
+tests/data/ds000114/
 CLAUDE.md
 reports

scripts/download_ds000114.py

@@ -0,0 +1,166 @@
+# /// script
+# dependencies = []
+# requires-python = ">=3.12"
+# ///
+"""Download a sub-01 subset of OpenNeuro ds000114 for longitudinal tests.
+
+ds000114 ("Test-Retest Reliability") is the canonical multi-session BIDS
+demo dataset. We pull sub-01 only (both ses-test and ses-retest) plus the
+top-level sidecars, because BIDS inheritance places RepetitionTime etc.
+in the dataset-level task JSON, not per-subject.
+
+OpenNeuro snapshot: 1.0.2 (tagged 2022-08-24, verified to contain
+``TaskName`` in ``task-fingerfootlips_bold.json`` so bids-validator is
+happy). The public S3 mirror only exposes the latest state at top-level
+(``s3://openneuro.org/ds000114/...``), so we can't pin via a snapshot
+path. Instead we verify the sha256 of ``task-fingerfootlips_bold.json``
+after download and refuse to proceed if the content has drifted.
+
+Runs against the public HTTP S3 endpoint with no credentials and no
+third-party dependencies, so it works on any environment where
+``uv run`` does.
+
+Usage::
+
+    uv run scripts/download_ds000114.py [TARGET_DIR]
+
+Default TARGET_DIR is ``tests/data/ds000114`` (under the repo root).
+Idempotent: exits early if the sentinel T1w file already exists.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import logging
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+SNAPSHOT_TAG = "1.0.2"
+S3_BASE = "https://s3.amazonaws.com/openneuro.org"
+DATASET = "ds000114"
+SUBJECT = "sub-01"
+SIDECARS = (
+    "dataset_description.json",
+    "participants.tsv",
+    "task-fingerfootlips_bold.json",
+)
+TASK_JSON_SHA256 = "9fd44f65a772e05282c20bdfa2a9775e02f9a7f562c5c96bbf4fd30632540355"
+S3_XMLNS = "{http://s3.amazonaws.com/doc/2006-03-01/}"
+
+_DEFAULT_TARGET = Path(__file__).resolve().parent.parent / "tests" / "data" / DATASET
+_SENTINEL_REL = Path(SUBJECT) / "ses-test" / "anat" / f"{SUBJECT}_ses-test_T1w.nii.gz"
+
+_logger = logging.getLogger("download_ds000114")
+
+
+def _list_keys(prefix: str) -> list[str]:
+    """Enumerate all S3 object keys beneath ``prefix`` (paginated)."""
+    keys: list[str] = []
+    continuation: str | None = None
+    while True:
+        query = f"list-type=2&prefix={prefix}"
+        if continuation is not None:
+            query += f"&continuation-token={urllib.parse.quote(continuation)}"
+        with urllib.request.urlopen(f"{S3_BASE}/?{query}") as resp:  # noqa: S310
+            root = ET.parse(resp).getroot()  # noqa: S314 (trusted S3 response)
+        for contents in root.findall(f"{S3_XMLNS}Contents"):
+            key = contents.findtext(f"{S3_XMLNS}Key")
+            if key:
+                keys.append(key)
+        if root.findtext(f"{S3_XMLNS}IsTruncated") != "true":
+            return keys
+        continuation = root.findtext(f"{S3_XMLNS}NextContinuationToken")
+        if not continuation:
+            return keys
+
+
+def _download(key: str, dest: Path) -> None:
+    """Fetch ``key`` from the public openneuro S3 mirror to ``dest``."""
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    url = f"{S3_BASE}/{key}"
+    _logger.info("GET %s", key)
+    with (
+        urllib.request.urlopen(url) as resp,  # noqa: S310
+        dest.open("wb") as out,
+    ):
+        while chunk := resp.read(1 << 20):
+            out.write(chunk)
+
+
+def _sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as fh:
+        while chunk := fh.read(1 << 20):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def download(target_dir: Path) -> None:
+    """Download the ds000114 sub-01 subset into ``target_dir`` if missing."""
+    sentinel = target_dir / _SENTINEL_REL
+    if sentinel.exists():
+        _logger.info("%s: already present at %s", DATASET, target_dir)
+        return
+
+    _logger.info(
+        "%s: downloading %s (both sessions) + sidecars to %s (snapshot %s)",
+        DATASET,
+        SUBJECT,
+        target_dir,
+        SNAPSHOT_TAG,
+    )
+
+    subject_keys = _list_keys(f"{DATASET}/{SUBJECT}/")
+    if not subject_keys:
+        raise RuntimeError(
+            f"No objects found under {DATASET}/{SUBJECT}/; bucket layout may "
+            "have changed."
+        )
+    for key in subject_keys:
+        rel = Path(key).relative_to(DATASET)
+        _download(key, target_dir / rel)
+
+    for name in SIDECARS:
+        _download(f"{DATASET}/{name}", target_dir / name)
+
+    actual = _sha256(target_dir / "task-fingerfootlips_bold.json")
+    if actual != TASK_JSON_SHA256:
+        raise RuntimeError(
+            "task-fingerfootlips_bold.json sha256 mismatch; upstream may have "
+            f"changed.\n  expected: {TASK_JSON_SHA256}\n  actual:   {actual}\n"
+            f"  verify and update SNAPSHOT_TAG + TASK_JSON_SHA256."
+        )
+    _logger.info("%s: download complete", DATASET)
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Entry point. Returns a shell-style exit code."""
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument(
+        "target_dir",
+        nargs="?",
+        type=Path,
+        default=_DEFAULT_TARGET,
+        help=f"Destination directory (default: {_DEFAULT_TARGET}).",
+    )
+    args = parser.parse_args(argv)
+
+    logging.basicConfig(level=logging.INFO, format="%(message)s", stream=sys.stderr)
+    try:
+        download(args.target_dir)
+    except urllib.error.URLError as exc:
+        _logger.error("%s: network error: %s", DATASET, exc)
+        return 2
+    except RuntimeError as exc:
+        _logger.error("%s: %s", DATASET, exc)
+        return 3
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/integration/longitudinal/__init__.py

@@ -0,0 +1 @@
+"""Longitudinal processing integration tests."""

tests/integration/longitudinal/conftest.py

@@ -0,0 +1,103 @@
+"""Fixtures for longitudinal integration tests.
+
+These fixtures own the ds000114 test dataset lifecycle (download + reuse)
+and run the cross-sectional anatomical stage once so that each
+longitudinal test can build on ``desc-brain`` T1w derivatives without
+paying the preprocessing cost per test.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_DOWNLOAD_SCRIPT = _REPO_ROOT / "scripts" / "download_ds000114.py"
+_DATASET_DIR = _REPO_ROOT / "tests" / "data" / "ds000114"
+_DATASET_SENTINEL = (
+    _DATASET_DIR / "sub-01" / "ses-test" / "anat" / "sub-01_ses-test_T1w.nii.gz"
+)
+
+
+@pytest.fixture(scope="session")
+def ds000114_dataset() -> Path:
+    """Return the ds000114 BIDS dataset root, downloading it on first use.
+
+    Skips the calling test if the dataset can't be fetched (most commonly
+    because the S3 mirror is unreachable or the uv runner isn't available),
+    so local developers without network access don't see hard failures
+    while CI still exercises the path.
+    """
+    if _DATASET_SENTINEL.exists():
+        return _DATASET_DIR
+
+    if not _DOWNLOAD_SCRIPT.exists():
+        pytest.skip(f"download script missing: {_DOWNLOAD_SCRIPT}")
+
+    uv = shutil.which("uv")
+    if uv is None:
+        pytest.skip("uv not found on PATH; cannot run download script")
+
+    result = subprocess.run(  # noqa: S603
+        [uv, "run", str(_DOWNLOAD_SCRIPT), str(_DATASET_DIR)],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0 or not _DATASET_SENTINEL.exists():
+        pytest.skip(
+            "ds000114 download failed; skipping longitudinal tests.\n"
+            f"--- stdout ---\n{result.stdout[-1000:]}\n"
+            f"--- stderr ---\n{result.stderr[-1000:]}"
+        )
+    return _DATASET_DIR
+
+
+@pytest.fixture(scope="session")
+def runner_backend(request: pytest.FixtureRequest) -> str:
+    """Styx runner backend selected via ``--runner`` on the pytest CLI."""
+    return request.config.getoption("--runner")
+
+
+@pytest.fixture(scope="session")
+def ds000114_anat_derivatives(
+    ds000114_dataset: Path,
+    tmp_path_factory: pytest.TempPathFactory,
+    runner_backend: str,
+) -> Path:
+    """Run ``rbc anatomical`` once against ds000114 sub-01 (both sessions).
+
+    The longitudinal template stage consumes the cross-sectional
+    ``desc-brain`` T1w derivatives, so we produce them up front. Session
+    scope ensures we only pay the registration + brain extraction cost
+    once across all longitudinal integration tests.
+    """
+    rbc = shutil.which("rbc")
+    if rbc is None:
+        pytest.skip("rbc CLI not found on PATH")
+
+    out = tmp_path_factory.mktemp("ds000114_derivatives")
+    result = subprocess.run(  # noqa: S603
+        [
+            rbc,
+            "anatomical",
+            str(ds000114_dataset),
+            "-o",
+            str(out),
+            "--runner",
+            runner_backend,
+            "--participant-label",
+            "01",
+        ],
+        capture_output=True,
+        text=True,
+        timeout=3600,
+    )
+    assert result.returncode == 0, (
+        f"rbc anatomical exited with code {result.returncode}\n"
+        f"--- stdout ---\n{result.stdout[-2000:]}\n"
+        f"--- stderr ---\n{result.stderr[-2000:]}"
+    )
+    return out

tests/integration/longitudinal/test_template.py

@@ -0,0 +1,72 @@
+"""Integration test for ``rbc longitudinal template``.
+
+Deferred from Stage 2 of the longitudinal refactor (tracker #301,
+Stage 2 landed in PR #306). Depends on the ds000114 multi-session test
+fixture; cross-sectional anatomical derivatives are produced by the
+session-scoped ``ds000114_anat_derivatives`` fixture so the template
+stage has ``desc-brain`` T1w volumes to consume.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from typing import TYPE_CHECKING
+
+import pytest
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+@pytest.mark.slow
+def test_rbc_longitudinal_template_builds_bids_tree(
+    ds000114_anat_derivatives: Path,
+    runner_backend: str,
+) -> None:
+    """Run ``rbc longitudinal template`` and verify the BIDS output tree."""
+    rbc = shutil.which("rbc")
+    assert rbc is not None, "rbc CLI not found on PATH"
+
+    result = subprocess.run(  # noqa: S603
+        [
+            rbc,
+            "longitudinal",
+            "template",
+            str(ds000114_anat_derivatives),
+            "-o",
+            str(ds000114_anat_derivatives),
+            "--runner",
+            runner_backend,
+            "--participant-label",
+            "01",
+        ],
+        capture_output=True,
+        text=True,
+        timeout=3600,
+    )
+    assert result.returncode == 0, (
+        f"rbc longitudinal template exited with code {result.returncode}\n"
+        f"--- stdout ---\n{result.stdout[-2000:]}\n"
+        f"--- stderr ---\n{result.stderr[-2000:]}"
+    )
+
+    ses_long = ds000114_anat_derivatives / "sub-01" / "ses-longitudinal" / "anat"
+    expected = [
+        "sub-01_ses-longitudinal_T1w.nii.gz",
+        "sub-01_ses-longitudinal_from-test_to-longitudinal_mode-image_xfm.txt",
+        "sub-01_ses-longitudinal_from-retest_to-longitudinal_mode-image_xfm.txt",
+    ]
+    missing = [name for name in expected if not (ses_long / name).is_file()]
+    if missing:
+        tree = sorted(
+            str(p.relative_to(ds000114_anat_derivatives))
+            for p in ds000114_anat_derivatives.rglob("*")
+            if p.is_file()
+        )
+        pytest.fail(
+            "Missing expected longitudinal derivatives:\n  "
+            + "\n  ".join(missing)
+            + "\n--- file tree ---\n"
+            + "\n".join(tree)
+        )