Follow symlinks recursively

bids2table/_indexing.py

@@ -8,11 +8,14 @@
 import fnmatch
 import importlib.metadata
 import re
+import sys
 from concurrent.futures import Executor, ProcessPoolExecutor
 from functools import lru_cache, partial
+from glob import glob
 from typing import Any, Callable, Generator, Iterable, Sequence
 
 import pyarrow as pa
+from cloudpathlib import CloudPath
 from tqdm import tqdm
 
 from ._entities import (
@@ -400,7 +403,16 @@
     _, subject = path.name.split("-", maxsplit=1)
 
     records = []
-    for p in path.rglob("sub-*"):
+    # Use built-in rglob methods for CloudPath and py3.13+
+    if isinstance(path, CloudPath):
+        paths = map(as_path, path.rglob("sub-*"))
+    elif sys.version_info >= (3, 13):
+        paths = map(as_path, path.rglob("sub-*", recurse_symlinks=True))
+    else:
+        # Fall back to glob.glob for <py3.13
+        paths = map(as_path, glob(f"{path}/**/sub-*", recursive=True))
+
+    for p in paths:
         if _is_bids_file(p):
             entities = _cache_parse_bids_entities(p)
             valid_entities, extra_entities = validate_bids_entities(entities)

tests/conftest.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def symlink_dataset(tmp_path: Path, sub: str = "sub-001", ses: str = "ses-001") -> Path:
+    data = tmp_path / "data"
+    data.mkdir()
+
+    # Directory paths
+    ds = data / "dataset" / "bids" / sub / ses
+    ds_desc = data / "dataset" / "bids" / "dataset_description.json"
+    ds_dwi = ds / "dwi"
+    d1_anat = data / "directory1" / "bids" / sub / ses / "anat"
+    d2 = data / "directory2"
+
+    # Create directories
+    ds_dwi.mkdir(parents=True)
+    d1_anat.mkdir(parents=True)
+    d2.mkdir(parents=True)
+
+    # Create files to be symlinked
+    fnames = ["T1w.json", "T1w.nii.gz", "mask.nii.gz"]
+    for fname in fnames:
+        (d2 / fname).touch()
+
+    # dataset1
+    for suffix in [".bval", ".bvec", ".json", ".nii.gz"]:
+        (ds_dwi / f"{sub}_{ses}_run-1_dwi{suffix}").touch()
+    for fpath, suffix in zip(
+        [d2 / fname for fname in fnames],
+        ["_T1w.json", "_T1w.nii.gz", "_desc-T1w_mask.nii.gz"],
+    ):
+        (d1_anat / f"{sub}_{ses}_run-1{suffix}").symlink_to(fpath)
+    (ds / "anat").symlink_to(d1_anat, target_is_directory=True)
+    ds_desc.write_text('{"Name": "Dataset 1"}')
+
+    # dataset2
+    ds2 = data / "dataset2" / "bids"
+    ds2.mkdir(parents=True)
+    (ds2 / "dataset_description.json").write_text('{"Name": "Dataset 2"}')
+    (ds2 / sub).symlink_to(ds.parent, target_is_directory=True)
+
+    # dataset3
+    ds3 = data / "dataset3"
+    ds3.mkdir(parents=True)
+    (ds3 / "bids").symlink_to(data / "dataset" / "bids", target_is_directory=True)
+
+    return data

tests/test_indexing.py

@@ -120,6 +120,15 @@ def test_batch_index_dataset(max_workers: int):
     assert len(table) == 9727
 
 
+@pytest.mark.parametrize("ds_name", ["dataset", "dataset2", "dataset3"])
+def test_indexing_on_symlinks(symlink_dataset: Path, ds_name: str):
+    tables = indexing.batch_index_dataset(
+        indexing.find_bids_datasets(symlink_dataset / ds_name), show_progress=False
+    )
+    table = pa.concat_tables(tables)
+    assert len(table) == 5
+
+
 @pytest.mark.parametrize(
     "path,expected_name",
     [