Add docstrings and update README.md

Author: clane9

README.md

@@ -1,9 +1,8 @@
 # bids2table
-<!-- [![CI](https://github.com/childmindresearch/bids2table/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/childmindresearch/bids2table/actions/workflows/ci.yaml?query=branch%3Amain)
-[![Documentation](https://img.shields.io/badge/documentation-8CA1AF?logo=readthedocs&logoColor=fff)](https://childmindresearch.github.io/bids2table)
-[![codecov](https://codecov.io/gh/childmindresearch/bids2table/branch/main/graph/badge.svg?token=22HWWFWPW5)](https://codecov.io/gh/childmindresearch/bids2table) -->
+[![CI](https://github.com/childmindresearch/bids2table/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/childmindresearch/bids2table/actions/workflows/ci.yaml?query=branch%3Amain)
+[![codecov](https://codecov.io/gh/childmindresearch/bids2table/branch/main/graph/badge.svg?token=22HWWFWPW5)](https://codecov.io/gh/childmindresearch/bids2table)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
-![Python3](https://img.shields.io/badge/python->=3.11-blue.svg)
+![Python3](https://img.shields.io/badge/python->=3.12-blue.svg)
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 
 Index BIDS datasets fast, locally or in the cloud.
@@ -13,5 +12,102 @@ Index BIDS datasets fast, locally or in the cloud.
 The latest development version can be installed with
 
 ```sh
-pip install git+https://github.com/childmindresearch/bids2table.git@develop/b2t2
+pip install "bids2table @ git+https://github.com/childmindresearch/bids2table.git@develop/b2t2"
+```
+
+To install with S3 support, include the `s3` extra
+
+```sh
+pip install "bids2table[s3] @ git+https://github.com/childmindresearch/bids2table.git@develop/b2t2"
+```
+
+## Usage
+
+### Finding BIDS datasets
+
+You can search a directory for valid BIDS datasets using `b2t2 find`
+
+```
+(bids2table) clane$ b2t2 find bids-examples | head -n 10
+bids-examples/asl002
+bids-examples/ds002
+bids-examples/ds005
+bids-examples/asl005
+bids-examples/ds051
+bids-examples/eeg_rishikesh
+bids-examples/asl004
+bids-examples/asl003
+bids-examples/ds003
+bids-examples/eeg_cbm
+```
+
+### Indexing datasets from the command line
+
+Indexing datasets is done with `b2t2 index`. Here we index a single example dataset, saving the output as a parquet file.
+
+```
+(bids2table) clane$ b2t2 index -v -o ds102.parquet bids-examples/ds102
+ds102: 100%|███████████████████████████████████████| 26/26 [00:00<00:00, 154.12it/s, sub=26, N=130]
+```
+
+You can also index a list of datasets. Note that each iteration in the progress bar represents one dataset.
+
+```
+(bids2table) clane$ b2t2 index -v -o bids-examples.parquet bids-examples/*
+100%|████████████████████████████████████████████| 87/87 [00:00<00:00, 113.59it/s, ds=None, N=9727]
+```
+
+You can pipe the output of `b2t2 find` to `b2t2 index` to create an index of all datasets under a root directory.
+
+```
+(bids2table) clane$ b2t2 find bids-examples | b2t2 index -v -o bids-examples.parquet
+97it [00:01, 96.05it/s, ds=ieeg_filtered_speech, N=10K]
+```
+
+The resulting index will include both top-level datasets (as in the previous command) as well nested derivatives datasets.
+
+### Indexing datasets hosted on S3
+
+bids2table supports indexing datasets hosted on S3 via [cloudpathlib](https://github.com/drivendataorg/cloudpathlib). To use this functionality, install cloudpathlib with S3 support
+
+```sh
+pip install cloudpathlib[s3]
+```
+
+You can also install bids2table with the s3 extra
+
+```sh
+pip install "bids2table[s3] @ git+https://github.com/childmindresearch/bids2table.git@develop/b2t2"
+```
+
+As an example, here we index all datasets on [OpenNeuro](https://openneuro.org/)
+
+```
+(bids2table) clane$ b2t2 index -v -o openneuro.parquet \
+  -j 8 --use-threads s3://openneuro.org/ds*
+100%|█████████████████████████████████████| 1408/1408 [12:25<00:00,  1.89it/s, ds=ds006193, N=1.2M]
+```
+
+Using 8 threads, we can index all ~1400 OpenNeuro datasets (1.2M files) in less than 15 minutes.
+
+
+### Indexing datasets from python
+
+You can also index datasets in Python using the Python API.
+
+```python
+import pyarrow as pa
+import bids2table as b2t2
+
+# Index a single dataset.
+tab = b2t2.index_dataset("bids-examples/ds102")
+
+# Find and index a batch of datasets.
+tabs = b2t2.batch_index_dataset(
+    b2t2.find_bids_datasets("bids-examples"),
+)
+tab = pa.concat_tables(tabs)
+
+# Index a dataset on S3.
+tab = b2t2.index_dataset("s3://openneuro.org/ds000224")
 ```

bids2table/__init__.py

@@ -7,12 +7,14 @@
     set_bids_schema,
     get_bids_schema,
     get_bids_entity_arrow_schema,
+    format_bids_path,
 )
 from ._indexing import (
     find_bids_datasets,
     index_dataset,
     batch_index_dataset,
     get_arrow_schema,
+    get_column_names,
 )
 from ._pathlib import Path, cloudpathlib_is_available
 from ._version import *

bids2table/_entities.py

@@ -74,7 +74,7 @@
 _logger = setup_logger(__package__)
 
 
-def set_bids_schema(path: str | Path | None = None):
+def set_bids_schema(path: str | Path | None = None) -> None:
     """Set the BIDS schema."""
     global _BIDS_SCHEMA, _BIDS_ENTITY_SCHEMA, _BIDS_NAME_ENTITY_MAP
     global _BIDS_ENTITY_ARROW_SCHEMA
@@ -134,12 +134,17 @@ def get_bids_entity_arrow_schema() -> pa.Schema:
     return _BIDS_ENTITY_ARROW_SCHEMA
 
 
-@lru_cache()
 def parse_bids_entities(path: str | Path) -> dict[str, str]:
     """Parse entities from BIDS file path.
 
     Parses all BIDS filename `"{key}-{value}"` entities as well as special entities:
     datatype, suffix, ext (extension). Does not validate entities or cast to types.
+
+    Args:
+        path: BIDS path to parse.
+
+    Returns:
+        entities: dict mapping BIDS entity keys to values.
     """
     if isinstance(path, str):
         path = Path(path)
@@ -175,6 +180,11 @@ def parse_bids_entities(path: str | Path) -> dict[str, str]:
     return entities
 
 
+# Version with caching to use internally. Decorating the public function loses the
+# docstring.
+_cache_parse_bids_entities = lru_cache(parse_bids_entities)
+
+
 def _parse_bids_datatype(path: Path) -> str | None:
     """Parse BIDS datatype from file path.
 
@@ -192,8 +202,14 @@ def validate_bids_entities(
     """Validate BIDS entities.
 
     Validates the type and allowed values of each entity against the BIDS schema.
-    Returns a tuple of `valid_entities` as well as any leftover `extra_entities` that
-    don't match a known entity.
+
+    Args:
+        entities: dict mapping BIDS keys to unvalidated entities
+
+    Returns:
+        valid_entities: A mapping of valid BIDS keys to type-casted values.
+        extra_entities: A mapping of any leftover entity mappings that didn't match a
+            known entity or failed validation.
     """
     valid_entities = {}
     extra_entities = {}
@@ -233,7 +249,12 @@ def validate_bids_entities(
 def format_bids_path(entities: dict[str, Any], int_format: str = "%d") -> Path:
     """Construct a formatted BIDS path from entities dict.
 
-    Integer (index) value indices are formatted using the `int_format`.
+    Args:
+        entities: dict mapping BIDS keys to values.
+        int_format: format string for integer (index) BIDS values.
+
+    Returns:
+        path: formatted `Path` instance.
     """
     special = {"datatype", "suffix", "ext"}
 

bids2table/_indexing.py

@@ -16,8 +16,8 @@
 from tqdm import tqdm
 
 from ._entities import (
+    _cache_parse_bids_entities,
     get_bids_entity_arrow_schema,
-    parse_bids_entities,
     validate_bids_entities,
 )
 from ._logging import setup_logger
@@ -379,7 +379,7 @@ def _index_bids_subject_dir(
 
     records = []
     for p in _find_bids_files(path):
-        entities = parse_bids_entities(p)
+        entities = _cache_parse_bids_entities(p)
         valid_entities, extra_entities = validate_bids_entities(entities)
         record = {
             "dataset": dataset,
@@ -412,7 +412,7 @@ def _is_bids_file(path: Path) -> bool:
     if path.suffix == "" or not path.name.startswith("sub-"):
         return False
 
-    entities = parse_bids_entities(path)
+    entities = _cache_parse_bids_entities(path)
     # if not (entities.get("suffix") and entities.get("datatype")):
     if not (entities.get("suffix") and entities.get("ext")):
         return False
@@ -435,7 +435,7 @@ def _is_bids_json_sidecar(path: Path) -> bool:
         return False
 
     # Other checks require entities.
-    entities = parse_bids_entities(path)
+    entities = _cache_parse_bids_entities(path)
 
     # Second pass using full compound extension, in case of data files that use a
     # compound extension ending in .json.