Merge remote-tracking branch 'upstream/main' into dev

Author: effigies

.readthedocs.yaml

@@ -9,13 +9,10 @@ build:
       - asdf plugin add uv
       - asdf install uv latest
       - asdf global uv latest
-      # Turn `python -m virtualenv` into `python -c pass`
-      - truncate --size 0 $( dirname $( uv python find ) )/../lib/python3*/site-packages/virtualenv/__main__.py
-    post_create_environment:
+    create_environment:
       - uv venv $READTHEDOCS_VIRTUALENV_PATH
-      # Turn `python -m pip` into `python -c pass`
-      - truncate --size 0 $( ls -d $READTHEDOCS_VIRTUALENV_PATH/lib/python3* )/site-packages/pip.py
-      # Use a cache dir in the same mount to halve the install time
-      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH uv pip install --cache-dir $READTHEDOCS_VIRTUALENV_PATH/../../uv_cache --upgrade sphinx
-    post_install:
+    install:
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH uv pip install --cache-dir $READTHEDOCS_VIRTUALENV_PATH/../../uv_cache -r docs/requirements.txt
+
+sphinx:
+  configuration: docs/conf.py

CHANGELOG.md

@@ -0,0 +1,19 @@
+
+<a id='changelog-2.0.1'></a>
+# 2.0.1 — 2024-12-10
+
+## Fixed
+
+- Improve handling of `.bidsignore` files in the web validator.
+  Ignores matching directories but not the files they contained could fail to match.
+  (#113)
+
+- Resolve issue with parsing headers of NIfTI files with large extensions.
+  Fixes [issue 126].
+
+[issue 126]: https://github.com/bids-standard/bids-validator/issues/126
+
+## Infrastructure
+
+- Adopting [scriv](https://scriv.readthedocs.io/en/latest/) for changelog
+  management.

README.md

@@ -7,7 +7,7 @@
 
 The BIDS Validator is a web application, command-line utility,
 and Javascript/Typescript library for assessing compliance with the
-[Brain Imaging Data Structure][BIDS] standard.
+[Brain Imaging Data Structure (BIDS)][BIDS] standard.
 
 ## Getting Started
 
@@ -23,7 +23,7 @@ The web validator runs in-browser, and does not transfer data to any remote serv
 
 In some contexts, such as when working on a remote server,
 it may be easier to use the command-line.
-The BIDS Validator can be run with the [Deno] runtime
+The BIDS Validator can be run with the [Deno][] runtime
 (see [Deno - Installation][] for detailed installation instructions):
 
 ```shell
@@ -80,9 +80,14 @@ To modify the schema a clone of bids-standard/bids-specification will need to be
 
 After changes to the schema have been made to a local copy the dereferenced single json file used by the validator will need to be built. The `bidsschematools` python package does this. It can be installed from pypi via pip or a local installation can be made. It lives in the specification repository here https://github.com/bids-standard/bids-specification/tree/master/tools/schemacode
 
-The command to compile a dereferenced schema is `bst -v export --output src/schema.json` (this assumes you are in the root of the bids-specification repo). Once compiled it can be passed to the validator via the `-s` flag, `./bids-validator-deno -s <path to schema> <path to dataset>`
+The command to compile a dereferenced schema is `bst -v export --schema src/schema --output src/schema.json` (this assumes you are in the root of the bids-specification repo). Once compiled it can be passed to the validator via the `-s` flag, `./bids-validator-deno -s <path to schema> <path to dataset>`
 
 
 ## Documentation
 
-The BIDS validator documentation is available on [Read the Docs](https://bids-validator.readthedocs.io/en/latest/).
+The BIDS validator documentation is available on [Read the Docs](https://bids-validator.readthedocs.io/en/latest/).
+
+[BIDS]: https://bids.neuroimaging.io
+[BIDS Validator]: https://bids-standard.github.io/bids-validator/
+[Deno]: https://deno.com/
+[Deno - Installation]: https://docs.deno.com/runtime/getting_started/installation/

deno.json

@@ -32,7 +32,7 @@
     "@cliffy/command": "jsr:@effigies/[email protected]",
     "@cliffy/table": "jsr:@effigies/[email protected]",
     "@hed/validator": "npm:[email protected]",
-    "@ignore": "npm:ignore@6.0.2",
+    "@ignore": "npm:ignore@7.0.3",
     "@libs/xml": "jsr:@libs/[email protected]",
     "@mango/nifti": "npm:@bids/[email protected]",
     "@std/assert": "jsr:@std/[email protected]",
@@ -41,6 +41,7 @@
     "@std/io": "jsr:@std/[email protected]",
     "@std/log": "jsr:@std/[email protected]",
     "@std/path": "jsr:@std/[email protected]",
+    "@std/streams": "jsr:@std/[email protected]",
     "@std/yaml": "jsr:@std/yaml@^1.0.4"
   },
   "tasks": {

deno.lock

@@ -2,7 +2,7 @@
 
 The BIDS Validator is a web application, command-line utility,
 and Javascript/Typescript library for assessing compliance with the
-[Brain Imaging Data Structure][BIDS] standard.
+[Brain Imaging Data Structure (BIDS)][BIDS] standard.
 
 ## Getting Started
 
@@ -18,7 +18,7 @@ The web validator runs in-browser, and does not transfer data to any remote serv
 
 In some contexts, such as when working on a remote server,
 it may be easier to use the command-line.
-The BIDS Validator can be run with the [Deno] runtime
+The BIDS Validator can be run with the [Deno][] runtime
 (see [Deno - Installation][] for detailed installation instructions):
 
 ```shell

docs/index.md

@@ -84,6 +84,8 @@ warnings, errors or ignored.
 }
 ```
 
+When a configuration specifies `location` without leading `/` it is interpreted
+as a glob pattern following gitignore syntax.
 The issues are partial matches of the [Issues] that the validator accumulates.
 Pass the `--json` flag to see the issues in detail.
 

docs/user_guide/command-line.md

@@ -3,7 +3,8 @@ import { readAll, readerFromStreamReader } from '@std/io'
 import { basename, dirname, fromFileUrl, join } from '@std/path'
 import { EOL } from '@std/fs'
 import type { FileTree } from '../types/filetree.ts'
-import { BIDSFileDeno, readFileTree, UnicodeDecodeError } from './deno.ts'
+import { BIDSFileDeno, readFileTree } from './deno.ts'
+import { UnicodeDecodeError } from './streams.ts'
 import { requestReadPermission } from '../setup/requestPermissions.ts'
 import { FileIgnoreRules } from './ignore.ts'
 
@@ -12,9 +13,11 @@ await requestReadPermission()
 // Use this file for testing file behavior
 const testUrl = import.meta.url
 const testPath = fromFileUrl(testUrl)
-const testDir = dirname(testPath)
+const testDir = dirname(testPath) // $REPO/src/files
 const testFilename = basename(testPath)
+const repoRoot = dirname(dirname(dirname(testPath)))
 const ignore = new FileIgnoreRules([])
+const prune = new FileIgnoreRules(['derivatives'], false)
 
 Deno.test('Deno implementation of BIDSFile', async (t) => {
   await t.step('implements basic file properties', () => {
@@ -53,7 +56,7 @@ Deno.test('Deno implementation of BIDSFile', async (t) => {
     'strips BOM characters when reading UTF-8 via .text()',
     async () => {
       // BOM is invalid in JSON but shows up often from certain tools, so abstract handling it
-      const bomDir = join(testPath, '..', '..', 'tests')
+      const bomDir = join(repoRoot, 'src', 'tests')
       const bomFilename = 'bom-utf8.json'
       const file = new BIDSFileDeno(bomDir, bomFilename, ignore)
       const text = await file.text()
@@ -75,4 +78,16 @@ Deno.test('Deno implementation of FileTree', async (t) => {
     assert(testObj !== undefined)
     assertEquals(testObj.path, `/${parent}/${testFilename}`)
   })
+
+  await t.step('implements pruning', async () => {
+    const dsDir = join(repoRoot, 'tests', 'data', 'valid_dataset')
+    const derivFile =
+      'derivatives/fmriprep/sub-01/ses-01/func/sub-01_ses-01_task-rest_confounds.tsv.gz'
+
+    const fullTree = await readFileTree(dsDir)
+    assert(fullTree.get(derivFile))
+
+    const prunedTree = await readFileTree(dsDir, prune)
+    assert(!prunedTree.get(derivFile))
+  })
 })

src/files/deno.test.ts

@@ -7,18 +7,9 @@ import { type BIDSFile, FileTree } from '../types/filetree.ts'
 import { requestReadPermission } from '../setup/requestPermissions.ts'
 import { FileIgnoreRules, readBidsIgnore } from './ignore.ts'
 import { logger } from '../utils/logger.ts'
+import { createUTF8Stream } from './streams.ts'
 export { type BIDSFile, FileTree }
 
-/**
- * Thrown when a text file is decoded as UTF-8 but contains UTF-16 characters
- */
-export class UnicodeDecodeError extends Error {
-  constructor(message: string) {
-    super(message)
-    this.name = 'UnicodeDecode'
-  }
-}
-
 /**
  * Deno implementation of BIDSFile
  */
@@ -67,27 +58,17 @@ export class BIDSFileDeno implements BIDSFile {
    * Read the entire file and decode as utf-8 text
    */
   async text(): Promise<string> {
-    const streamReader = this.stream
-      .pipeThrough(new TextDecoderStream('utf-8'))
-      .getReader()
-    let data = ''
+    const reader = this.stream.pipeThrough(createUTF8Stream()).getReader()
+    const chunks: string[] = []
     try {
-      // Read once to check for unicode issues
-      const { done, value } = await streamReader.read()
-      // Check for UTF-16 BOM
-      if (value && value.startsWith('\uFFFD')) {
-        throw new UnicodeDecodeError('This file appears to be UTF-16')
-      }
-      if (done) return data
-      data += value
-      // Continue reading the rest of the file if no unicode issues were found
       while (true) {
-        const { done, value } = await streamReader.read()
-        if (done) return data
-        data += value
+        const { done, value } = await reader.read()
+        if (done) break
+        chunks.push(value)
       }
+      return chunks.join('')
     } finally {
-      streamReader.releaseLock()
+      reader.releaseLock()
     }
   }
 
@@ -120,17 +101,22 @@ async function _readFileTree(
   rootPath: string,
   relativePath: string,
   ignore: FileIgnoreRules,
+  prune: FileIgnoreRules,
   parent?: FileTree,
 ): Promise<FileTree> {
   await requestReadPermission()
   const name = basename(relativePath)
   const tree = new FileTree(relativePath, name, parent, ignore)
 
   for await (const dirEntry of Deno.readDir(join(rootPath, relativePath))) {
+    const thisPath = posix.join(relativePath, dirEntry.name)
+    if (prune.test(thisPath)) {
+      continue
+    }
     if (dirEntry.isFile || dirEntry.isSymlink) {
       const file = new BIDSFileDeno(
         rootPath,
-        posix.join(relativePath, dirEntry.name),
+        thisPath,
         ignore,
       )
       file.parent = tree
@@ -139,8 +125,9 @@ async function _readFileTree(
     if (dirEntry.isDirectory) {
       const dirTree = await _readFileTree(
         rootPath,
-        posix.join(relativePath, dirEntry.name),
+        thisPath,
         ignore,
+        prune,
         tree,
       )
       tree.directories.push(dirTree)
@@ -152,9 +139,13 @@ async function _readFileTree(
 /**
  * Read in the target directory structure and return a FileTree
  */
-export async function readFileTree(rootPath: string): Promise<FileTree> {
+export async function readFileTree(
+  rootPath: string,
+  prune?: FileIgnoreRules,
+): Promise<FileTree> {
+  prune ??= new FileIgnoreRules([], false)
   const ignore = new FileIgnoreRules([])
-  const tree = await _readFileTree(rootPath, '/', ignore)
+  const tree = await _readFileTree(rootPath, '/', ignore, prune)
   const bidsignore = tree.get('.bidsignore')
   if (bidsignore) {
     try {