Merge pull request #1155 from nellh/filename-validation-bidsignore

Add support for passing .bidsignore to --filename only validation

Author: rwblair

bids-validator/utils/__tests__/filenamesOnly.spec.js

@@ -6,6 +6,7 @@ describe('test filenames mode', () => {
   })
   it('throws an error when obviously non-BIDS input', async () => {
     async function* badData() {
+      yield '0001'
       yield 'nope'
       yield 'not-bids'
       yield 'data'
@@ -15,6 +16,7 @@ describe('test filenames mode', () => {
   })
   it('passes validation with a simple dataset', async () => {
     async function* goodData() {
+      yield '0001'
       yield 'CHANGES'
       yield 'dataset_description.json'
       yield 'participants.tsv'
@@ -25,4 +27,19 @@ describe('test filenames mode', () => {
     const res = await validateFilenames(goodData())
     expect(res).toBe(true)
   })
+  it('passes validation with .bidsignore', async () => {
+    async function* goodData() {
+      yield 'sub-02/*'
+      yield '0001'
+      yield 'CHANGES'
+      yield 'dataset_description.json'
+      yield 'participants.tsv'
+      yield 'README'
+      yield 'sub-01/anat/sub-01_T1w.nii.gz'
+      yield 'T1w.json'
+      yield 'sub-02/not-bids-file.txt'
+    }
+    const res = await validateFilenames(goodData())
+    expect(res).toBe(true)
+  })
 })

bids-validator/utils/filenamesOnly.js

@@ -4,6 +4,7 @@
 import readline from 'readline'
 import path from 'path'
 import consoleFormat from './consoleFormat'
+import { defaultIgnore } from './files/readDir'
 import quickTest from '../validators/bids/quickTest'
 import fullTest from '../validators/bids/fullTest'
 
@@ -21,26 +22,50 @@ const defaultOptions = {
 }
 
 async function generateFileObjects(stream) {
+  const ig = defaultIgnore()
   const inputFiles = {}
+  let bidsIgnore = true
   let index = 0
   for await (const line of stream) {
-    const rootPath = `/${line}`
-    /**
-     * Simulated file object based on input
-     * File size is 1 to prevent 0 size errors but makes some checks inaccurate
-     */
-    const file = {
-      name: path.basename(line),
-      path: rootPath,
-      relativePath: rootPath,
-      size: 1,
+    // Part 1, parse bidsignore until 0001 (git delimiter packet)
+    if (line === '0001') {
+      bidsIgnore = false
+    } else {
+      if (bidsIgnore) {
+        ig.add(line)
+      } else {
+        // Done with bidsignore, read filename data
+        const rootPath = `/${line}`
+        /**
+         * Simulated file object based on input
+         * File size is 1 to prevent 0 size errors but makes some checks inaccurate
+         */
+        const file = {
+          name: path.basename(line),
+          path: rootPath,
+          relativePath: rootPath,
+          size: 1,
+        }
+        if (ig.ignores(rootPath)) {
+          file.ignore = true
+        }
+        inputFiles[index] = file
+        index++
+      }
     }
-    inputFiles[index] = file
-    index++
   }
   return inputFiles
 }
 
+/**
+ * Validate input from stdin as bidsignore + filenames
+ *
+ * Protocol uses `0000` line to separate the two streams
+ * .bidsignore lines are read first
+ * One filename per line is read in and bidsignore rules applied
+ *
+ * @param {AsyncIterable} stream Readline stream
+ */
 export async function validateFilenames(stream) {
   const inputFiles = await generateFileObjects(stream)
   const couldBeBIDS = quickTest(inputFiles)

bids-validator/utils/files/readDir.js

@@ -314,13 +314,17 @@ async function getFilesFromFs(dir, rootPath, ig, options) {
   return filesAccumulator
 }
 
-async function getBIDSIgnore(dir) {
-  const ig = ignore()
+export function defaultIgnore() {
+  return ignore()
     .add('.*')
     .add('!*.icloud')
     .add('/derivatives')
     .add('/sourcedata')
     .add('/code')
+}
+
+async function getBIDSIgnore(dir) {
+  const ig = defaultIgnore()
 
   const bidsIgnoreFileObj = getBIDSIgnoreFileObj(dir)
   if (bidsIgnoreFileObj) {