Merge pull request #839 from bids-standard/remote-files-option

Remote files option

Author: nellh

bids-validator/.eslintrc.json

@@ -1,7 +1,7 @@
 {
   "plugins": ["prettier"],
   "parserOptions": {
-    "ecmaVersion": 2017,
+    "ecmaVersion": 2018,
     "sourceType": "module"
   },
   "env": {

bids-validator/bin/bids-validator

@@ -25,6 +25,9 @@ var argv = require('yargs')
     'ignoreSymlinks',
     'Skip any symlinked directories when validating a dataset',
   )
+  .boolean('remoteFiles')
+  .describe('remoteFiles', 'Validate remote files.')
+  .default('remoteFiles', false)
   .boolean('gitTreeMode')
   .describe(
     'gitTreeMode',

bids-validator/utils/files/readDir.js

@@ -352,7 +352,7 @@ function getBIDSIgnoreFileObjNode(dir) {
   var bidsIgnoreFileObj = null
   var path = dir + '/.bidsignore'
   if (fs.existsSync(path)) {
-    bidsIgnoreFileObj = { path: path }
+    bidsIgnoreFileObj = { path: path, stats: { size: null } }
   }
   return bidsIgnoreFileObj
 }

bids-validator/utils/files/testFile.js

@@ -1,6 +1,7 @@
 const fs = require('fs')
 const Issue = require('../../utils/issues').Issue
 const remoteFiles = require('./remoteFiles')
+const { getOptions } = require('../../utils/options')
 
 /**
  * Test File
@@ -10,52 +11,60 @@ const remoteFiles = require('./remoteFiles')
  * or null and stats if it is.
  */
 function testFile(file, annexed, dir, callback) {
-  fs.stat(file.path, function(statErr, stats) {
-    if (statErr) {
+  fs.access(file.path, function(accessErr) {
+    if (!accessErr) {
+      // accessible
+      handleFsAccess(file, callback)
+    } else {
+      // inaccessible
       fs.lstat(file.path, function(lstatErr, lstats) {
-        if (lstatErr) {
-          callback(new Issue({ code: 44, file: file }), stats)
-        } else if (lstats && lstats.isSymbolicLink()) {
-          if (annexed) {
-            // Set byte retrieval limits based on file type
-            const limit = file.name.includes('.nii') ? 500 : false
-            // Call process to get remote files
-            // It will call callback with content or error
-            remoteFiles.getAnnexedFile(file, dir, limit, callback)
-          } else {
-            callback(new Issue({ code: 43, file: file }), stats)
-          }
+        if (!lstatErr && lstats && lstats.isSymbolicLink()) {
+          // symlink
+          if (getOptions().remoteFiles)
+            // only follow symlinks when --remoteFiles option is on
+            handleRemoteAccess(file, annexed, dir, callback)
+          else
+            callback(
+              new Issue({
+                code: 114,
+                file,
+              }),
+              file.stats,
+            )
         } else {
-          callback(new Issue({ code: 44, file: file }), stats)
+          // inaccessible local file
+          callback(new Issue({ code: 44, file: file }), file.stats)
         }
       })
-    } else {
-      fs.access(file.path, function(accessErr) {
-        handleFsAccess(accessErr, file, stats, callback)
-      })
     }
   })
 }
 
-function handleFsAccess(accessErr, file, stats, callback) {
-  if (!accessErr) {
-    process.nextTick(function() {
-      if (stats.size === 0) {
-        callback(
-          new Issue({
-            code: 99,
-            file: file,
-            reason: `Empty files (${file.path}) not allowed.`,
-          }),
-          stats,
-        )
-      }
-      callback(null, stats)
-    })
+function handleFsAccess(file, callback) {
+  process.nextTick(function() {
+    if (file.stats.size === 0) {
+      callback(
+        new Issue({
+          code: 99,
+          file: file,
+          reason: `Empty files (${file.path}) not allowed.`,
+        }),
+        file.stats,
+      )
+    }
+    callback(null, file.stats)
+  })
+}
+
+function handleRemoteAccess(file, annexed, dir, callback) {
+  if (annexed) {
+    // Set byte retrieval limits based on file type
+    const limit = file.name.includes('.nii') ? 500 : false
+    // Call process to get remote files
+    // It will call callback with content or error
+    remoteFiles.getAnnexedFile(file, dir, limit, callback)
   } else {
-    process.nextTick(function() {
-      callback(new Issue({ code: 44, file: file }), stats)
-    })
+    callback(new Issue({ code: 43, file: file }), file.stats)
   }
 }
 

bids-validator/utils/issues/list.js

@@ -632,4 +632,10 @@ module.exports = {
     reason:
       'The Authors field of dataset_description.json should contain an array of fields - with one author per field. This was triggered because there are no authors, which will make DOI registration from dataset metadata impossible.',
   },
+  114: {
+    key: 'INCOMPLETE_DATASET',
+    severity: 'error',
+    reason:
+      'This dataset contains remote files. If you would like to validate with remote files, use the --remoteFiles option.',
+  },
 }

bids-validator/utils/options.js

@@ -1,18 +1,21 @@
 var files = require('./files')
 var json = require('./json')
 
+let options
+
 module.exports = {
   /**
    * Parse
    */
-  parse: function(options, callback) {
-    options = options ? options : {}
+  parse: function(args, callback) {
+    options = args ? args : {}
     options = {
       ignoreWarnings: Boolean(options.ignoreWarnings),
       ignoreNiftiHeaders: Boolean(options.ignoreNiftiHeaders),
       ignoreSymlinks: Boolean(options.ignoreSymlinks),
       verbose: Boolean(options.verbose),
       gitTreeMode: Boolean(options.gitTreeMode),
+      remoteFiles: Boolean(options.remoteFiles),
       gitRef: options.gitRef || 'HEAD',
       config: options.config || {},
     }
@@ -26,6 +29,11 @@ module.exports = {
     }
   },
 
+  getOptions: () => {
+    const readonlyOptions = Object.freeze({ ...options })
+    return readonlyOptions
+  },
+
   /**
    * Load Config
    */