Merge pull request #568 from bGrass/custom_column_without_description

Data dictionary warning refinement

Author: olgn

bids_validator/tsv/non_custom_columns.json

@@ -0,0 +1,28 @@
+{
+  "channels": [
+    "description",
+    "high_cutoff",
+    "low_cutoff",
+    "name",
+    "notch",
+    "sampling_frequency",
+    "software_filters",
+    "status",
+    "status_description",
+    "type",
+    "units"
+  ],
+  "events": [
+    "duration",
+    "HED",
+    "onset",
+    "trial_type",
+    "response_time",
+    "stim_file"
+  ],
+  "misc": [],
+  "participants": ["participant_id"],
+  "phenotype": ["participant_id"],
+  "scans": ["acq_time", "filename"],
+  "sessions": ["acq_time", "session_id"]
+}

tests/bids.spec.js

@@ -194,10 +194,11 @@ var suite = describe('BIDS example datasets ', function() {
     })
   })
 
-  it('checks for tabular files without corresponding data dictionaries', function(isdone) {
+  it('checks for tabular files with custom columns not described in a data dictionary', function(isdone) {
     var options = { ignoreNiftiHeaders: true }
     validate.BIDS(
       'tests/data/BIDS-examples-' + test_version + '/ds001',
+      //'tests/data/ds001344-1.0.0',
       options,
       function(issues) {
         assert(issues.warnings.length === 2 && issues.warnings[1].code === '82')

utils/files/generatedMergedSidecarDict.js renamed to utils/files/generateMergedSidecarDict.js

@@ -6,7 +6,7 @@ const readFile = require('./readFile')
 const readNiftiHeader = require('./readNiftiHeader')
 const readDir = require('./readDir')
 const potentialLocations = require('./potentialLocations')
-const generateMergedSidecarDict = require('./generatedMergedSidecarDict')
+const generateMergedSidecarDict = require('./generateMergedSidecarDict')
 const getBFileContent = require('./getBFileContent')
 
 // public API ---------------------------------------------------------------------

utils/files/index.js

@@ -446,9 +446,10 @@ module.exports = {
     reason: "Third column of the channels file must be named 'notch'",
   },
   82: {
-    key: 'TABULARFILE_WITHOUT_DATADICTIONARY',
+    key: 'CUSTOM_COLUMN_WITHOUT_DESCRIPTION',
     severity: 'warning',
-    reason: 'Tabular file does not have accompanying data dictionary.',
+    reason:
+      'Tabular file contains custom columns not described in a data dictionary',
   },
   83: {
     key: 'ECHOTIME1_2_DIFFERENCE_UNREASONABLE',

utils/issues/list.js

@@ -170,7 +170,7 @@ BIDS = {
   fullTest: function(fileList, callback) {
     var self = this
 
-    var jsonContentsDict = {},
+    let jsonContentsDict = {},
       bContentsDict = {},
       events = [],
       stimuli = {
@@ -184,6 +184,8 @@ BIDS = {
       participants = null,
       phenotypeParticipants = [],
       hasSubjectDir = false
+
+    let tsvs = []
     var hasDatasetDescription = false
 
     var summary = {
@@ -194,7 +196,6 @@ BIDS = {
       totalFiles: Object.keys(fileList).length,
       size: 0,
     }
-
     // collect file directory statistics
     async.eachOfLimit(fileList, 200, function(file) {
       // collect file stats
@@ -341,32 +342,14 @@ BIDS = {
 
         // validate tsv
         else if (file.name && file.name.endsWith('.tsv')) {
-          // Generate name for corresponding data dictionary file
-          let dict_path = file.relativePath.replace('.tsv', '.json')
-          let exists = false
-          let potentialDicts = utils.files.potentialLocations(dict_path)
-          // Need to check for .json file at all levels of heirarchy
-          // Get list of fileList keys
-          let idxs = Object.keys(fileList)
-          for (let i of idxs) {
-            if (potentialDicts.indexOf(fileList[i].relativePath) > -1) {
-              exists = true
-              break
-            }
-          }
-
-          // Check if data dictionary file exists
-          if (!exists) {
-            self.issues.push(
-              new Issue({
-                code: 82,
-                file: file,
-              }),
-            )
-          }
           utils.files
             .readFile(file)
             .then(contents => {
+              // Push TSV to list for custom column verification after all data dictionaries have been read
+              tsvs.push({
+                file: file,
+                contents: contents,
+              })
               if (file.name.endsWith('_events.tsv')) {
                 events.push({
                   file: file,
@@ -715,6 +698,11 @@ BIDS = {
                   self.issues,
                 )
 
+                // validate custom fields in all TSVs and add any issues to the list
+                self.issues = self.issues.concat(
+                  TSV.validateTsvColumns(tsvs, jsonContentsDict),
+                )
+
                 // validation session files
                 self.issues = self.issues.concat(session(fileList))
 

validators/bids.js

@@ -4,6 +4,7 @@ var files = require('../utils/files')
 var utils = require('../utils')
 var dateIsValid = require('date-fns/isValid')
 var parseDate = require('date-fns/parse')
+const nonCustomColumns = require('../bids_validator/tsv/non_custom_columns.json')
 /**
  * TSV
  *
@@ -344,7 +345,100 @@ const checkAcqTimeFormat = function(rows, file, issues) {
   }
 }
 
+/**
+ * @param {Object} file - BIDS file object
+ * Accepts file object and returns a type based on file path
+ */
+const getTsvType = function(file) {
+  let tsvType = 'misc'
+  if (file.relativePath.includes('phenotype/')) {
+    tsvType = 'phenotype'
+  } else if (file.name === 'participants.tsv') {
+    tsvType = 'participants'
+  } else if (
+    file.name.endsWith('_channels.tsv') ||
+    file.name.endsWith('_events.tsv') ||
+    file.name.endsWith('_scans.tsv') ||
+    file.name.endsWith('_sessions.tsv')
+  ) {
+    const split = file.name.split('_')
+    tsvType = split[split.length - 1].replace('.tsv', '')
+  }
+  return tsvType
+}
+
+/**
+ *
+ * @param {array} headers -Array of column names
+ * @param {string} type - Type from getTsvType
+ * Checks TSV column names to determine if they're core or custom
+ * Returns array of custom column names
+ */
+const getCustomColumns = function(headers, type) {
+  const customCols = []
+  // Iterate column headers
+  for (let col of headers) {
+    // If it's a custom column
+    if (!nonCustomColumns[type].includes(col)) {
+      customCols.push(col)
+    }
+  }
+  return customCols
+}
+
+/**
+ *
+ * @param {array} tsvs - Array of objects containing TSV file objects and contents
+ * @param {Object} jsonContentsDict
+ */
+const validateTsvColumns = function(tsvs, jsonContentsDict) {
+  let tsvIssues = []
+  tsvs.map(tsv => {
+    const customColumns = getCustomColumns(
+      tsv.contents.split('\n')[0].split('\t'),
+      getTsvType(tsv.file),
+    )
+    if (customColumns.length > 0) {
+      // Get merged data dictionary for this file
+      const potentialSidecars = utils.files.potentialLocations(
+        tsv.file.relativePath.replace('.tsv', '.json'),
+      )
+      const mergedDict = utils.files.generateMergedSidecarDict(
+        potentialSidecars,
+        jsonContentsDict,
+      )
+      const keys = Object.keys(mergedDict)
+      // Gather undefined columns for the file
+      let undefinedCols = customColumns.filter(col => !keys.includes(col))
+      // Create an issue for all undefined columns in this file
+      undefinedCols.length &&
+        tsvIssues.push(
+          customColumnIssue(
+            tsv.file,
+            undefinedCols.join(', '),
+            potentialSidecars,
+          ),
+        )
+    }
+  })
+  // Return array of all instances of undescribed custom columns
+  return tsvIssues
+}
+
+const customColumnIssue = function(file, col, locations) {
+  return new Issue({
+    code: 82,
+    file: file,
+    evidence:
+      'Columns: ' +
+      col +
+      ' not defined, please define in: ' +
+      locations.toString().replace(',', ', '),
+  })
+}
+
 module.exports = {
   TSV: TSV,
   checkphenotype: checkphenotype,
+  validateTsvColumns: validateTsvColumns,
 }