rf: Validate additional columns when defined

Author: effigies

src/schema/tables.ts

@@ -218,46 +218,50 @@ export function evalColumns(
   schemaPath: string,
 ): void {
   if (!rule.columns || !['.tsv', '.tsv.gz'].includes(context.extension)) return
-  const headers = [...Object.keys(context.columns)]
-  for (const [ruleHeader, requirement] of Object.entries(rule.columns)) {
-    const columnObject: ColumnSchema = schema.objects.columns[ruleHeader]
+  const columns = rule.columns as Record<string, string | { level: string }>
 
-    // What is this?
-    if (!('name' in columnObject) || !columnObject['name']) {
-      return
-    }
+  const columnLookup = Object.fromEntries(
+    Object.keys(rule.columns).map((col) => [schema.objects.columns[col].name, col]),
+  )
+  const additionalColumns = Object.keys(context.columns).filter((name) => !(name in columnLookup))
 
-    const name = columnObject.name
-    if (!headers.includes(name)) {
-      if (
-        getRequirement(rule as ColumnRule, ruleHeader) === 'required' &&
-        // A more precise error will be generated by evalInitialColumns
-        !(rule?.initial_columns && rule.initial_columns.includes(ruleHeader))
-      ) {
-        context.dataset.issues.add({
-          code: 'TSV_COLUMN_MISSING',
-          subCode: name,
-          location: context.path,
-          rule: schemaPath,
-        })
-      }
-      return
-    }
+  for (const name of new Set([...Object.keys(columnLookup), ...additionalColumns])) {
+    const ruleHeader = columnLookup[name]
+    const sidecarDef = context.sidecar[name] as ColumnDefinition | undefined
+    const requirement = getRequirement(rule as ColumnRule, ruleHeader)
+    const column = context.columns[name] as string[]
+
+    const issue = { subCode: name, location: context.path, rule: schemaPath }
 
     let signature: ValueSignature
-    try {
-      signature = getValueSignature(columnObject, context?.sidecar[name])
-    } catch (e: any) {
-      if (e?.code) {
-        context.dataset.issues.add({
-          ...e,
-          subCode: name,
-          location: context.sidecarKeyOrigin[name],
-          rule: schemaPath,
-        })
-        signature = getValueSignature(columnObject, undefined)
-      } else {
-        throw e
+    if (!ruleHeader) { // Additional column
+      if (requirement === 'not_allowed' || !sidecarDef) {
+        continue
+      }
+      signature = extractDefinition(sidecarDef)
+    } else {
+      const columnObject = schema.objects.columns[ruleHeader]
+      if (!column) {
+        if (requirement === 'required' && !(rule?.initial_columns?.includes(ruleHeader))) {
+          context.dataset.issues.add({ code: 'TSV_COLUMN_MISSING', ...issue })
+        }
+        continue
+      }
+
+      try {
+        signature = getValueSignature(columnObject, sidecarDef)
+      } catch (e: any) {
+        if (e?.code) {
+          context.dataset.issues.add({
+            ...e,
+            subCode: name,
+            location: context.sidecarKeyOrigin[name],
+            rule: schemaPath,
+          })
+          signature = extractSchema(columnObject)
+        } else {
+          throw e
+        }
       }
     }
 
@@ -267,17 +271,9 @@ export function evalColumns(
 
     const spec = compileSignature(signature, schema.objects.formats)
 
-    const issue = {
-      code: 'TSV_VALUE_INCORRECT_TYPE',
-      subCode: name,
-      location: context.path,
-      rule: schemaPath,
-    }
-
     const ageCheck = name === 'age'
     let ageWarningIssued = false
 
-    const column = context.columns[name] as string[]
     for (const [index, value] of column.entries()) {
       if (!checkValue(value, spec)) {
         if (ageCheck && value === '89+') {
@@ -295,6 +291,7 @@ export function evalColumns(
           (value.match(/^\s*(NA|na|nan|NaN|n\/a)\s*$/) ? ", did you mean 'n/a'?" : '')
         context.dataset.issues.add({
           ...issue,
+          code: 'TSV_VALUE_INCORRECT_TYPE',
           line: index + 2,
           issueMessage,
         })