Merge pull request #234 from effigies/feat/sidecar-columns

feat: Unify schema and sidecar-defined column validation

Author: effigies

changelog.d/20250824_210257_markiewicz_sidecar_columns.md

@@ -0,0 +1,48 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+For top level release notes, leave all the headers commented out.
+-->
+
+<!--
+### Added
+
+- A bullet item for the Added category.
+
+-->
+### Changed
+
+- Improved handling of TSV columns with sidecar definitions.
+  `"Format"`, `"Minimum"` and `"Maximum"` keys are now supported.
+
+<!--
+### Fixed
+
+- A bullet item for the Fixed category.
+
+-->
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category.
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category.
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category.
+
+-->
+<!--
+### Infrastructure
+
+- A bullet item for the Infrastructure category.
+
+-->

src/schema/applyRules.ts

@@ -94,7 +94,6 @@ export function evalCheck(src: string, context: BIDSContext) {
  * We associate theys keys from a rule object to a function adds an
  * issue to the context if the rule evaluation fails.
  */
-// @ts-expect-error
 const evalMap: Record<
   keyof GenericRule,
   (
@@ -105,9 +104,13 @@ const evalMap: Record<
   ) => boolean | void
 > = {
   checks: evalRuleChecks,
+  // @ts-expect-error
   columns: evalColumns,
+  // @ts-expect-error
   additional_columns: evalAdditionalColumns,
+  // @ts-expect-error
   initial_columns: evalInitialColumns,
+  // @ts-expect-error
   index_columns: evalIndexColumns,
   fields: evalJsonCheck,
 }

src/schema/tables.test.ts

@@ -26,6 +26,18 @@ const schemaDefs = {
           index_columns: ['filename'],
           additional_columns: 'allowed',
         },
+        Participants: {
+          selectors: ['path == "/participants.tsv"'],
+          initial_columns: ['participant_id'],
+          columns: {
+            participant_id: 'required',
+            age: 'recommended',
+            sex: 'recommended',
+            strain_rrid: 'recommended',
+          },
+          index_columns: ['participant_id'],
+          additional_columns: 'allowed',
+        },
       },
       made_up: {
         MadeUp: {
@@ -93,6 +105,67 @@ Deno.test('tables eval* tests', async (t) => {
     assertEquals(context.dataset.issues.size, 0)
   })
 
+  await t.step('verify column override behavior', () => {
+    const context = {
+      path: '/participants.tsv',
+      extension: '.tsv',
+      sidecar: {
+        participant_id: {
+          Description: 'A participant identifier',
+          Format: 'string',
+        },
+        age: {
+          Description: 'Age in weeks',
+          Format: 'number',
+        },
+        sex: {
+          Description: 'Phenotypic sex',
+          Format: 'string',
+          Levels: {
+            'F': { Description: 'Female' },
+            'M': { Description: 'Male' },
+            'O': { Description: 'Other' },
+          },
+        },
+        strain_rrid: {
+          Description: 'Invalid override',
+          Format: 'integer',
+        },
+      },
+      sidecarKeyOrigin: {
+        participant_id: '/participants.json',
+        age: '/participants.json',
+        sex: '/participants.json',
+        strain_rrid: '/participants.json',
+      },
+      columns: {
+        participant_id: ['sub-01', 'sub-02', 'sub-03'],
+        age: ['10', '20', '30'],
+        sex: ['M', 'F', 'f'],
+        strain_rrid: ['RRID:SCR_012345', 'RRID:SCR_012345', 'n/a'],
+      },
+      dataset: { issues: new DatasetIssues() },
+    }
+    const rule = schemaDefs.rules.tabular_data.modality_agnostic.Participants
+    evalColumns(rule, context, schema, 'rules.tabular_data.modality_agnostic.Participants')
+
+    // participant_id column definition is compatible with schema
+    // age and sex may be overridden
+    // strain_rrid can't be redefined to numeric
+    let issues = context.dataset.issues.get({ code: 'TSV_COLUMN_TYPE_REDEFINED' })
+    assertEquals(issues.length, 1)
+    assertEquals(issues[0].subCode, 'strain_rrid')
+    assertEquals(issues[0].issueMessage, 'Format "integer" must be rrid')
+
+    // Overriding the default sex definition uses the provided values
+    // Values in the default definition may raise issues
+    issues = context.dataset.issues.get({ code: 'TSV_VALUE_INCORRECT_TYPE_NONREQUIRED' })
+    assertEquals(issues.length, 1)
+    assertEquals(issues[0].subCode, 'sex')
+    assertEquals(issues[0].line, 4)
+    assertEquals(issues[0].issueMessage, "'f'")
+  })
+
   await t.step('verify column ordering', () => {
     const context = {
       path: '/sub-01/sub-01_scans.tsv',