Merge pull request #240 from effigies/feat/tsv-gz

 feat: Enable type checking on tsv.gz files

Author: effigies

changelog.d/20250828_104256_markiewicz_tsv_gz.md

@@ -0,0 +1,47 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+For top level release notes, leave all the headers commented out.
+-->
+
+### Added
+
+- Load `.tsv.gz` column contents for validation.
+
+<!--
+### Changed
+
+- A bullet item for the Changed category.
+
+-->
+<!--
+### Fixed
+
+- A bullet item for the Fixed category.
+
+-->
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category.
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category.
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category.
+
+-->
+<!--
+### Infrastructure
+
+- A bullet item for the Infrastructure category.
+
+-->

src/files/tsv.test.ts

@@ -6,7 +6,7 @@ import {
   assertStrictEquals,
 } from '@std/assert'
 import { pathToFile } from './filetree.ts'
-import { loadTSV } from './tsv.ts'
+import { loadTSV, loadTSVGZ } from './tsv.ts'
 import { streamFromString } from '../tests/utils.ts'
 import { ColumnsMap } from '../types/columns.ts'
 
@@ -64,7 +64,7 @@ Deno.test('TSV loading', async (t) => {
     try {
       await loadTSV(file)
     } catch (e: any) {
-      assertObjectMatch(e, { code: 'TSV_EQUAL_ROWS', location: '/mismatched_row.tsv', line: 3 })
+      assertObjectMatch(e, { code: 'TSV_EQUAL_ROWS', line: 3 })
     }
   })
 
@@ -178,3 +178,112 @@ Deno.test('TSV loading', async (t) => {
   // Tests will have populated the memoization cache
   loadTSV.cache.clear()
 })
+
+Deno.test('TSVGZ loading', async (t) => {
+  await t.step('No header and empty file produces empty map', async () => {
+    const file = pathToFile('/empty.tsv.gz')
+    file.stream = streamFromString('').pipeThrough(new CompressionStream('gzip'))
+
+    const map = await loadTSVGZ(file, [])
+    // map.size looks for a column called map, so work around it
+    assertEquals(Object.keys(map).length, 0)
+  })
+
+  await t.step('Empty file produces header-only map', async () => {
+    const file = pathToFile('/empty.tsv.gz')
+    file.stream = streamFromString('').pipeThrough(new CompressionStream('gzip'))
+
+    const map = await loadTSVGZ(file, ['a', 'b', 'c'])
+    assertEquals(map.a, [])
+    assertEquals(map.b, [])
+    assertEquals(map.c, [])
+  })
+
+  await t.step('Single column file produces single column maps', async () => {
+    const file = pathToFile('/single_column.tsv')
+    file.stream = streamFromString('1\n2\n3\n').pipeThrough(new CompressionStream('gzip'))
+
+    const map = await loadTSVGZ(file, ['a'])
+    assertEquals(map.a, ['1', '2', '3'])
+  })
+
+  await t.step('Mismatched header length throws issue', async () => {
+    const file = pathToFile('/single_column.tsv.gz')
+    file.stream = streamFromString('1\n2\n3\n').pipeThrough(new CompressionStream('gzip'))
+
+    try {
+      await loadTSVGZ(file, ['a', 'b'])
+    } catch (e: any) {
+      assertObjectMatch(e, { code: 'TSV_EQUAL_ROWS', line: 1 })
+    }
+  })
+
+  await t.step('Missing final newline is ignored', async () => {
+    const file = pathToFile('/missing_newline.tsv.gz')
+    file.stream = streamFromString('1\n2\n3').pipeThrough(new CompressionStream('gzip'))
+
+    const map = await loadTSVGZ(file, ['a'])
+    assertEquals(map.a, ['1', '2', '3'])
+  })
+
+  await t.step('Empty row throws issue', async () => {
+    const file = pathToFile('/empty_row.tsv.gz')
+    file.stream = streamFromString('1\t2\t3\n\n4\t5\t6\n').pipeThrough(new CompressionStream('gzip'))
+
+    try {
+      await loadTSVGZ(file, ['a', 'b', 'c'])
+    } catch (e: any) {
+      assertObjectMatch(e, { code: 'TSV_EMPTY_LINE', line: 2 })
+    }
+  })
+
+  await t.step('Mislabeled TSV throws issue', async () => {
+    const file = pathToFile('/mismatched_row.tsv.gz')
+    file.stream = streamFromString('a\tb\tc\n1\t2\t3\n4\t5\n')
+
+    try {
+      await loadTSVGZ(file, ['a', 'b', 'c'])
+    } catch (e: any) {
+      assertObjectMatch(e, { code: 'INVALID_GZIP' })
+    }
+  })
+
+  await t.step('maxRows limits the number of rows read', async () => {
+    const file = pathToFile('/long.tsv.gz')
+    // Use 1500 to avoid overlap with default initial capacity
+    const headers = ['a', 'b', 'c']
+    const text = '1\t2\t3\n'.repeat(1500)
+    file.stream = streamFromString(text).pipeThrough(new CompressionStream('gzip'))
+
+    let map = await loadTSVGZ(file, headers, 0)
+    assertEquals(map.a, [])
+    assertEquals(map.b, [])
+    assertEquals(map.c, [])
+
+    file.stream = streamFromString(text).pipeThrough(new CompressionStream('gzip'))
+    map = await loadTSVGZ(file, headers, 1)
+    assertEquals(map.a, ['1'])
+    assertEquals(map.b, ['2'])
+    assertEquals(map.c, ['3'])
+
+    file.stream = streamFromString(text).pipeThrough(new CompressionStream('gzip'))
+    map = await loadTSVGZ(file, headers, 2)
+    assertEquals(map.a, ['1', '1'])
+    assertEquals(map.b, ['2', '2'])
+    assertEquals(map.c, ['3', '3'])
+
+    file.stream = streamFromString(text).pipeThrough(new CompressionStream('gzip'))
+    map = await loadTSVGZ(file, headers, -1)
+    assertEquals(map.a, Array(1500).fill('1'))
+    assertEquals(map.b, Array(1500).fill('2'))
+    assertEquals(map.c, Array(1500).fill('3'))
+
+    // Check that maxRows does not truncate shorter files
+    file.stream = streamFromString('1\t2\t3\n4\t5\t6\n7\t8\t9\n').pipeThrough(new CompressionStream('gzip'))
+    map = await loadTSVGZ(file, headers, 4)
+    assertEquals(map.a, ['1', '4', '7'])
+    assertEquals(map.b, ['2', '5', '8'])
+    assertEquals(map.c, ['3', '6', '9'])
+  })
+
+})

src/files/tsv.ts

@@ -8,6 +8,72 @@ import type { BIDSFile } from '../types/filetree.ts'
 import { filememoizeAsync } from '../utils/memoize.ts'
 import { createUTF8Stream } from './streams.ts'
 
+async function loadColumns(
+  reader: ReadableStreamDefaultReader<string>,
+  headers: string[],
+  maxRows: number,
+  startRow: number = 0,
+): Promise<ColumnsMap> {
+  // Initialize columns in array for construction efficiency
+  const initialCapacity = maxRows >= 0 ? maxRows : 1000
+  const columns: string[][] = headers.map(() => new Array<string>(initialCapacity))
+
+  maxRows = maxRows >= 0 ? maxRows : Infinity
+  let rowIndex = 0 // Keep in scope after loop
+  for (; rowIndex < maxRows; rowIndex++) {
+    const { done, value } = await reader.read()
+    if (done) break
+
+    // Expect a newline at the end of the file, but otherwise error on empty lines
+    if (!value) {
+      const nextRow = await reader.read()
+      if (nextRow.done) break
+      throw { code: 'TSV_EMPTY_LINE', line: rowIndex + startRow + 1 }
+    }
+
+    const values = value.split('\t')
+    if (values.length !== headers.length) {
+      throw { code: 'TSV_EQUAL_ROWS', line: rowIndex + startRow + 1 }
+    }
+    columns.forEach((column, columnIndex) => {
+      // Double array size if we exceed the current capacity
+      if (rowIndex >= column.length) {
+        column.length = column.length * 2
+      }
+      column[rowIndex] = values[columnIndex]
+    })
+  }
+
+  // Construct map, truncating columns to number of rows read
+  return new ColumnsMap(
+    headers.map((header, index) => [header, columns[index].slice(0, rowIndex)]),
+  )
+}
+
+export async function loadTSVGZ(
+  file: BIDSFile,
+  headers: string[],
+  maxRows: number = -1,
+): Promise<ColumnsMap> {
+  const reader = file.stream
+    .pipeThrough(new DecompressionStream('gzip'))
+    .pipeThrough(createUTF8Stream())
+    .pipeThrough(new TextLineStream())
+    .getReader()
+
+  try {
+    return await loadColumns(reader, headers, maxRows)
+  } catch (e: any) {
+    // Cancel the reader if we interrupted the read
+    // Cancelling for I/O errors will just re-trigger the error
+    if (e.code) {
+      await reader.cancel()
+      throw e
+    }
+    throw { code: 'INVALID_GZIP', location: file.path }
+  }
+}
+
 async function _loadTSV(file: BIDSFile, maxRows: number = -1): Promise<ColumnsMap> {
   const reader = file.stream
     .pipeThrough(createUTF8Stream())
@@ -26,40 +92,7 @@ async function _loadTSV(file: BIDSFile, maxRows: number = -1): Promise<ColumnsMa
       }
     }
 
-    // Initialize columns in array for construction efficiency
-    const initialCapacity = maxRows >= 0 ? maxRows : 1000
-    const columns: string[][] = headers.map(() => new Array<string>(initialCapacity))
-
-    maxRows = maxRows >= 0 ? maxRows : Infinity
-    let rowIndex = 0 // Keep in scope after loop
-    for (; rowIndex < maxRows; rowIndex++) {
-      const { done, value } = await reader.read()
-      if (done) break
-
-      // Expect a newline at the end of the file, but otherwise error on empty lines
-      if (!value) {
-        const nextRow = await reader.read()
-        if (nextRow.done) break
-        throw { code: 'TSV_EMPTY_LINE', location: file.path, line: rowIndex + 2 }
-      }
-
-      const values = value.split('\t')
-      if (values.length !== headers.length) {
-        throw { code: 'TSV_EQUAL_ROWS', location: file.path, line: rowIndex + 2 }
-      }
-      columns.forEach((column, columnIndex) => {
-        // Double array size if we exceed the current capacity
-        if (rowIndex >= column.length) {
-          column.length = column.length * 2
-        }
-        column[rowIndex] = values[columnIndex]
-      })
-    }
-
-    // Construct map, truncating columns to number of rows read
-    return new ColumnsMap(
-      headers.map((header, index) => [header, columns[index].slice(0, rowIndex)]),
-    )
+    return await loadColumns(reader, headers, maxRows, 1)
   } finally {
     await reader.cancel()
   }

src/issues/list.ts

@@ -133,6 +133,10 @@ export const bidsIssues: IssueDefinitionRecord = {
     reason:
       'A column required in a TSV file has been redefined in a sidecar file. This redefinition is being ignored.',
   },
+  INVALID_GZIP: {
+    severity: 'error',
+    reason: 'The gzip file could not be decompressed. It may be corrupt or misnamed.',
+  },
   MULTIPLE_INHERITABLE_FILES: {
     severity: 'error',
     reason: 'Multiple files in a directory were found to be valid candidates for inheritance.',

src/schema/context.ts

@@ -18,7 +18,7 @@ import { readEntities } from './entities.ts'
 import { DatasetIssues } from '../issues/datasetIssues.ts'
 import { walkBack } from '../files/inheritance.ts'
 import { parseGzip } from '../files/gzip.ts'
-import { loadTSV } from '../files/tsv.ts'
+import { loadTSV, loadTSVGZ } from '../files/tsv.ts'
 import { parseTIFF } from '../files/tiff.ts'
 import { loadJSON } from '../files/json.ts'
 import { loadHeader } from '../files/nifti.ts'
@@ -254,21 +254,40 @@ export class BIDSContext implements Context {
   }
 
   async loadColumns(): Promise<void> {
-    if (this.extension !== '.tsv') {
-      return
+    if (this.extension == '.tsv') {
+      this.columns = await loadTSV(this.file, this.dataset.options?.maxRows)
+        .catch((error) => {
+          if (error.code) {
+            this.dataset.issues.add({ ...error, location: this.file.path })
+          }
+          logger.warn(
+            `tsv file could not be opened by loadColumns '${this.file.path}'`,
+          )
+          logger.debug(error)
+          return new Map<string, string[]>() as ColumnsMap
+        }) as Record<string, string[]>
+    } else if (this.extension == '.tsv.gz') {
+      const headers = this.sidecar.Columns as string[];
+      if (!headers || this.size === 0) {
+        // Missing Columns will be caught by sidecar rules
+        // Note that these rules currently select for suffix, and will need to be generalized
+        // or duplicated for new .tsv.gz files
+        // `this.size === 0` will show as `EMPTY_FILE`, so do not add INVALID_GZIP
+        return
+      }
+      this.columns = await loadTSVGZ(this.file, headers, this.dataset.options?.maxRows)
+        .catch((error) => {
+          if (error.code) {
+            this.dataset.issues.add({ ...error, location: this.file.path })
+          }
+          logger.warn(
+            `tsv.gz file could not be opened by loadColumns '${this.file.path}'`,
+          )
+          logger.debug(error)
+          return new Map<string, string[]>() as ColumnsMap
+        }) as Record<string, string[]>
     }
 
-    this.columns = await loadTSV(this.file, this.dataset.options?.maxRows)
-      .catch((error) => {
-        if (error.code) {
-          this.dataset.issues.add({ ...error, location: this.file.path })
-        }
-        logger.warn(
-          `tsv file could not be opened by loadColumns '${this.file.path}'`,
-        )
-        logger.debug(error)
-        return new Map<string, string[]>() as ColumnsMap
-      }) as Record<string, string[]>
     return
   }
 
@@ -340,15 +359,24 @@ export class BIDSContext implements Context {
   }
 
   async asyncLoads() {
-    await Promise.allSettled([
-      this.loadSubjects(),
+    // loaders that may be depended on by other loaders
+    const initial = [
       this.loadSidecar(),
-      this.loadColumns(),
       this.loadAssociations(),
+    ]
+    // loaders that do not depend on other loaders
+    const independent = [
+      this.loadSubjects(),
       this.loadNiftiHeader(),
       this.loadJSON(),
       this.loadGzip(),
       this.loadTIFF(),
-    ])
+    ]
+
+    // Loaders with dependencies
+    await Promise.allSettled(initial)
+    await this.loadColumns()
+
+    await Promise.allSettled(independent)
   }
 }