throw utf8 encoding error for tsv files, add non utf-8 test data and test.

Author: rwblair

changelog.d/20250916_123856_rosswilsonblair_error_on_non_utf8_tsv.md

@@ -0,0 +1,47 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+For top level release notes, leave all the headers commented out.
+-->
+
+<!--
+### Added
+
+- A bullet item for the Added category.
+
+-->
+<!--
+### Changed
+
+- A bullet item for the Changed category.
+
+-->
+### Fixed
+
+- Throw utf-8 encoding error for tsv files similar to json files.
+
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category.
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category.
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category.
+
+-->
+<!--
+### Infrastructure
+
+- A bullet item for the Infrastructure category.
+
+-->

src/files/streams.ts

@@ -16,15 +16,23 @@ export class UTF8StreamTransformer implements Transformer<Uint8Array, string> {
   private decoder: TextDecoder
   private firstChunk: boolean
 
-  constructor() {
-    this.decoder = new TextDecoder('utf-8')
+  constructor(fatal: boolean = false) {
+    this.decoder = new TextDecoder('utf-8', {fatal})
     this.firstChunk = true
   }
 
   transform(chunk: Uint8Array, controller: TransformStreamDefaultController<string>) {
     // Check first chunk for UTF-16 BOM
     if (this.firstChunk) {
-      const decoded = this.decoder.decode(chunk, { stream: true })
+      let decoded
+      try {
+        decoded = this.decoder.decode(chunk, { stream: true })
+      } catch {
+        throw { code: 'INVALID_FILE_ENCODING' }
+      }
+      if (decoded === undefined) {
+        throw { code: 'FILE_DECODING_ERROR' }
+      }
       if (decoded.startsWith('\uFFFD')) {
         throw new UnicodeDecodeError('This file appears to be UTF-16')
       }
@@ -46,6 +54,6 @@ export class UTF8StreamTransformer implements Transformer<Uint8Array, string> {
 /**
  * Creates a TransformStream that validates and decodes UTF-8 text
  */
-export function createUTF8Stream() {
-  return new TransformStream(new UTF8StreamTransformer())
+export function createUTF8Stream(fatal: boolean = false) {
+  return new TransformStream(new UTF8StreamTransformer(fatal))
 }

src/files/tsv.test.ts

@@ -6,6 +6,7 @@ import {
   assertStrictEquals,
 } from '@std/assert'
 import { pathToFile } from './filetree.ts'
+import { BIDSFileDeno } from './deno.ts'
 import { loadTSV, loadTSVGZ } from './tsv.ts'
 import { streamFromString } from '../tests/utils.ts'
 import { ColumnsMap } from '../types/columns.ts'
@@ -182,6 +183,18 @@ Deno.test('TSV loading', async (t) => {
     }
   })
 
+  await t.step('Raises issue on non utf-8', async () => {
+    const file = new BIDSFileDeno('', './tests/data/iso8859.tsv')
+
+    try {
+      await loadTSV(file)
+      assert(false, 'Expected error')
+    } catch (e: any) {
+      assertObjectMatch(e, { code: 'INVALID_FILE_ENCODING' })
+    }
+  })
+
+
   // Tests will have populated the memoization cache
   loadTSV.cache.clear()
 })

src/files/tsv.ts

@@ -8,6 +8,7 @@ import type { BIDSFile } from '../types/filetree.ts'
 import { filememoizeAsync } from '../utils/memoize.ts'
 import { createUTF8Stream } from './streams.ts'
 import { openStream } from './access.ts'
+import { BIDSFileDeno } from './deno.ts'
 
 async function loadColumns(
   reader: ReadableStreamDefaultReader<string>,
@@ -58,7 +59,7 @@ export async function loadTSVGZ(
 ): Promise<ColumnsMap> {
   const reader = openStream(file)
     .pipeThrough(new DecompressionStream('gzip'))
-    .pipeThrough(createUTF8Stream())
+    .pipeThrough(createUTF8Stream(true))
     .pipeThrough(new TextLineStream())
     .getReader()
 
@@ -77,7 +78,7 @@ export async function loadTSVGZ(
 
 async function _loadTSV(file: BIDSFile, maxRows: number = -1): Promise<ColumnsMap> {
   const reader = openStream(file)
-    .pipeThrough(createUTF8Stream())
+    .pipeThrough(createUTF8Stream(true))
     .pipeThrough(new TextLineStream())
     .getReader()
 

tests/data/iso8859.tsv

@@ -0,0 +1,3 @@
+Name	Age	City
+José	30	São Paulo
+Müller	25	München