Merge pull request bids-standard#264 from bids-standard/enh/error_on_non_utf8_tsv

throw utf8 encoding error for tsv files, add non utf-8 test data and test.

Author: effigies

changelog.d/20250916_123856_rosswilsonblair_error_on_non_utf8_tsv.md

@@ -0,0 +1,47 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+For top level release notes, leave all the headers commented out.
+-->
+
+<!--
+### Added
+
+- A bullet item for the Added category.
+
+-->
+<!--
+### Changed
+
+- A bullet item for the Changed category.
+
+-->
+### Fixed
+
+- Throw utf-8 encoding error for tsv files similar to json files.
+
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category.
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category.
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category.
+
+-->
+<!--
+### Infrastructure
+
+- A bullet item for the Infrastructure category.
+
+-->

src/files/streams.ts

@@ -8,6 +8,16 @@ export class UnicodeDecodeError extends Error {
   }
 }
 
+const _decode = TextDecoder.prototype.decode
+
+TextDecoder.prototype.decode = function(input, options) {
+  try {
+    return _decode.call(this, input, options)
+  } catch (error) {
+    throw { code: 'INVALID_FILE_ENCODING', message: error}
+  }
+}
+
 /**
  * A transformer that ensures the input stream is valid UTF-8 and throws
  * a UnicodeDecodeError if UTF-16 BOM is detected
@@ -16,15 +26,15 @@ export class UTF8StreamTransformer implements Transformer<Uint8Array, string> {
   private decoder: TextDecoder
   private firstChunk: boolean
 
-  constructor() {
-    this.decoder = new TextDecoder('utf-8')
+  constructor(options = {fatal: false}) {
+    this.decoder = new TextDecoder('utf-8', options)
     this.firstChunk = true
   }
 
   transform(chunk: Uint8Array, controller: TransformStreamDefaultController<string>) {
     // Check first chunk for UTF-16 BOM
     if (this.firstChunk) {
-      const decoded = this.decoder.decode(chunk, { stream: true })
+      let decoded = this.decoder.decode(chunk, { stream: true })
       if (decoded.startsWith('\uFFFD')) {
         throw new UnicodeDecodeError('This file appears to be UTF-16')
       }
@@ -46,6 +56,6 @@ export class UTF8StreamTransformer implements Transformer<Uint8Array, string> {
 /**
  * Creates a TransformStream that validates and decodes UTF-8 text
  */
-export function createUTF8Stream() {
-  return new TransformStream(new UTF8StreamTransformer())
+export function createUTF8Stream(options = {fatal: false}) {
+  return new TransformStream(new UTF8StreamTransformer(options))
 }

src/files/tsv.test.ts

@@ -6,6 +6,7 @@ import {
   assertStrictEquals,
 } from '@std/assert'
 import { pathToFile } from './filetree.ts'
+import { BIDSFileDeno } from './deno.ts'
 import { loadTSV, loadTSVGZ } from './tsv.ts'
 import { streamFromString } from '../tests/utils.ts'
 import { ColumnsMap } from '../types/columns.ts'
@@ -182,6 +183,18 @@ Deno.test('TSV loading', async (t) => {
     }
   })
 
+  await t.step('Raises issue on non utf-8', async () => {
+    const file = new BIDSFileDeno('', './tests/data/iso8859.tsv')
+
+    try {
+      await loadTSV(file)
+      assert(false, 'Expected error')
+    } catch (e: any) {
+      assertObjectMatch(e, { code: 'INVALID_FILE_ENCODING' })
+    }
+  })
+
+
   // Tests will have populated the memoization cache
   loadTSV.cache.clear()
 })

src/files/tsv.ts

@@ -8,6 +8,7 @@ import type { BIDSFile } from '../types/filetree.ts'
 import { filememoizeAsync } from '../utils/memoize.ts'
 import { createUTF8Stream } from './streams.ts'
 import { openStream } from './access.ts'
+import { BIDSFileDeno } from './deno.ts'
 
 async function loadColumns(
   reader: ReadableStreamDefaultReader<string>,
@@ -58,7 +59,7 @@ export async function loadTSVGZ(
 ): Promise<ColumnsMap> {
   const reader = openStream(file)
     .pipeThrough(new DecompressionStream('gzip'))
-    .pipeThrough(createUTF8Stream())
+    .pipeThrough(createUTF8Stream({ fatal: true }))
     .pipeThrough(new TextLineStream())
     .getReader()
 
@@ -77,7 +78,7 @@ export async function loadTSVGZ(
 
 async function _loadTSV(file: BIDSFile, maxRows: number = -1): Promise<ColumnsMap> {
   const reader = openStream(file)
-    .pipeThrough(createUTF8Stream())
+    .pipeThrough(createUTF8Stream({ fatal: true }))
     .pipeThrough(new TextLineStream())
     .getReader()
 

tests/data/iso8859.tsv

@@ -0,0 +1,3 @@
+Name	Age	City
+José	30	São Paulo
+Müller	25	München