An attempt to convert parquet files to columnmaps.

Author: rwblair

deno.json

@@ -32,6 +32,7 @@
     "@cliffy/command": "jsr:@effigies/[email protected]",
     "@cliffy/table": "jsr:@effigies/[email protected]",
     "@hed/validator": "npm:[email protected]",
+    "@hyparquet": "npm:[email protected]",
     "@ignore": "npm:[email protected]",
     "@libs/xml": "jsr:@libs/[email protected]",
     "@mango/nifti": "npm:@bids/[email protected]",

src/files/parquet.test.ts

@@ -0,0 +1,18 @@
+import { assertEquals } from '@std/assert'
+import { FileIgnoreRules } from './ignore.ts'
+import { BIDSFileDeno } from './deno.ts'
+
+import { loadParquet } from './parquet.ts'
+
+Deno.test('Test loading parquet file', async (t) => {
+  const ignore = new FileIgnoreRules([])
+  await t.step('Load participants.parquet', async () => {
+    const path = 'participants.parquet'
+    const root = './tests/data/'
+    const file = new BIDSFileDeno(root, path, ignore)
+    const participantsMap = await loadParquet(file)
+    const keys = Object.keys(participantsMap)
+    assertEquals(keys.length, 3)
+    keys.map(key => assertEquals(participantsMap.get(key)?.length, 16))
+  })
+})

src/files/parquet.ts

@@ -0,0 +1,15 @@
+import { asyncBufferFromFile, parquetRead, ParquetReadOptions, ColumnData } from '@hyparquet'
+import type { BIDSFile } from '../types/filetree.ts'
+import { ColumnsMap } from '../types/columns.ts'
+import { createUTF8Stream } from './streams.ts'
+
+export async function loadParquet(file: BIDSFile, maxRows: number = -1): Promise<ColumnsMap> {
+  let columnsMap = new ColumnsMap()
+  const readOpts: ParquetReadOptions = {
+    file: (await file.readBytes(file.size)).buffer,
+    // @ts-expect-error
+    onChunk: (data) => columnsMap[data.columnName] = data.columnData.map(entry => String(entry)), 
+  }
+  await parquetRead(readOpts)
+  return columnsMap
+}