Merge pull request #280 from effigies/proxy-files

rf: Make BIDSFile generic wrapper, with FileOpener providing the file type handling

Author: rwblair

changelog.d/20251013_111301_markiewicz_proxy_files.md

@@ -0,0 +1,3 @@
+### Changed
+
+- Refactored file access classes to be more DRY.

src/files/access.ts

@@ -10,12 +10,12 @@ function IOErrorToIssue(err: { code: string; name: string }): Issue {
   return { code: 'FILE_READ', subCode: err.name, issueMessage }
 }
 
-export function openStream(file: BIDSFile): ReadableStream<Uint8Array<ArrayBuffer>> {
-  try {
-    return file.stream
-  } catch (err: any) {
+export async function openStream(
+  file: BIDSFile,
+): Promise<ReadableStream<Uint8Array<ArrayBuffer>>> {
+  return file.stream().catch((err: any) => {
     throw { location: file.path, ...IOErrorToIssue(err) }
-  }
+  })
 }
 
 export async function readBytes(

src/files/browser.ts

@@ -1,56 +1,17 @@
-import { type BIDSFile, FileTree } from '../types/filetree.ts'
+import { BIDSFile, FileTree } from '../types/filetree.ts'
 import { filesToTree } from './filetree.ts'
 import { FileIgnoreRules, readBidsIgnore } from './ignore.ts'
-import { parse, SEPARATOR_PATTERN } from '@std/path'
-import * as posix from '@std/path/posix'
+import { BrowserFileOpener } from './openers.ts'
 
 /**
  * Browser implement of BIDSFile wrapping native File/FileList types
  */
-export class BIDSFileBrowser implements BIDSFile {
-  #ignore: FileIgnoreRules
-  #file: File
-  name: string
-  path: string
-  #parent!: WeakRef<FileTree>
-  viewed: boolean = false
-
+export class BIDSFileBrowser extends BIDSFile {
   constructor(file: File, ignore: FileIgnoreRules, parent?: FileTree) {
-    this.#file = file
-    this.#ignore = ignore
-    this.name = file.name
-    const relativePath = this.#file.webkitRelativePath
+    const relativePath = file.webkitRelativePath
     const prefixLength = relativePath.indexOf('/')
-    this.path = relativePath.substring(prefixLength)
-    this.parent = parent ?? new FileTree('', '/', undefined)
-  }
-
-  get parent(): FileTree {
-    return this.#parent.deref() as FileTree
-  }
-
-  set parent(tree: FileTree) {
-    this.#parent = new WeakRef(tree)
-  }
-
-  get size(): number {
-    return this.#file.size
-  }
-
-  get stream(): ReadableStream<Uint8Array<ArrayBuffer>> {
-    return this.#file.stream() as ReadableStream<Uint8Array<ArrayBuffer>>
-  }
-
-  get ignored(): boolean {
-    return this.#ignore.test(this.path)
-  }
-
-  text(): Promise<string> {
-    return this.#file.text()
-  }
-
-  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
-    return new Uint8Array(await this.#file.slice(offset, size).arrayBuffer())
+    const opener = new BrowserFileOpener(file)
+    super(relativePath.substring(prefixLength), opener, ignore, parent)
   }
 }
 

src/files/deno.test.ts

@@ -31,7 +31,7 @@ Deno.test('Deno implementation of BIDSFile', async (t) => {
   })
   await t.step('can be read as ReadableStream', async () => {
     const file = new BIDSFileDeno(testDir, testFilename, ignore)
-    const stream = file.stream
+    const stream = await file.stream()
     const streamReader = stream.getReader()
     const denoReader = readerFromStreamReader(streamReader)
     const fileBuffer = await readAll(denoReader)

src/files/deno.ts

@@ -3,105 +3,14 @@
  */
 import { basename, join } from '@std/path'
 import * as posix from '@std/path/posix'
-import { type BIDSFile, FileTree } from '../types/filetree.ts'
+import { BIDSFile, FileTree } from '../types/filetree.ts'
 import { requestReadPermission } from '../setup/requestPermissions.ts'
 import { FileIgnoreRules, readBidsIgnore } from './ignore.ts'
-import { logger } from '../utils/logger.ts'
-import { createUTF8Stream } from './streams.ts'
-export { type BIDSFile, FileTree }
-
-/**
- * Deno implementation of BIDSFile
- */
-export class BIDSFileDeno implements BIDSFile {
-  #ignore: FileIgnoreRules
-  name: string
-  path: string
-  #parent!: WeakRef<FileTree>
-  #fileInfo?: Deno.FileInfo
-  #datasetAbsPath: string
-  viewed: boolean = false
+import { FsFileOpener } from './openers.ts'
 
+export class BIDSFileDeno extends BIDSFile {
   constructor(datasetPath: string, path: string, ignore?: FileIgnoreRules, parent?: FileTree) {
-    this.#datasetAbsPath = datasetPath
-    this.path = path
-    this.name = basename(path)
-    this.#ignore = ignore ?? new FileIgnoreRules([])
-    try {
-      this.#fileInfo = Deno.statSync(this._getPath())
-    } catch (error) {
-      if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') {
-        this.#fileInfo = Deno.lstatSync(this._getPath())
-      }
-    }
-    this.parent = parent ?? new FileTree('', '/', undefined)
-  }
-
-  private _getPath(): string {
-    return join(this.#datasetAbsPath, this.path)
-  }
-
-  get parent(): FileTree {
-    return this.#parent.deref() as FileTree
-  }
-
-  set parent(tree: FileTree) {
-    this.#parent = new WeakRef(tree)
-  }
-
-  get size(): number {
-    return this.#fileInfo ? this.#fileInfo.size : -1
-  }
-
-  get stream(): ReadableStream<Uint8Array<ArrayBuffer>> {
-    const handle = this.#openHandle()
-    return handle.readable
-  }
-
-  get ignored(): boolean {
-    return this.#ignore.test(this.path)
-  }
-
-  /**
-   * Read the entire file and decode as utf-8 text
-   */
-  async text(): Promise<string> {
-    const reader = this.stream.pipeThrough(createUTF8Stream()).getReader()
-    const chunks: string[] = []
-    try {
-      while (true) {
-        const { done, value } = await reader.read()
-        if (done) break
-        chunks.push(value)
-      }
-      return chunks.join('')
-    } finally {
-      reader.releaseLock()
-    }
-  }
-
-  /**
-   * Read bytes in a range efficiently from a given file
-   *
-   * Reads up to size bytes, starting at offset.
-   * If EOF is encountered, the resulting array may be smaller.
-   */
-  async readBytes(size: number, offset = 0): Promise<Uint8Array<ArrayBuffer>> {
-    const handle = this.#openHandle()
-    const buf = new Uint8Array(size)
-    await handle.seek(offset, Deno.SeekMode.Start)
-    const nbytes = await handle.read(buf) ?? 0
-    handle.close()
-    return buf.subarray(0, nbytes)
-  }
-
-  /**
-   * Return a Deno file handle
-   */
-  #openHandle(): Deno.FsFile {
-    // Avoid asking for write access
-    const openOptions = { read: true, write: false }
-    return Deno.openSync(this._getPath(), openOptions)
+    super(path, new FsFileOpener(datasetPath, path), ignore, parent)
   }
 }
 
@@ -122,12 +31,12 @@ async function _readFileTree(
       continue
     }
     if (dirEntry.isFile || dirEntry.isSymlink) {
-      const file = new BIDSFileDeno(
-        rootPath,
+      const file = new BIDSFile(
         thisPath,
+        new FsFileOpener(rootPath, thisPath),
         ignore,
+        tree,
       )
-      file.parent = tree
       tree.files.push(file)
     }
     if (dirEntry.isDirectory) {

src/files/filetree.test.ts

@@ -1,7 +1,25 @@
 import { assert, assertEquals } from '@std/assert'
-import type { FileIgnoreRules } from './ignore.ts'
-import type { BIDSFile, FileTree } from '../types/filetree.ts'
-import { filesToTree, pathsToTree } from './filetree.ts'
+import { FileIgnoreRules } from './ignore.ts'
+import { BIDSFile, type FileOpener, type FileTree } from '../types/filetree.ts'
+import { filesToTree } from './filetree.ts'
+import { asyncStreamFromString } from '../tests/utils.ts'
+
+class NullFileOpener implements FileOpener {
+  size = 0
+  stream = () => asyncStreamFromString('')
+  text = () => Promise.resolve('')
+  readBytes = async (size: number, offset?: number) => new Uint8Array()
+}
+
+export function pathToFile(path: string, ignored: boolean = false): BIDSFile {
+  const name = path.split('/').pop() as string
+  return new BIDSFile(path, new NullFileOpener(), ignored)
+}
+
+export function pathsToTree(paths: string[], ignore?: string[]): FileTree {
+  const ignoreRules = new FileIgnoreRules(ignore ?? [])
+  return filesToTree(paths.map((path) => pathToFile(path, ignoreRules.test(path))))
+}
 
 Deno.test('FileTree generation', async (t) => {
   await t.step('converts a basic list', async () => {

src/files/filetree.ts

@@ -1,31 +1,8 @@
 import { parse, SEPARATOR_PATTERN } from '@std/path'
 import * as posix from '@std/path/posix'
-import { type BIDSFile, FileTree } from '../types/filetree.ts'
+import { BIDSFile, FileTree } from '../types/filetree.ts'
 import { FileIgnoreRules } from './ignore.ts'
 
-const nullFile = {
-  size: 0,
-  stream: new ReadableStream({
-    start(controller) {
-      controller.close()
-    },
-  }),
-  text: () => Promise.resolve(''),
-  readBytes: async (size: number, offset?: number) => new Uint8Array(),
-  parent: new FileTree('', '/'),
-  viewed: false,
-}
-
-export function pathToFile(path: string, ignored: boolean = false): BIDSFile {
-  const name = path.split('/').pop() as string
-  return { name, path, ignored, ...nullFile }
-}
-
-export function pathsToTree(paths: string[], ignore?: string[]): FileTree {
-  const ignoreRules = new FileIgnoreRules(ignore ?? [])
-  return filesToTree(paths.map((path) => pathToFile(path, ignoreRules.test(path))))
-}
-
 export function filesToTree(fileList: BIDSFile[], ignore?: FileIgnoreRules): FileTree {
   ignore = ignore ?? new FileIgnoreRules([])
   const tree: FileTree = new FileTree('/', '/')

src/files/inheritance.test.ts

@@ -1,6 +1,6 @@
 import { assert, assertEquals, assertThrows } from '@std/assert'
 import type { BIDSFile } from '../types/filetree.ts'
-import { pathsToTree } from './filetree.ts'
+import { pathsToTree } from './filetree.test.ts'
 import { walkBack } from './inheritance.ts'
 
 Deno.test('walkback inheritance tests', async (t) => {

src/files/json.test.ts

@@ -3,7 +3,7 @@ import type { BIDSFile } from '../types/filetree.ts'
 import type { FileIgnoreRules } from './ignore.ts'
 import { testAsyncFileAccess } from './access.test.ts'
 
-import { pathsToTree } from '../files/filetree.ts'
+import { pathsToTree } from '../files/filetree.test.ts'
 import { loadJSON } from './json.ts'
 
 function encodeUTF16(text: string) {