Merge pull request #287 from effigies/fix/s3

feat: Fall back to an empty file for absent symlinks with no public S3 URL

Author: rwblair

changelog.d/20251030_132706_markiewicz_s3.md

@@ -0,0 +1,5 @@
+### Changed
+
+- [Annexed](https://git-annex.branchable.com/) files that are not local are treated as
+  empty instead of erroring if a remote URL could not be constructed. The size of missing
+  files are included in the summary estimate of dataset size.

src/files/deno.ts

@@ -6,8 +6,8 @@ import * as posix from '@std/path/posix'
 import { BIDSFile, type FileOpener, FileTree } from '../types/filetree.ts'
 import { requestReadPermission } from '../setup/requestPermissions.ts'
 import { FileIgnoreRules, readBidsIgnore } from './ignore.ts'
-import { FsFileOpener, HTTPOpener } from './openers.ts'
-import { resolveAnnexedFile } from './repo.ts'
+import { FsFileOpener, HTTPOpener, NullFileOpener } from './openers.ts'
+import { parseAnnexedFile, resolveAnnexedFile } from './repo.ts'
 import fs from 'node:fs'
 
 export class BIDSFileDeno extends BIDSFile {
@@ -51,12 +51,13 @@ async function _readFileTree({
       try {
         const fileInfo = await Deno.stat(fullPath)
         opener = new FsFileOpener(rootPath, thisPath, fileInfo)
-      } catch (error) {
+      } catch (_) {
+        const { key, size, gitdir } = await parseAnnexedFile(fullPath)
         try {
-          const { url, size } = await resolveAnnexedFile(fullPath, preferredRemote, { cache, fs })
+          const { url } = await resolveAnnexedFile(key, preferredRemote, { cache, fs, gitdir })
           opener = new HTTPOpener(url, size)
         } catch (_) {
-          throw error
+          opener = new NullFileOpener(size)
         }
       }
       tree.files.push(new BIDSFile(thisPath, opener, ignore, tree))

src/files/openers.ts

@@ -126,3 +126,18 @@ export class HTTPOpener implements FileOpener {
     return new Uint8Array(await response.arrayBuffer())
   }
 }
+
+export class NullFileOpener implements FileOpener {
+  size: number
+  constructor(size = 0) {
+    this.size = size
+  }
+  stream = async () =>
+    new ReadableStream({
+      start(controller) {
+        controller.close()
+      },
+    })
+  text = async () => ''
+  readBytes = async (size: number, offset?: number) => new Uint8Array()
+}

src/files/repo.ts

@@ -84,27 +84,29 @@ export async function readRemotes(options: any): Promise<Record<string, Record<s
   return byUUID
 }
 
-/**
- * Resolve an annexed file location to an HTTP URL, if a public S3 remote is available
- */
-export async function resolveAnnexedFile(
+export async function parseAnnexedFile(
   path: string,
-  remote?: string,
-  options?: any,
-): Promise<{ url: string; size: number }> {
-  // path is known to be a symlink
+): Promise<{ key: string; size: number; gitdir: string }> {
   const target = await Deno.readLink(path)
   const { dir, base } = parse(target)
 
-  if (!options?.gitdir) {
-    const dirs = dir.split(SEPARATOR_PATTERN)
-    const gitdir = join(dirname(path), ...dirs.slice(0, dirs.indexOf('.git') + 1))
-    options = { ...options, gitdir }
-  }
+  const dirs = dir.split(SEPARATOR_PATTERN)
+  const gitdir = join(dirname(path), ...dirs.slice(0, dirs.indexOf('.git') + 1))
 
   const size = +base.match(annexKeyRegex)?.groups?.size!
 
-  const rmet = await readRmet(base, options)
+  return { key: base, size, gitdir }
+}
+
+/**
+ * Resolve an annexed file location to an HTTP URL, if a public S3 remote is available
+ */
+export async function resolveAnnexedFile(
+  key: string,
+  remote?: string,
+  options?: any,
+): Promise<{ url: string }> {
+  const rmet = await readRmet(key, options)
   const remotes = await readRemotes(options)
   let uuid: string
   if (remote) {
@@ -137,5 +139,5 @@ export async function resolveAnnexedFile(
   const metadata = rmet[uuid]
   const url = `${publicurl}/${metadata.path}?versionId=${metadata.version}`
 
-  return { url, size }
+  return { url }
 }

src/schema/walk.ts

@@ -1,6 +1,7 @@
 import { BIDSContext, type BIDSContextDataset } from './context.ts'
-import { BIDSFile, FileOpener, FileTree } from '../types/filetree.ts'
+import { BIDSFile, type FileTree } from '../types/filetree.ts'
 import type { DatasetIssues } from '../issues/datasetIssues.ts'
+import { NullFileOpener } from '../files/openers.ts'
 import { loadTSV } from '../files/tsv.ts'
 import { loadJSON } from '../files/json.ts'
 import { queuedAsyncIterator } from '../utils/queue.ts'
@@ -14,21 +15,6 @@ function* quickWalk(dir: FileTree): Generator<BIDSFile> {
   }
 }
 
-class NullFileOpener implements FileOpener {
-  size: number
-  constructor(size = 0) {
-    this.size = size
-  }
-  stream = async () =>
-    new ReadableStream({
-      start(controller) {
-        controller.close()
-      },
-    })
-  text = async () => ''
-  readBytes = async (size: number, offset?: number) => new Uint8Array()
-}
-
 function pseudoFile(dir: FileTree, opaque: boolean): BIDSFile {
   return new BIDSFile(
     // Use a trailing slash to indicate directory