Add ability to reload local files across page using FileSystemFileHandle API (#5419)

Author: cmdcolin

packages/core/package.json

@@ -131,7 +131,8 @@
     "./util/useMeasure": "./src/util/useMeasure.ts",
     "./ui/ReturnToImportFormDialog": "./src/ui/ReturnToImportFormDialog.tsx",
     "./util/nanoid": "./src/util/nanoid.ts",
-    "./ReExports/list": "./src/ReExports/list.ts"
+    "./ReExports/list": "./src/ReExports/list.ts",
+    "./util/fileHandleStore": "./src/util/fileHandleStore.ts"
   },
   "files": [
     "esm"
@@ -177,6 +178,7 @@
     "fast-deep-equal": "^3.1.3",
     "file-saver-es": "^2.0.5",
     "generic-filehandle2": "^2.0.18",
+    "idb": "^8.0.3",
     "librpc-web-mod": "^2.1.1",
     "load-script": "^2.0.0",
     "mobx": "^6.15.0",
@@ -633,6 +635,10 @@
       "./ReExports/list": {
         "types": "./esm/ReExports/list.d.ts",
         "import": "./esm/ReExports/list.js"
+      },
+      "./util/fileHandleStore": {
+        "types": "./esm/util/fileHandleStore.d.ts",
+        "import": "./esm/util/fileHandleStore.js"
       }
     }
   },

packages/core/scripts/generateExports.mjs

@@ -9,7 +9,11 @@ const repoRoot = join(packageRoot, '../..')
 const srcDir = join(packageRoot, 'src')
 
 // Exports to keep even if not used internally (for backwards compatibility)
-const preservedExports = ['@jbrowse/core/util/nanoid']
+const preservedExports = [
+  '@jbrowse/core/util/nanoid',
+  '@jbrowse/core/ReExports/list',
+  '@jbrowse/core/util/fileHandleStore',
+]
 
 // Scan the codebase for all @jbrowse/core imports
 function findAllImports() {

packages/core/src/pluggableElementTypes/RpcMethodType.test.ts

@@ -1,5 +1,6 @@
 import PluginManager from '../PluginManager.ts'
-import RpcMethodType from './RpcMethodType.ts'
+import RpcMethodType, { convertFileHandleLocations } from './RpcMethodType.ts'
+import { clearFileFromCache, setFileInCache } from '../util/tracks.ts'
 
 const pluginManager = new PluginManager()
 
@@ -51,3 +52,172 @@ test('test serialize arguments with augmentLocationObject', async () => {
     '',
   )
 })
+
+describe('convertFileHandleLocations', () => {
+  afterEach(() => {
+    clearFileFromCache('test-handle-1')
+    clearFileFromCache('test-handle-2')
+    clearFileFromCache('test-handle-nested')
+  })
+
+  test('converts FileHandleLocation to BlobLocation in object property', () => {
+    const mockFile = new File(['test content'], 'test.bam', {
+      type: 'application/octet-stream',
+    })
+    setFileInCache('test-handle-1', mockFile)
+
+    const obj = {
+      adapter: {
+        fileLocation: {
+          locationType: 'FileHandleLocation',
+          handleId: 'test-handle-1',
+          name: 'test.bam',
+        },
+      },
+    }
+    const blobMap: Record<string, File> = {}
+
+    convertFileHandleLocations(obj, blobMap)
+
+    const converted = obj.adapter.fileLocation as any
+    expect(converted.locationType).toBe('BlobLocation')
+    expect(converted.name).toBe('test.bam')
+    expect(converted.blobId).toBe('fh-blob-test-handle-1')
+    expect(blobMap[converted.blobId]).toBe(mockFile)
+  })
+
+  test('converts FileHandleLocation in array', () => {
+    const mockFile = new File(['test'], 'array-file.vcf')
+    setFileInCache('test-handle-1', mockFile)
+
+    const obj = {
+      locations: [
+        {
+          locationType: 'FileHandleLocation',
+          handleId: 'test-handle-1',
+          name: 'array-file.vcf',
+        },
+      ],
+    }
+    const blobMap: Record<string, File> = {}
+
+    convertFileHandleLocations(obj, blobMap)
+
+    const converted = obj.locations[0] as any
+    expect(converted.locationType).toBe('BlobLocation')
+    expect(converted.name).toBe('array-file.vcf')
+    expect(blobMap[converted.blobId]).toBe(mockFile)
+  })
+
+  test('converts multiple FileHandleLocations', () => {
+    const mockFile1 = new File(['content1'], 'file1.bam')
+    const mockFile2 = new File(['content2'], 'file2.bam.bai')
+    setFileInCache('test-handle-1', mockFile1)
+    setFileInCache('test-handle-2', mockFile2)
+
+    const obj = {
+      bamLocation: {
+        locationType: 'FileHandleLocation',
+        handleId: 'test-handle-1',
+        name: 'file1.bam',
+      },
+      indexLocation: {
+        locationType: 'FileHandleLocation',
+        handleId: 'test-handle-2',
+        name: 'file2.bam.bai',
+      },
+    }
+    const blobMap: Record<string, File> = {}
+
+    convertFileHandleLocations(obj, blobMap)
+
+    expect((obj.bamLocation as any).locationType).toBe('BlobLocation')
+    expect((obj.indexLocation as any).locationType).toBe('BlobLocation')
+    expect(Object.keys(blobMap).length).toBe(2)
+  })
+
+  test('handles deeply nested FileHandleLocation', () => {
+    const mockFile = new File(['nested'], 'nested.gff')
+    setFileInCache('test-handle-nested', mockFile)
+
+    const obj = {
+      level1: {
+        level2: {
+          level3: {
+            location: {
+              locationType: 'FileHandleLocation',
+              handleId: 'test-handle-nested',
+              name: 'nested.gff',
+            },
+          },
+        },
+      },
+    }
+    const blobMap: Record<string, File> = {}
+
+    convertFileHandleLocations(obj, blobMap)
+
+    const converted = obj.level1.level2.level3.location as any
+    expect(converted.locationType).toBe('BlobLocation')
+    expect(blobMap[converted.blobId]).toBe(mockFile)
+  })
+
+  test('leaves non-FileHandleLocation objects unchanged', () => {
+    const obj = {
+      uriLocation: {
+        locationType: 'UriLocation',
+        uri: 'https://example.com/file.bam',
+      },
+      blobLocation: {
+        locationType: 'BlobLocation',
+        blobId: 'existing-blob',
+        name: 'existing.bam',
+      },
+    }
+    const blobMap: Record<string, File> = {}
+
+    convertFileHandleLocations(obj, blobMap)
+
+    expect(obj.uriLocation.locationType).toBe('UriLocation')
+    expect(obj.blobLocation.locationType).toBe('BlobLocation')
+    expect(obj.blobLocation.blobId).toBe('existing-blob')
+    expect(Object.keys(blobMap).length).toBe(0)
+  })
+
+  test('throws error when file not in cache', () => {
+    const obj = {
+      location: {
+        locationType: 'FileHandleLocation',
+        handleId: 'nonexistent-handle',
+        name: 'missing.bam',
+      },
+    }
+    const blobMap: Record<string, File> = {}
+
+    expect(() => {
+      convertFileHandleLocations(obj, blobMap)
+    }).toThrow(/File not in cache for handleId: nonexistent-handle/)
+  })
+
+  test('handles circular references without infinite loop', () => {
+    const mockFile = new File(['circular'], 'circular.bam')
+    setFileInCache('test-handle-1', mockFile)
+
+    const obj: Record<string, unknown> = {
+      location: {
+        locationType: 'FileHandleLocation',
+        handleId: 'test-handle-1',
+        name: 'circular.bam',
+      },
+    }
+    obj.self = obj
+
+    const blobMap: Record<string, File> = {}
+
+    convertFileHandleLocations(obj, blobMap)
+
+    const converted = obj.location as any
+    expect(converted.locationType).toBe('BlobLocation')
+    expect(Object.keys(blobMap).length).toBe(1)
+  })
+})

packages/core/src/pluggableElementTypes/RpcMethodType.ts

@@ -1,19 +1,76 @@
 import PluggableElementBase from './PluggableElementBase.ts'
 import mapObject from '../util/map-obj/index.ts'
 import { isRpcResult } from '../util/rpc.ts'
-import { getBlobMap, setBlobMap } from '../util/tracks.ts'
+import { getBlobMap, getFileFromCache, setBlobMap } from '../util/tracks.ts'
 import {
   RetryError,
   isAppRootModel,
   isAuthNeededException,
+  isFileHandleLocation,
   isUriLocation,
 } from '../util/types/index.ts'
 
 import type PluginManager from '../PluginManager.ts'
-import type { UriLocation } from '../util/types/index.ts'
+import type { FileHandleLocation, UriLocation } from '../util/types/index.ts'
 
 export type RpcMethodConstructor = new (pm: PluginManager) => RpcMethodType
 
+// Note: We use custom recursion instead of mapObject here because mapObject's
+// mapper function only receives object properties, not array items directly.
+// FileHandleLocation objects can appear as array elements (e.g., in adapter
+// configs with multiple file locations), so we need direct array item access.
+export function convertFileHandleLocations(
+  obj: unknown,
+  blobMap: Record<string, File>,
+  seen = new WeakSet<object>(),
+) {
+  const convertLocation = (loc: FileHandleLocation) => {
+    const file = getFileFromCache(loc.handleId)
+    if (!file) {
+      throw new Error(
+        `File not in cache for handleId: ${loc.handleId}. ` +
+          `The file "${loc.name}" may need to be reopened.`,
+      )
+    }
+    // Use deterministic blobId based on handleId so the same FileHandleLocation
+    // always converts to the same BlobLocation. This ensures adapter config
+    // hashes remain stable across render calls.
+    const blobId = `fh-blob-${loc.handleId}`
+    blobMap[blobId] = file
+    return { locationType: 'BlobLocation' as const, name: loc.name, blobId }
+  }
+
+  const convert = (current: unknown): void => {
+    if (!current || typeof current !== 'object' || seen.has(current)) {
+      return
+    }
+    seen.add(current)
+
+    if (Array.isArray(current)) {
+      for (let i = 0; i < current.length; i++) {
+        const item = current[i]
+        if (isFileHandleLocation(item)) {
+          current[i] = convertLocation(item)
+        } else {
+          convert(item)
+        }
+      }
+    } else {
+      const record = current as Record<string, unknown>
+      for (const key of Object.keys(record)) {
+        const val = record[key]
+        if (isFileHandleLocation(val)) {
+          record[key] = convertLocation(val)
+        } else {
+          convert(val)
+        }
+      }
+    }
+  }
+
+  convert(obj)
+}
+
 export default abstract class RpcMethodType extends PluggableElementBase {
   pluginManager: PluginManager
 
@@ -127,20 +184,15 @@ export default abstract class RpcMethodType extends PluggableElementBase {
   ) {
     const rootModel = this.pluginManager.rootModel
 
-    // Skip expensive deep traversal only if we have a valid root model with no internet accounts
-    // (Don't skip if rootModel isn't set up - let serializeNewAuthArguments handle that case)
-    if (isAppRootModel(rootModel) && rootModel.internetAccounts.length === 0) {
-      return thing
-    }
-
     const uris = [] as UriLocation[]
+    const blobMap = getBlobMap()
 
-    // exclude renderingProps from deep traversal - it is only needed
-    // client-side for React components and can contain circular references
-    // (e.g. d3 hierarchy nodes) or non-serializable objects like callbacks
+    // Convert FileHandleLocation to BlobLocation in-place
+    // Skip renderingProps as it may have circular references
     const { renderingProps, ...rest } = thing
+    convertFileHandleLocations(rest, blobMap)
 
-    // using map-obj avoids cycles, seen in circular view svg export
+    // Collect UriLocations for auth augmentation using map-obj (handles cycles)
     mapObject(
       rest,
       (key, val: unknown) => {
@@ -158,6 +210,12 @@ export default abstract class RpcMethodType extends PluggableElementBase {
       },
       { deep: true },
     )
+
+    // Skip URI auth augmentation if we have a valid root model with no internet accounts
+    if (isAppRootModel(rootModel) && rootModel.internetAccounts.length === 0) {
+      return thing
+    }
+
     for (const uri of uris) {
       await this.serializeNewAuthArguments(uri, rpcDriverClassName)
     }

packages/core/src/pluggableElementTypes/renderers/LayoutSession.ts

@@ -57,14 +57,15 @@ export class LayoutSession implements LayoutSessionLike {
   }
 
   cachedLayoutIsValid(cachedLayout: CachedLayout) {
-    return (
-      cachedLayout.props.bpPerPx === this.props.bpPerPx &&
-      deepEqual(
-        readConfObject(this.props.config),
-        readConfObject(cachedLayout.props.config),
-      ) &&
-      deepEqual(this.props.filters, cachedLayout.props.filters)
+    const bpPerPxMatch = cachedLayout.props.bpPerPx === this.props.bpPerPx
+    const currentConfig = readConfObject(this.props.config)
+    const cachedConfig = readConfObject(cachedLayout.props.config)
+    const configMatch = deepEqual(currentConfig, cachedConfig)
+    const filtersMatch = deepEqual(
+      this.props.filters,
+      cachedLayout.props.filters,
     )
+    return bpPerPxMatch && configMatch && filtersMatch
   }
 
   get layout(): BaseMultiLayout {

packages/core/src/rpc/methods/CoreRender.ts

@@ -1,6 +1,9 @@
 import { validateRendererType } from './util.ts'
-import RpcMethodType from '../../pluggableElementTypes/RpcMethodType.ts'
+import RpcMethodType, {
+  convertFileHandleLocations,
+} from '../../pluggableElementTypes/RpcMethodType.ts'
 import { renameRegionsIfNeeded } from '../../util/index.ts'
+import { getBlobMap, setBlobMap } from '../../util/tracks.ts'
 
 import type {
   RenderArgs,
@@ -41,10 +44,18 @@ export default class CoreRender extends RpcMethodType {
       throw new Error('must pass a unique session id')
     }
 
+    // Convert FileHandleLocation to BlobLocation for consistent adapter caching.
+    // Even though we're on the main thread and don't need to transfer files,
+    // the adapter config hash must match what the serialized path produces.
+    const { renderingProps, ...rest } = renamedArgs
+    const blobMap = getBlobMap()
+    convertFileHandleLocations(rest, blobMap)
+    setBlobMap(blobMap)
+
     return validateRendererType(
       rendererType,
       this.pluginManager.getRendererType(rendererType),
-    ).renderDirect(renamedArgs)
+    ).renderDirect({ ...rest, renderingProps } as RenderArgs)
   }
 
   async execute(