refactor(server): Lazy-initialize Redis, Redlock, and Elasticsearch clients, fix import cycles

package.json

@@ -3,6 +3,7 @@
   "scripts": {
     "start": "docker-compose up --build -V",
     "build": "yarn pnpify tsc -b",
+    "dump-schema": "yarn tsx scripts/dump-schema.ts",
     "bootstrap": "lerna bootstrap",
     "test": "vitest",
     "coverage": "vitest --coverage",
@@ -43,6 +44,7 @@
     "react": "18.2.0",
     "sass": "1.93.2",
     "tsc-watch": "6.0.4",
+    "tsx": "^3.7.0",
     "typescript": "5.6.3",
     "typescript-eslint": "^8.11.0",
     "vite": "7.3.2",

packages/openneuro-server/src/app.ts

@@ -22,7 +22,7 @@ import * as jwt from "./libs/authentication/jwt"
 import * as auth from "./libs/authentication/states"
 import { sitemapHandler } from "./handlers/sitemap"
 import { setupPassportAuth } from "./libs/authentication/passport"
-import { redis } from "./libs/redis"
+import { getRedis } from "./libs/redis"
 import { version } from "./lerna.json"
 export { Express } from "express-serve-static-core"
 
@@ -67,7 +67,7 @@ export async function expressApolloSetup() {
     // Always allow introspection - our schema is public
     introspection: true,
     // @ts-expect-error Type mismatch for keyv and ioredis recent releases
-    cache: new KeyvAdapter(new Keyv({ store: new KeyvRedis(redis) })),
+    cache: new KeyvAdapter(new Keyv({ store: new KeyvRedis(getRedis()) })),
     plugins: [
       ApolloServerPluginLandingPageLocalDefault(),
       ApolloServerPluginDrainHttpServer({

packages/openneuro-server/src/datalad/__tests__/contributors.spec.ts

@@ -27,7 +27,7 @@ vi.mock("../../cache/item")
 vi.mock("../files")
 vi.mock("../../utils/datasetOrSnapshot")
 vi.mock("../libs/redis", () => ({
-  redis: vi.fn(),
+  getRedis: vi.fn(),
 }))
 
 const mockYamlLoad = vi.mocked(yaml.load)

packages/openneuro-server/src/datalad/__tests__/files.spec.ts

@@ -13,7 +13,7 @@ import type { TreeEntry } from "../../cache/tree"
 
 vi.mock("ioredis")
 vi.mock("../../config.ts")
-vi.mock("../../libs/redis", () => ({ redis: {} }))
+vi.mock("../../libs/redis", () => ({ getRedis: () => ({}) }))
 vi.mock("../../libs/presign", () => ({
   getPresignedUrl: vi.fn().mockResolvedValue(
     "https://s3.amazonaws.com/bucket/key?presigned=true",

packages/openneuro-server/src/datalad/__tests__/snapshots.spec.ts

@@ -10,12 +10,12 @@ import { connect } from "mongoose"
 // Mock requests to Datalad service
 vi.mock("superagent")
 vi.mock("../../libs/redis.js", () => ({
-  redis: {
+  getRedis: () => ({
     del: vi.fn(),
-  },
-  redlock: {
+  }),
+  getRedlock: () => ({
     lock: vi.fn().mockImplementation(() => ({ unlock: vi.fn() })),
-  },
+  }),
 }))
 // Mock draft files calls
 vi.mock("../draft.ts", () => ({

packages/openneuro-server/src/datalad/contributors.ts

@@ -1,6 +1,6 @@
 import * as Sentry from "@sentry/node"
 import CacheItem, { CacheType } from "../cache/item"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import {
   type DatasetOrSnapshot,
   datasetOrSnapshot,
@@ -26,7 +26,7 @@ export const contributors = async (
   if (!datasetId) return []
 
   const revisionShort = revision ? revision.substring(0, 7) : "HEAD"
-  const dataciteCache = new CacheItem(redis, CacheType.dataciteYml, [
+  const dataciteCache = new CacheItem(getRedis(), CacheType.dataciteYml, [
     datasetId,
     revisionShort,
   ])

packages/openneuro-server/src/datalad/dataset.ts

@@ -12,7 +12,7 @@ import type * as Mongoose from "mongoose"
 import config from "../config"
 import * as subscriptions from "../handlers/subscriptions"
 import { generateDataladCookie } from "../libs/authentication/jwt"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import { getDraftRevision, updateDatasetRevision } from "./draft"
 import { encodeFilePath, filesUrl, fileUrl, getFileName } from "./files"
@@ -114,7 +114,7 @@ export const deleteDataset = async (datasetId, user) => {
 export const cacheDatasetConnection = (options) => (connectionArguments) => {
   const connection = datasetsConnection(options)
   const cache = new CacheItem(
-    redis,
+    getRedis(),
     CacheType.datasetsConnection,
     [objectHash(options)],
     60,

packages/openneuro-server/src/datalad/description.ts

@@ -3,7 +3,7 @@
  */
 import config from "../config"
 import request from "superagent"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import { commitFiles } from "./dataset"
 import { fileUrl } from "./files"
 import { generateDataladCookie } from "../libs/authentication/jwt"
@@ -135,7 +135,7 @@ export const description = async (obj) => {
     Name: datasetId,
     BIDSVersion: "1.8.0",
   }
-  const cache = new CacheItem(redis, CacheType.datasetDescription, [
+  const cache = new CacheItem(getRedis(), CacheType.datasetDescription, [
     datasetId,
     revision.substring(0, 7),
   ])

packages/openneuro-server/src/datalad/draft.ts

@@ -5,7 +5,7 @@ import request from "superagent"
 import Dataset from "../models/dataset"
 import { getDatasetWorker } from "../libs/datalad-service"
 import CacheItem, { CacheType } from "../cache/item"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 
 // Draft info resolver
 type DraftInfo = {
@@ -22,7 +22,12 @@ export const getDraftRevision = async (datasetId): Promise<string> => {
 }
 
 export const getDraftInfo = async (datasetId) => {
-  const cache = new CacheItem(redis, CacheType.draftRevision, [datasetId], 10)
+  const cache = new CacheItem(
+    getRedis(),
+    CacheType.draftRevision,
+    [datasetId],
+    10,
+  )
   return cache.get(async (_doNotCache): Promise<DraftInfo> => {
     const draftUrl = `http://${
       getDatasetWorker(

packages/openneuro-server/src/datalad/files.ts

@@ -1,4 +1,4 @@
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import { getDatasetWorker } from "../libs/datalad-service"
 import {
   getPresignedUrl,
@@ -184,7 +184,7 @@ export async function entryToDatasetFile(
   }
   let url: string
   if (entry.p && entry.k && entry.v) {
-    url = await getPresignedUrl(redis, entry.b, entry.k, entry.v)
+    url = await getPresignedUrl(getRedis(), entry.b, entry.k, entry.v)
   } else if (entry.k && entry.v) {
     url = publicS3Url(entry.b, entry.k, entry.v)
   } else {
@@ -260,15 +260,15 @@ async function cacheWorkerTrees(
         (f) => f.directory || f.urls[0]?.includes("s3.amazonaws.com"),
       )
       if (allExported) {
-        void setTree(redis, hash, entries)
+        void setTree(getRedis(), hash, entries)
         permanentHashes.push(hash)
       } else {
-        void setTree(redis, hash, entries, 600)
+        void setTree(getRedis(), hash, entries, 600)
       }
     }
   }
   if (permanentHashes.length > 0) {
-    void addDatasetTrees(redis, datasetId, permanentHashes)
+    void addDatasetTrees(getRedis(), datasetId, permanentHashes)
   }
   return result
 }
@@ -283,7 +283,10 @@ export const resolveGitRef = async (
   datasetId: string,
   treeish: string,
 ): Promise<string> => {
-  const cache = new CacheItem(redis, CacheType.gitRef, [datasetId, treeish])
+  const cache = new CacheItem(getRedis(), CacheType.gitRef, [
+    datasetId,
+    treeish,
+  ])
   return cache.get(async () => {
     const url = `http://${
       getDatasetWorker(datasetId)
@@ -324,7 +327,7 @@ export const getFiles = async (
     }
   }
   // Try cache first
-  const cached = await getTree(redis, treeish)
+  const cached = await getTree(getRedis(), treeish)
   if (cached) {
     return entriesToDatasetFiles(cached, datasetId)
   }
@@ -354,10 +357,10 @@ export async function getFilesRecursive(
 ): Promise<DatasetFile[]> {
   const needsPresign = await datasetNeedsPresign(datasetId)
   // Check for cached commit-to-trees mapping
-  const cachedTreeHashes = await getCommitTrees(redis, tree)
+  const cachedTreeHashes = await getCommitTrees(getRedis(), tree)
   if (cachedTreeHashes) {
     // Bulk-fetch all trees in one pipeline
-    const treesMap = await getTreesBulk(redis, cachedTreeHashes)
+    const treesMap = await getTreesBulk(getRedis(), cachedTreeHashes)
     if (treesMap.size < cachedTreeHashes.length) {
       // Batch-fetch all missing trees from the worker in one request
       const missingHashes = cachedTreeHashes.filter((h) => !treesMap.has(h))
@@ -381,7 +384,7 @@ export async function getFilesRecursive(
 
   while (pendingHashes.length > 0) {
     // Check cache for all pending hashes
-    const cached = await getTreesBulk(redis, pendingHashes)
+    const cached = await getTreesBulk(getRedis(), pendingHashes)
     const uncached = pendingHashes.filter((h) => !cached.has(h))
 
     // Fetch all uncached trees in one worker request
@@ -417,8 +420,8 @@ export async function getFilesRecursive(
   // Cache the commit-to-trees mapping for next time
   if (collectedHashes.size > 0) {
     const hashArray = [...collectedHashes]
-    void setCommitTrees(redis, tree, hashArray)
-    void addDatasetTrees(redis, datasetId, hashArray)
+    void setCommitTrees(getRedis(), tree, hashArray)
+    void addDatasetTrees(getRedis(), datasetId, hashArray)
   }
 
   return reconstructFromTrees(treesMap, tree, path, datasetId)
@@ -485,7 +488,7 @@ async function reconstructFromTrees(
   // Bulk-resolve presigned URLs in minimal Redis requests
   if (presignIndices.length > 0) {
     const urls = await getPresignedUrlsBulk(
-      redis,
+      getRedis(),
       presignIndices.map((i) => ({
         bucket: fileEntries[i].entry.b,
         s3Key: fileEntries[i].entry.k,

packages/openneuro-server/src/datalad/readme.ts

@@ -1,5 +1,5 @@
 import { addFileString, commitFiles } from "./dataset"
-import { redis } from "../libs/redis"
+import { getRedis } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import { getDatasetWorker } from "../libs/datalad-service"
 import { datasetOrSnapshot } from "../utils/datasetOrSnapshot"
@@ -14,7 +14,7 @@ export const readmeUrl = (datasetId, revision) => {
 
 export const readme = (obj) => {
   const { datasetId, revision } = datasetOrSnapshot(obj)
-  const cache = new CacheItem(redis, CacheType.readme, [
+  const cache = new CacheItem(getRedis(), CacheType.readme, [
     datasetId,
     revision.substring(0, 7),
   ])

packages/openneuro-server/src/datalad/snapshots.ts

@@ -3,13 +3,10 @@
  */
 import * as Sentry from "@sentry/node"
 import request from "superagent"
-import { redis, redlock } from "../libs/redis"
+import { getRedis, getRedlock } from "../libs/redis"
 import CacheItem, { CacheType } from "../cache/item"
 import config from "../config"
-import {
-  snapshotCreationComparison,
-  updateDatasetName,
-} from "../graphql/resolvers/dataset"
+import { snapshotCreationComparison } from "../utils/snapshots"
 import { createDraftDoi } from "../libs/doi/index"
 import { assembleMetadata } from "../libs/doi/metadata"
 import Doi from "../models/doi"
@@ -25,7 +22,7 @@ import { createEvent, updateEvent } from "../libs/events"
 import { queueIndexDataset } from "../queues/producer-methods"
 
 const lockSnapshot = (datasetId, tag) => {
-  return redlock.lock(
+  return getRedlock().lock(
     `openneuro:create-snapshot-lock:${datasetId}:${tag}`,
     1800000,
   )
@@ -110,7 +107,12 @@ const postSnapshot = async (
 export const getSnapshots = async (datasetId): Promise<SnapshotDocument[]> => {
   const dataset = await Dataset.findOne({ id: datasetId })
   if (!dataset) return null
-  const cache = new CacheItem(redis, CacheType.snapshot, [datasetId], 432000)
+  const cache = new CacheItem(
+    getRedis(),
+    CacheType.snapshot,
+    [datasetId],
+    432000,
+  )
   return cache.get(() => {
     const url = `${getDatasetWorker(datasetId)}/datasets/${datasetId}/snapshots`
     return request
@@ -144,7 +146,7 @@ export const createSnapshot = async (
   descriptionFieldUpdates = {},
   snapshotChanges = [],
 ) => {
-  const snapshotCache = new CacheItem(redis, CacheType.snapshot, [
+  const snapshotCache = new CacheItem(getRedis(), CacheType.snapshot, [
     datasetId,
     tag,
   ])
@@ -182,10 +184,13 @@ export const createSnapshot = async (
       createSnapshotMetadata(datasetId, tag, snapshot.hexsha, snapshot.created),
 
       // Trigger an async update for the name field (cache for sorting)
-      updateDatasetName(datasetId),
+      // Dynamic import breaks circular dependency: datalad/snapshots → resolvers/dataset
+      import("../graphql/resolvers/dataset").then((m) =>
+        m.updateDatasetName(datasetId)
+      ),
     ])
 
-    const snapshotListCache = new CacheItem(redis, CacheType.snapshot, [
+    const snapshotListCache = new CacheItem(getRedis(), CacheType.snapshot, [
       datasetId,
     ])
     await snapshotListCache.drop()
@@ -215,12 +220,12 @@ export const deleteSnapshot = (datasetId, tag) => {
     )
   }/datasets/${datasetId}/snapshots/${tag}`
   return request.del(url).then(async ({ body }) => {
-    const snapshotCache = new CacheItem(redis, CacheType.snapshot, [
+    const snapshotCache = new CacheItem(getRedis(), CacheType.snapshot, [
       datasetId,
       tag,
     ])
     await snapshotCache.drop()
-    const snapshotListCache = new CacheItem(redis, CacheType.snapshot, [
+    const snapshotListCache = new CacheItem(getRedis(), CacheType.snapshot, [
       datasetId,
     ])
     await snapshotListCache.drop()
@@ -244,7 +249,7 @@ export const getSnapshot = (
     )
   }/datasets/${datasetId}/snapshots/${commitRef}`
   const cache = new CacheItem(
-    redis,
+    getRedis(),
     CacheType.snapshot,
     [datasetId, commitRef],
     432000,