@tus/file-store: support async config stores & add Redis store (#407)

Co-authored-by: Murderlon <[email protected]>

Author: mitjap

packages/file-store/configstores/FileConfigstore.ts

@@ -0,0 +1,52 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import {Upload} from '@tus/server'
+import PQueue from 'p-queue'
+
+import {Configstore} from './Types'
+
+/**
+ * FileConfigstore writes the `Upload` JSON metadata to disk next the uploaded file itself.
+ * It uses a queue which only processes one operation at a time to prevent unsafe concurrent access.
+ */
+export class FileConfigstore implements Configstore {
+  directory: string
+  queue: PQueue
+
+  constructor(path: string) {
+    this.directory = path
+    this.queue = new PQueue({concurrency: 1})
+  }
+
+  async get(key: string): Promise<Upload | undefined> {
+    try {
+      const buffer = await this.queue.add(() => fs.readFile(this.resolve(key), 'utf8'))
+      return JSON.parse(buffer as string)
+    } catch {
+      return undefined
+    }
+  }
+
+  async set(key: string, value: Upload): Promise<void> {
+    await this.queue.add(() => fs.writeFile(this.resolve(key), JSON.stringify(value)))
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.queue.add(() => fs.rm(this.resolve(key)))
+  }
+
+  async list(): Promise<Array<string>> {
+    return this.queue.add(async () => {
+      const files = await fs.readdir(this.directory, {withFileTypes: true})
+      const promises = files
+        .filter((file) => file.isFile() && file.name.endsWith('.json'))
+        .map((file) => fs.readFile(path.resolve(file.path, file.name), 'utf8'))
+
+      return Promise.all(promises)
+    }) as Promise<string[]>
+  }
+
+  private resolve(key: string): string {
+    return path.resolve(this.directory, `${key}.json`)
+  }
+}

packages/file-store/configstores/MemoryConfigstore.ts

@@ -0,0 +1,36 @@
+import {Upload} from '@tus/server'
+import {Configstore} from './Types'
+
+/**
+ * Memory based configstore.
+ * Used mostly for unit tests.
+ *
+ * @author Mitja Puzigaća <[email protected]>
+ */
+export class MemoryConfigstore implements Configstore {
+  data: Map<string, string> = new Map()
+
+  async get(key: string): Promise<Upload | undefined> {
+    return this.deserializeValue(this.data.get(key))
+  }
+
+  async set(key: string, value: Upload): Promise<void> {
+    this.data.set(key, this.serializeValue(value))
+  }
+
+  async delete(key: string): Promise<void> {
+    this.data.delete(key)
+  }
+
+  async list(): Promise<Array<string>> {
+    return [...this.data.keys()]
+  }
+
+  private serializeValue(value: Upload): string {
+    return JSON.stringify(value)
+  }
+
+  private deserializeValue(buffer: string | undefined): Upload | undefined {
+    return buffer ? new Upload(JSON.parse(buffer)) : undefined
+  }
+}

packages/file-store/configstores/RedisConfigstore.ts

@@ -0,0 +1,40 @@
+import {RedisClientType} from '@redis/client'
+
+import {Upload} from '@tus/server'
+import {Configstore} from './Types'
+
+/**
+ * Redis based configstore.
+ *
+ * @author Mitja Puzigaća <[email protected]>
+ */
+export class RedisConfigstore implements Configstore {
+  constructor(private redis: RedisClientType, private prefix: string = '') {
+    this.redis = redis
+    this.prefix = prefix
+  }
+
+  async get(key: string): Promise<Upload | undefined> {
+    return this.deserializeValue(await this.redis.get(this.prefix + key))
+  }
+
+  async set(key: string, value: Upload): Promise<void> {
+    await this.redis.set(this.prefix + key, this.serializeValue(value))
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.redis.del(this.prefix + key)
+  }
+
+  async list(): Promise<Array<string>> {
+    return this.redis.keys(this.prefix + '*')
+  }
+
+  private serializeValue(value: Upload): string {
+    return JSON.stringify(value)
+  }
+
+  private deserializeValue(buffer: string | null): Upload | undefined {
+    return buffer ? JSON.parse(buffer) : undefined
+  }
+}

packages/file-store/configstores/Types.ts

@@ -0,0 +1,9 @@
+import {Upload} from '@tus/server'
+
+export interface Configstore {
+  get(key: string): Promise<Upload | undefined>
+  set(key: string, value: Upload): Promise<void>
+  delete(key: string): Promise<void>
+
+  list?(): Promise<Array<string>>
+}

packages/file-store/configstores/index.ts

@@ -0,0 +1,4 @@
+export {FileConfigstore} from './FileConfigstore'
+export {MemoryConfigstore} from './MemoryConfigstore'
+export {RedisConfigstore} from './RedisConfigstore'
+export {Configstore} from './Types'

packages/file-store/index.ts

@@ -5,21 +5,15 @@ import stream from 'node:stream'
 import http from 'node:http'
 
 import debug from 'debug'
-import Configstore from 'configstore'
 
+import {Configstore, FileConfigstore} from './configstores'
 import {DataStore, Upload, ERRORS} from '@tus/server'
-import pkg from './package.json'
 
-type Store = {
-  get(key: string): Upload | undefined
-  set(key: string, value: Upload): void
-  delete(key: string): void
-  all: Record<string, Upload>
-}
+export * from './configstores'
 
 type Options = {
   directory: string
-  configstore?: Store
+  configstore?: Configstore
   expirationPeriodInMilliseconds?: number
 }
 
@@ -30,13 +24,13 @@ const log = debug('tus-node-server:stores:filestore')
 
 export class FileStore extends DataStore {
   directory: string
-  configstore: Store
+  configstore: Configstore
   expirationPeriodInMilliseconds: number
 
   constructor({directory, configstore, expirationPeriodInMilliseconds}: Options) {
     super()
     this.directory = directory
-    this.configstore = configstore ?? new Configstore(`${pkg.name}-${pkg.version}`)
+    this.configstore = configstore ?? new FileConfigstore(directory)
     this.expirationPeriodInMilliseconds = expirationPeriodInMilliseconds ?? 0
     this.extensions = [
       'creation',
@@ -65,13 +59,13 @@ export class FileStore extends DataStore {
    */
   create(file: Upload): Promise<Upload> {
     return new Promise((resolve, reject) => {
-      fs.open(path.join(this.directory, file.id), 'w', (err, fd) => {
+      fs.open(path.join(this.directory, file.id), 'w', async (err, fd) => {
         if (err) {
           log('[FileStore] create: Error', err)
           return reject(err)
         }
 
-        this.configstore.set(file.id, file)
+        await this.configstore.set(file.id, file)
 
         return fs.close(fd, (exception) => {
           if (exception) {
@@ -143,7 +137,7 @@ export class FileStore extends DataStore {
   }
 
   async getUpload(id: string): Promise<Upload> {
-    const file = this.configstore.get(id)
+    const file = await this.configstore.get(id)
 
     if (!file) {
       throw ERRORS.FILE_NOT_FOUND
@@ -188,25 +182,29 @@ export class FileStore extends DataStore {
   }
 
   async declareUploadLength(id: string, upload_length: number) {
-    const file = this.configstore.get(id)
+    const file = await this.configstore.get(id)
 
     if (!file) {
       throw ERRORS.FILE_NOT_FOUND
     }
 
     file.size = upload_length
 
-    this.configstore.set(id, file)
+    await this.configstore.set(id, file)
   }
 
   async deleteExpired(): Promise<number> {
     const now = new Date()
     const toDelete: Promise<void>[] = []
 
-    const uploadInfos = this.configstore.all
-    for (const file_id of Object.keys(uploadInfos)) {
+    if (!this.configstore.list) {
+      throw ERRORS.UNSUPPORTED_EXPIRATION_EXTENSION
+    }
+
+    const uploadKeys = await this.configstore.list()
+    for (const file_id of uploadKeys) {
       try {
-        const info = uploadInfos[file_id]
+        const info = await this.configstore.get(file_id)
         if (
           info &&
           'creation_date' in info &&
@@ -227,7 +225,8 @@ export class FileStore extends DataStore {
       }
     }
 
-    return Promise.all(toDelete).then(() => toDelete.length)
+    await Promise.all(toDelete)
+    return toDelete.length
   }
 
   getExpiration(): number {

packages/file-store/package.json

@@ -21,12 +21,11 @@
     "test": "mocha test.ts --exit --extension ts --require ts-node/register"
   },
   "dependencies": {
-    "configstore": "^5.0.1",
-    "debug": "^4.3.3"
+    "debug": "^4.3.3",
+    "p-queue": "^6.6.2"
   },
   "devDependencies": {
     "@tus/server": "workspace:^",
-    "@types/configstore": "^6.0.0",
     "@types/debug": "^4.1.7",
     "@types/mocha": "^10.0.1",
     "@types/node": "latest",
@@ -39,6 +38,9 @@
   "peerDependencies": {
     "@tus/server": "workspace:^"
   },
+  "optionalDependencies": {
+    "@redis/client": "^1.5.5"
+  },
   "engines": {
     "node": ">=16"
   }

packages/file-store/test.ts

@@ -7,9 +7,10 @@ import path from 'node:path'
 import sinon from 'sinon'
 
 import {FileStore} from './'
-import {Upload, MemoryConfigstore} from '@tus/server'
+import {Upload} from '@tus/server'
 
 import * as shared from '../../test/stores.test'
+import {MemoryConfigstore} from './configstores/MemoryConfigstore'
 
 const fixturesPath = path.resolve('../', '../', 'test', 'fixtures')
 const storePath = path.resolve('../', '../', 'test', 'output')

packages/server/src/index.ts

@@ -1,4 +1,4 @@
 export {Server} from './server'
 export * from './types'
-export * from './models/'
+export * from './models'
 export * from './constants'