diff --git a/.changeset/breezy-llamas-juggle.md b/.changeset/breezy-llamas-juggle.md new file mode 100644 index 00000000..90c07b58 --- /dev/null +++ b/.changeset/breezy-llamas-juggle.md @@ -0,0 +1,5 @@ +--- +"@tus/utils": minor +--- + +Add IoRedisKvStore & use redis.scan instead of discouraged redis.keys diff --git a/.changeset/smooth-gifts-beam.md b/.changeset/smooth-gifts-beam.md new file mode 100644 index 00000000..e63e9d6d --- /dev/null +++ b/.changeset/smooth-gifts-beam.md @@ -0,0 +1,5 @@ +--- +"@tus/server": minor +--- + +Add ioredis as optional dependency diff --git a/package-lock.json b/package-lock.json index 65f3bb40..fab883f1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@tus/file-store": "^1.5.0", "@tus/gcs-store": "^1.4.0", "@tus/s3-store": "^1.6.0", - "@tus/server": "^1.8.0", + "@tus/server": "^1.9.0", "tus-js-client": "^2.3.2" }, "devDependencies": { @@ -1724,6 +1724,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@ioredis/commands": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@ioredis/commands/-/commands-1.2.0.tgz", + "integrity": "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==", + "devOptional": true + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.1", "dev": true, @@ -3352,8 +3358,8 @@ }, "node_modules/cluster-key-slot": { "version": "1.1.2", + "devOptional": true, "license": "Apache-2.0", - "optional": true, "engines": { "node": ">=0.10.0" } @@ -3680,6 +3686,15 @@ "resolved": "demo", "link": true }, + "node_modules/denque": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/denque/-/denque-2.1.0.tgz", + "integrity": "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==", + "devOptional": true, + "engines": { + "node": ">=0.10" + } + }, "node_modules/detect-indent": { "version": "6.1.0", "dev": true, @@ -4628,6 +4643,30 @@ "node": ">= 0.4" } }, + "node_modules/ioredis": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-5.4.1.tgz", + "integrity": "sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==", + "devOptional": true, + "dependencies": { + "@ioredis/commands": "^1.1.1", + "cluster-key-slot": "^1.1.0", + "debug": "^4.3.4", + "denque": "^2.1.0", + "lodash.defaults": "^4.2.0", + "lodash.isarguments": "^3.1.0", + "redis-errors": "^1.2.0", + "redis-parser": "^3.0.0", + "standard-as-callback": "^2.1.0" + }, + "engines": { + "node": ">=12.22.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ioredis" + } + }, "node_modules/is-array-buffer": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.4.tgz", @@ -5102,11 +5141,23 @@ "lodash._basetostring": "~4.12.0" } }, + "node_modules/lodash.defaults": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.defaults/-/lodash.defaults-4.2.0.tgz", + "integrity": "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==", + "devOptional": true + }, "node_modules/lodash.get": { "version": "4.4.2", "dev": true, "license": "MIT" }, + "node_modules/lodash.isarguments": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", + "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", + "devOptional": true + }, "node_modules/lodash.startcase": { "version": "4.4.0", "dev": true, @@ -6160,6 +6211,27 @@ "node": ">=8" } }, + "node_modules/redis-errors": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/redis-errors/-/redis-errors-1.2.0.tgz", + "integrity": "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==", + "devOptional": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/redis-parser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redis-parser/-/redis-parser-3.0.0.tgz", + "integrity": "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==", + "devOptional": true, + "dependencies": { + "redis-errors": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/regenerator-runtime": { "version": "0.14.1", "dev": true, @@ -6636,6 +6708,12 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/standard-as-callback": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/standard-as-callback/-/standard-as-callback-2.1.0.tgz", + "integrity": "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==", + "devOptional": true + }, "node_modules/stream-events": { "version": "1.0.5", "dev": true, @@ -7635,7 +7713,7 @@ }, "packages/server": { "name": "@tus/server", - "version": "1.8.0", + "version": "1.9.0", "license": "MIT", "dependencies": { "@tus/utils": "^0.4.0", @@ -7660,7 +7738,8 @@ "node": ">=16" }, "optionalDependencies": { - "@redis/client": "^1.5.13" + "@redis/client": "^1.5.13", + "ioredis": "^5.4.1" } }, "packages/utils": { @@ -7671,6 +7750,7 @@ "@types/debug": "^4.1.12", "@types/mocha": "^10.0.6", "@types/node": "^20.11.5", + "ioredis": "^5.4.1", "mocha": "^10.4.0", "should": "^13.2.3", "ts-node": "^10.9.2" @@ -7684,7 +7764,7 @@ "@tus/file-store": "^1.5.0", "@tus/gcs-store": "^1.4.0", "@tus/s3-store": "^1.6.0", - "@tus/server": "^1.8.0" + "@tus/server": "^1.9.0" }, "devDependencies": { "@types/mocha": "^10.0.6", diff --git a/packages/server/README.md b/packages/server/README.md index 9335d75e..a49d202a 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -342,7 +342,6 @@ import S3Store, {type MetadataValue} from '@tus/s3-store' import {createClient} from '@redis/client' const client = await createClient().connect() -const path = './uploads' const prefix = 'foo' // prefix for the key (foo${id}) new S3Store({ @@ -351,6 +350,22 @@ new S3Store({ }) ``` +#### `IoRedisKvStore` + +```ts +import { IoRedisKvStore } from '@tus/server'; +import S3Store, { type MetadataValue } from '@tus/s3-store'; +import Redis from 'ioredis'; + +const client = new Redis(); +const prefix = 'foo'; // prefix for the key (foo${id}) + +new S3Store({ + // ... + cache: new IoRedisKvStore(client, prefix), +}); +``` + ## Examples ### Example: integrate tus into Express diff --git a/packages/server/package.json b/packages/server/package.json index 9f6d81fb..16c86942 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -33,7 +33,8 @@ "ts-node": "^10.9.2" }, "optionalDependencies": { - "@redis/client": "^1.5.13" + "@redis/client": "^1.5.13", + "ioredis": "^5.4.1" }, "engines": { "node": ">=16" diff --git a/packages/utils/package.json b/packages/utils/package.json index 19b6f7f3..0400dd27 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -17,6 +17,7 @@ "@types/debug": "^4.1.12", "@types/mocha": "^10.0.6", "@types/node": "^20.11.5", + "ioredis": "^5.4.1", "mocha": "^10.4.0", "should": "^13.2.3", "ts-node": "^10.9.2" diff --git a/packages/utils/src/kvstores/IoRedisKvStore.ts b/packages/utils/src/kvstores/IoRedisKvStore.ts new file mode 100644 index 00000000..8d4a1891 --- /dev/null +++ b/packages/utils/src/kvstores/IoRedisKvStore.ts @@ -0,0 +1,54 @@ +import type {Redis as IoRedis} from 'ioredis' +import type {KvStore} from './Types' +import type {Upload} from '../models' + +export class IoRedisKvStore implements KvStore { + constructor( + private redis: IoRedis, + private prefix = '' + ) { + this.redis = redis + this.prefix = prefix + } + + private prefixed(key: string): string { + return `${this.prefix}${key}` + } + + async get(key: string): Promise { + return this.deserializeValue(await this.redis.get(this.prefixed(key))) + } + + async set(key: string, value: T): Promise { + await this.redis.set(this.prefixed(key), this.serializeValue(value)) + } + + async delete(key: string): Promise { + await this.redis.del(this.prefixed(key)) + } + + async list(): Promise> { + const keys = new Set() + let cursor = '0' + do { + const [next, batch] = await this.redis.scan( + cursor, + 'MATCH', + this.prefixed('*'), + 'COUNT', + '20' + ) + cursor = next + for (const key of batch) keys.add(key) + } while (cursor !== '0') + return Array.from(keys) + } + + private serializeValue(value: T): string { + return JSON.stringify(value) + } + + private deserializeValue(buffer: string | null): T | undefined { + return buffer ? JSON.parse(buffer) : undefined + } +} diff --git a/packages/utils/src/kvstores/RedisKvStore.ts b/packages/utils/src/kvstores/RedisKvStore.ts index 562fe068..c1550ecd 100644 --- a/packages/utils/src/kvstores/RedisKvStore.ts +++ b/packages/utils/src/kvstores/RedisKvStore.ts @@ -29,7 +29,14 @@ export class RedisKvStore implements KvStore { } async list(): Promise> { - return this.redis.keys(`${this.prefix}*`) + const keys = new Set() + let cursor = 0 + do { + const result = await this.redis.scan(cursor, {MATCH: `${this.prefix}*`, COUNT: 20}) + cursor = result.cursor + for (const key of result.keys) keys.add(key) + } while (cursor !== 0) + return Array.from(keys) } private serializeValue(value: T): string { diff --git a/packages/utils/src/kvstores/index.ts b/packages/utils/src/kvstores/index.ts index 0ec840c1..5d2aff72 100644 --- a/packages/utils/src/kvstores/index.ts +++ b/packages/utils/src/kvstores/index.ts @@ -1,4 +1,5 @@ export {FileKvStore} from './FileKvStore' export {MemoryKvStore} from './MemoryKvStore' export {RedisKvStore} from './RedisKvStore' +export {IoRedisKvStore} from './IoRedisKvStore' export {KvStore} from './Types'