add ioredis store, use scan instead of keys

Author: vktrl

package-lock.json

@@ -17,6 +17,7 @@
     "@types/debug": "^4.1.12",
     "@types/mocha": "^10.0.6",
     "@types/node": "^20.11.5",
+    "ioredis": "^5.4.1",
     "mocha": "^10.4.0",
     "should": "^13.2.3",
     "ts-node": "^10.9.2"

packages/utils/package.json

@@ -0,0 +1,51 @@
+
+import type {Redis as IoRedis} from "ioredis"
+import type {KvStore} from './Types'
+import type {Upload} from '../models'
+
+export class IoRedisKvStore<T = Upload> implements KvStore<T> {
+  constructor(
+    private redis: IoRedis,
+    private prefix = ''
+  ) {
+    this.redis = redis
+    this.prefix = prefix
+  }
+
+  private prefixed(key: string): string {
+    return `${this.prefix}${key}`
+  }
+
+  async get(key: string): Promise<T | undefined> {
+    return this.deserializeValue(await this.redis.get(this.prefixed(key)))
+  }
+
+  async set(key: string, value: T): Promise<void> {
+    await this.redis.set(this.prefixed(key), this.serializeValue(value))
+  }
+
+  async delete(key: string): Promise<void> {
+    await this.redis.del(this.prefixed(key))
+  }
+
+  async list(): Promise<Array<string>> {
+    const keys = new Set<string>()
+    let cursor = "0"
+    do {
+      const [next, batch] = await this.redis.scan(cursor, "MATCH", this.prefixed("*"), "COUNT", "20")
+      cursor = next
+      for (const key of batch) keys.add(key)
+    } while (cursor !== "0")
+    return Array.from(keys)
+  }
+
+  private serializeValue(value: T): string {
+    return JSON.stringify(value)
+  }
+
+  private deserializeValue(buffer: string | null): T | undefined {
+    return buffer ? JSON.parse(buffer) : undefined
+  }
+}
+
+

packages/utils/src/kvstores/IoRedisKvStore.ts

@@ -29,7 +29,14 @@ export class RedisKvStore<T = Upload> implements KvStore<T> {
   }
 
   async list(): Promise<Array<string>> {
-    return this.redis.keys(`${this.prefix}*`)
+    const keys = new Set<string>()
+    let cursor = 0
+    do {
+      const result = await this.redis.scan(cursor, { MATCH: `${this.prefix}*`, COUNT: 20 })
+      cursor = result.cursor
+      for (const key of result.keys) keys.add(key)
+    } while (cursor !== 0)
+    return Array.from(keys)
   }
 
   private serializeValue(value: T): string {

packages/utils/src/kvstores/RedisKvStore.ts

@@ -1,4 +1,5 @@
 export {FileKvStore} from './FileKvStore'
 export {MemoryKvStore} from './MemoryKvStore'
 export {RedisKvStore} from './RedisKvStore'
+export {IoRedisKvStore} from './IoRedisKvStore'
 export {KvStore} from './Types'