feat!: DeleteAll match pattern and v5 adjustments (#174)

* feat: delete all match pattern

* test coverage

* make sure all valid chars are supported

* safeguard require deleteAll (breaking change)

* fix!: hide 404s on container operations (#175)

Author: moritzraho

doc/api.md

@@ -60,10 +60,10 @@ Cloud State Management
     * *[.getRegionalEndpoint(endpoint, region)](#AdobeState+getRegionalEndpoint) ⇒ <code>string</code>*
     * *[.get(key)](#AdobeState+get) ⇒ [<code>Promise.&lt;AdobeStateGetReturnValue&gt;</code>](#AdobeStateGetReturnValue)*
     * *[.put(key, value, [options])](#AdobeState+put) ⇒ <code>Promise.&lt;string&gt;</code>*
-    * *[.delete(key)](#AdobeState+delete) ⇒ <code>Promise.&lt;string&gt;</code>*
-    * *[.deleteAll()](#AdobeState+deleteAll) ⇒ <code>Promise.&lt;boolean&gt;</code>*
+    * *[.delete(key)](#AdobeState+delete) ⇒ <code>Promise.&lt;(string\|null)&gt;</code>*
+    * *[.deleteAll(options)](#AdobeState+deleteAll) ⇒ <code>Promise.&lt;{keys: number}&gt;</code>*
     * *[.any()](#AdobeState+any) ⇒ <code>Promise.&lt;boolean&gt;</code>*
-    * *[.stats()](#AdobeState+stats) ⇒ <code>Promise.&lt;({bytesKeys: number, bytesValues: number, keys: number}\|boolean)&gt;</code>*
+    * *[.stats()](#AdobeState+stats) ⇒ <code>Promise.&lt;{bytesKeys: number, bytesValues: number, keys: number}&gt;</code>*
     * *[.list(options)](#AdobeState+list) ⇒ <code>AsyncGenerator.&lt;{keys: Array.&lt;string&gt;}&gt;</code>*
 
 <a name="AdobeState+getRegionalEndpoint"></a>
@@ -108,37 +108,48 @@ Creates or updates a state key-value pair
 
 <a name="AdobeState+delete"></a>
 
-### *adobeState.delete(key) ⇒ <code>Promise.&lt;string&gt;</code>*
+### *adobeState.delete(key) ⇒ <code>Promise.&lt;(string\|null)&gt;</code>*
 Deletes a state key-value pair
 
 **Kind**: instance method of [<code>AdobeState</code>](#AdobeState)  
-**Returns**: <code>Promise.&lt;string&gt;</code> - key of deleted state or `null` if state does not exist  
+**Returns**: <code>Promise.&lt;(string\|null)&gt;</code> - key of deleted state or `null` if state does not exist  
 
 | Param | Type | Description |
 | --- | --- | --- |
 | key | <code>string</code> | state key identifier |
 
 <a name="AdobeState+deleteAll"></a>
 
-### *adobeState.deleteAll() ⇒ <code>Promise.&lt;boolean&gt;</code>*
-Deletes all key-values
+### *adobeState.deleteAll(options) ⇒ <code>Promise.&lt;{keys: number}&gt;</code>*
+Deletes multiple key-values. The match option is required as a safeguard.
+CAUTION: use `{ match: '*' }` to delete all key-values.
 
 **Kind**: instance method of [<code>AdobeState</code>](#AdobeState)  
-**Returns**: <code>Promise.&lt;boolean&gt;</code> - true if deleted, false if not  
+**Returns**: <code>Promise.&lt;{keys: number}&gt;</code> - returns an object with the number of deleted keys.  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| options | <code>object</code> | deleteAll options. |
+| options.match | <code>string</code> | REQUIRED, a glob pattern to specify which keys to delete. |
+
+**Example**  
+```js
+await state.deleteAll({ match: 'abc*' })
+```
 <a name="AdobeState+any"></a>
 
 ### *adobeState.any() ⇒ <code>Promise.&lt;boolean&gt;</code>*
-There exists key-values.
+There exists key-values in the region.
 
 **Kind**: instance method of [<code>AdobeState</code>](#AdobeState)  
 **Returns**: <code>Promise.&lt;boolean&gt;</code> - true if exists, false if not  
 <a name="AdobeState+stats"></a>
 
-### *adobeState.stats() ⇒ <code>Promise.&lt;({bytesKeys: number, bytesValues: number, keys: number}\|boolean)&gt;</code>*
+### *adobeState.stats() ⇒ <code>Promise.&lt;{bytesKeys: number, bytesValues: number, keys: number}&gt;</code>*
 Get stats.
 
 **Kind**: instance method of [<code>AdobeState</code>](#AdobeState)  
-**Returns**: <code>Promise.&lt;({bytesKeys: number, bytesValues: number, keys: number}\|boolean)&gt;</code> - namespace stats or false if not exists  
+**Returns**: <code>Promise.&lt;{bytesKeys: number, bytesValues: number, keys: number}&gt;</code> - State container stats.  
 <a name="AdobeState+list"></a>
 
 ### *adobeState.list(options) ⇒ <code>AsyncGenerator.&lt;{keys: Array.&lt;string&gt;}&gt;</code>*

e2e/e2e.js

@@ -22,7 +22,7 @@ const { MAX_TTL_SECONDS } = require('../lib/constants')
 const stateLib = require('../index')
 const { randomInt } = require('node:crypto')
 
-const uniquePrefix = `${Date.now()}.${randomInt(10)}`
+const uniquePrefix = `${Date.now()}.${randomInt(100)}`
 const testKey = `${uniquePrefix}__e2e_test_state_key`
 const testKey2 = `${uniquePrefix}__e2e_test_state_key2`
 
@@ -34,13 +34,33 @@ const initStateEnv = async (n = 1) => {
   process.env.__OW_API_KEY = process.env[`TEST_AUTH_${n}`]
   process.env.__OW_NAMESPACE = process.env[`TEST_NAMESPACE_${n}`]
   const state = await stateLib.init()
-  // // make sure we cleanup the namespace, note that delete might fail as it is an op under test
-  // await state.delete(`${uniquePrefix}*`)
-  await state.delete(testKey)
-  await state.delete(testKey2)
+  // make sure we cleanup the namespace, note that delete might fail as it is an op under test
+  await state.deleteAll({ match: `${uniquePrefix}*` })
   return state
 }
 
+// helpers
+const genKeyStrings = (n, identifier) => {
+  return (new Array(n).fill(0).map((_, idx) => {
+    const char = String.fromCharCode(97 + idx % 26)
+    // list-[a-z]-[0-(N-1)]
+    return `${uniquePrefix}__${identifier}_${char}_${idx}`
+  }))
+}
+const putKeys = async (state, keys, ttl) => {
+  const _putKeys = async (keys, ttl) => {
+    await Promise.all(keys.map(async (k, idx) => await state.put(k, `value-${idx}`, { ttl })))
+  }
+
+  const batchSize = 20
+  let i = 0
+  while (i < keys.length - batchSize) {
+    await _putKeys(keys.slice(i, i + batchSize), ttl)
+    i += batchSize
+  }
+  // final call
+  await _putKeys(keys.slice(i), ttl)
+}
 const waitFor = (ms) => new Promise(resolve => setTimeout(resolve, ms))
 
 test('env vars', () => {
@@ -146,34 +166,12 @@ describe('e2e tests using OpenWhisk credentials (as env vars)', () => {
     await expect(state.put(testKey, testValue, { ttl: -1 })).rejects.toThrow()
   })
 
-  test('listKeys test: few < 128 keys, many, and expired entries', async () => {
+  test('listKeys test: few, many, and expired entries', async () => {
     const state = await initStateEnv()
 
-    const genKeyStrings = (n) => {
-      return (new Array(n).fill(0).map((_, idx) => {
-        const char = String.fromCharCode(97 + idx % 26)
-        // list-[a-z]-[0-(N-1)]
-        return `${uniquePrefix}__list_${char}_${idx}`
-      }))
-    }
-    const putKeys = async (keys, ttl) => {
-      const _putKeys = async (keys, ttl) => {
-        await Promise.all(keys.map(async (k, idx) => await state.put(k, `value-${idx}`, { ttl })))
-      }
-
-      const batchSize = 20
-      let i = 0
-      while (i < keys.length - batchSize) {
-        await _putKeys(keys.slice(i, i + batchSize), ttl)
-        i += batchSize
-      }
-      // final call
-      await _putKeys(keys.slice(i), ttl)
-    }
-
     // 1. test with not many elements, one iteration should return all
-    const keys90 = genKeyStrings(90).sort()
-    await putKeys(keys90, 60)
+    const keys90 = genKeyStrings(90, 'list').sort()
+    await putKeys(state, keys90, 60)
 
     let it, ret
 
@@ -203,8 +201,8 @@ describe('e2e tests using OpenWhisk credentials (as env vars)', () => {
     expect(await it.next()).toEqual({ done: true, value: undefined })
 
     // 2. test with many elements and large countHint
-    const keys900 = genKeyStrings(900)
-    await putKeys(keys900, 60)
+    const keys900 = genKeyStrings(900, 'list')
+    await putKeys(state, keys900, 60)
 
     // note: we can't list in isolation without prefix
     it = state.list({ countHint: 1000 })
@@ -255,7 +253,7 @@ describe('e2e tests using OpenWhisk credentials (as env vars)', () => {
     expect(retArray.length).toEqual(1)
 
     // 4. make sure expired keys aren't listed
-    await putKeys(keys90, 1)
+    await putKeys(state, keys90, 1)
     await waitFor(2000)
 
     it = state.list({ countHint: 1000, match: `${uniquePrefix}__list_*` })
@@ -264,6 +262,23 @@ describe('e2e tests using OpenWhisk credentials (as env vars)', () => {
     expect(await it.next()).toEqual({ done: true, value: undefined })
   })
 
+  test('deleteAll test', async () => {
+    const state = await initStateEnv()
+
+    // < 100 keys
+    const keys90 = genKeyStrings(90, 'deleteAll').sort()
+    await putKeys(state, keys90, 60)
+    expect(await state.deleteAll({ match: `${uniquePrefix}__deleteAll_a*` })).toEqual({ keys: 4 })
+    expect(await state.deleteAll({ match: `${uniquePrefix}__deleteAll_*` })).toEqual({ keys: 86 })
+
+    // > 1000 keys
+    const keys1100 = genKeyStrings(1100, 'deleteAll').sort()
+    await putKeys(state, keys1100, 60)
+    expect(await state.deleteAll({ match: `${uniquePrefix}__deleteAll_*_1` })).toEqual({ keys: 1 })
+    expect(await state.deleteAll({ match: `${uniquePrefix}__deleteAll_*_1*0` })).toEqual({ keys: 21 }) // 10, 100 - 190, 1000-1090
+    expect(await state.deleteAll({ match: `${uniquePrefix}__deleteAll_*` })).toEqual({ keys: 1078 })
+  })
+
   test('throw error when get/put with invalid keys', async () => {
     const invalidKey = 'some/invalid:key'
     const state = await initStateEnv()

lib/AdobeState.js

@@ -28,7 +28,7 @@ const {
   MAX_LIST_COUNT_HINT,
   REQUEST_ID_HEADER,
   MIN_LIST_COUNT_HINT,
-  REGEX_PATTERN_LIST_KEY_MATCH,
+  REGEX_PATTERN_MATCH_KEY,
   MAX_TTL_SECONDS
 } = require('./constants')
 
@@ -370,7 +370,7 @@ class AdobeState {
    * Deletes a state key-value pair
    *
    * @param {string} key state key identifier
-   * @returns {Promise<string>} key of deleted state or `null` if state does not exist
+   * @returns {Promise<string|null>} key of deleted state or `null` if state does not exist
    * @memberof AdobeState
    */
   async delete (key) {
@@ -383,6 +383,23 @@ class AdobeState {
       }
     }
 
+    const schema = {
+      type: 'object',
+      properties: {
+        key: {
+          type: 'string',
+          pattern: REGEX_PATTERN_STORE_KEY
+        }
+      }
+    }
+    const { valid, errors } = validate(schema, { key })
+    if (!valid) {
+      logAndThrow(new codes.ERROR_BAD_ARGUMENT({
+        messageValues: utils.formatAjvErrors(errors),
+        sdkDetails: { key, errors }
+      }))
+    }
+
     logger.debug('delete', requestOptions)
 
     const promise = this.fetchRetry.exponentialBackoff(this.createRequestUrl(`/data/${key}`), requestOptions)
@@ -395,12 +412,16 @@ class AdobeState {
   }
 
   /**
-   * Deletes all key-values
-   *
-   * @returns {Promise<boolean>} true if deleted, false if not
+   * Deletes multiple key-values. The match option is required as a safeguard.
+   * CAUTION: use `{ match: '*' }` to delete all key-values.
+   * @example
+   *  await state.deleteAll({ match: 'abc*' })
+   * @param {object} options deleteAll options.
+   * @param {string} options.match REQUIRED, a glob pattern to specify which keys to delete.
+   * @returns {Promise<{ keys: number }>} returns an object with the number of deleted keys.
    * @memberof AdobeState
    */
-  async deleteAll () {
+  async deleteAll (options = {}) {
     const requestOptions = {
       method: 'DELETE',
       headers: {
@@ -410,13 +431,37 @@ class AdobeState {
 
     logger.debug('deleteAll', requestOptions)
 
-    const promise = this.fetchRetry.exponentialBackoff(this.createRequestUrl(), requestOptions)
+    const schema = {
+      type: 'object',
+      properties: {
+        match: { type: 'string', pattern: REGEX_PATTERN_MATCH_KEY }
+      },
+      required: ['match'] // safeguard, you cannot call deleteAll without matching specific keys!
+    }
+    const { valid, errors } = validate(schema, options)
+    if (!valid) {
+      logAndThrow(new codes.ERROR_BAD_ARGUMENT({
+        messageValues: utils.formatAjvErrors(errors),
+        sdkDetails: { options, errors }
+      }))
+    }
+
+    const queryParams = { matchData: options.match }
+
+    // ! be extra cautious, if the `matchData` param is not specified the whole container will be deleted
+    const promise = this.fetchRetry.exponentialBackoff(this.createRequestUrl('', queryParams), requestOptions)
     const response = await _wrap(promise, {})
-    return (response.status !== 404)
+
+    if (response.status === 404) {
+      return { keys: 0 }
+    } else {
+      const { keys } = await response.json()
+      return { keys }
+    }
   }
 
   /**
-   * There exists key-values.
+   * There exists key-values in the region.
    *
    * @returns {Promise<boolean>} true if exists, false if not
    * @memberof AdobeState
@@ -439,7 +484,7 @@ class AdobeState {
   /**
    * Get stats.
    *
-   * @returns {Promise<{ bytesKeys: number, bytesValues: number, keys: number} | boolean>} namespace stats or false if not exists
+   * @returns {Promise<{ bytesKeys: number, bytesValues: number, keys: number }>} State container stats.
    * @memberof AdobeState
    */
   async stats () {
@@ -455,9 +500,10 @@ class AdobeState {
     const promise = this.fetchRetry.exponentialBackoff(this.createRequestUrl(), requestOptions)
     const response = await _wrap(promise, {})
     if (response.status === 404) {
-      return false
+      return { keys: 0, bytesKeys: 0, bytesValues: 0 }
     } else {
-      return response.json()
+      const { keys, bytesKeys, bytesValues } = await response.json()
+      return { keys, bytesKeys, bytesValues }
     }
   }
 
@@ -504,7 +550,7 @@ class AdobeState {
     const schema = {
       type: 'object',
       properties: {
-        match: { type: 'string', pattern: REGEX_PATTERN_LIST_KEY_MATCH }, // this is an important check
+        match: { type: 'string', pattern: REGEX_PATTERN_MATCH_KEY },
         countHint: { type: 'integer' }
       }
     }

lib/constants.js

@@ -46,8 +46,8 @@ const HEADER_KEY_EXPIRES = 'x-key-expires-ms'
 const REGEX_PATTERN_STORE_NAMESPACE = '^(development-)?([0-9]{3,10})-([a-z0-9]{1,20})(-([a-z0-9]{1,20}))?$'
 // The regex for keys, allowed chars are alphanumerical with _ - .
 const REGEX_PATTERN_STORE_KEY = `^[a-zA-Z0-9-_.]{1,${MAX_KEY_SIZE}}$`
-// The regex for list key pattern, allowed chars are alphanumerical with _ - . and * for glob matching
-const REGEX_PATTERN_LIST_KEY_MATCH = `^[a-zA-Z0-9-_.*]{1,${MAX_KEY_SIZE}}$`
+// Same as REGEX_PATTERN_STORE_KEY with an added * to support glob-style matching
+const REGEX_PATTERN_MATCH_KEY = `^[a-zA-Z0-9-_.*]{1,${MAX_KEY_SIZE}}$`
 const MAX_LIST_COUNT_HINT = 1000
 const MIN_LIST_COUNT_HINT = 100
 
@@ -62,7 +62,7 @@ module.exports = {
   REGEX_PATTERN_STORE_NAMESPACE,
   REGEX_PATTERN_STORE_KEY,
   HEADER_KEY_EXPIRES,
-  REGEX_PATTERN_LIST_KEY_MATCH,
+  REGEX_PATTERN_MATCH_KEY,
   MAX_LIST_COUNT_HINT,
   MIN_LIST_COUNT_HINT,
   REQUEST_ID_HEADER,