From 217a9b3b91a8a5568481a75fb922ab5e839d8423 Mon Sep 17 00:00:00 2001 From: Moritz Raho Date: Tue, 12 Mar 2024 11:41:31 +0100 Subject: [PATCH] feat: support for stats() and . in key (#140) --- README.md | 68 ++++++----------------------------------- doc/api.md | 8 +++++ e2e/.env.example | 9 +++--- e2e/e2e.js | 18 ++++++----- e2e/e2e.md | 2 ++ lib/AdobeState.js | 28 +++++++++++++++-- lib/constants.js | 4 +-- test/AdobeState.test.js | 34 ++++++++++++++++++--- types.d.ts | 5 +++ 9 files changed, 97 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index dcd7ee2..d269715 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ A Node JavaScript abstraction on top of distributed/cloud DBs that exposes a sim You can initialize the lib with your Adobe I/O Runtime (a.k.a OpenWhisk) credentials. -Please note that currently you must be a customer of [Adobe Developer App Builder](https://www.adobe.io/apis/experienceplatform/project-firefly.html) to use this library. App Builder is a complete framework that enables enterprise developers to build and deploy custom web applications that extend Adobe Experience Cloud solutions and run on Adobe infrastructure. +Please note that currently, you must be a customer of [Adobe Developer App Builder](https://www.adobe.io/apis/experienceplatform/project-firefly.html) to use this library. App Builder is a complete framework that enables enterprise developers to build and deploy custom web applications that extend Adobe Experience Cloud solutions and run on Adobe infrastructure. ## Install @@ -46,7 +46,7 @@ npm install @adobe/aio-lib-state // put await state.put('key', 'value') - await state.put('another key', 'another value', { ttl: -1 }) // -1 for no expiry, defaults to 86400 (24 hours) + await state.put('another key', 'another value', { ttl: -1 }) // -1 for max expiry (365 days), defaults to 86400 (24 hours) // delete await state.delete('key') @@ -68,64 +68,14 @@ set `DEBUG=@adobe/aio-lib-state*` to see debug logs. ## Adobe I/O State Store Limitations (per user) -Apply when init with OW credentials (and not own cloud DB credentials): +Apply when init with I/O Runtime credentials: -- Max state value size: `2MB` -- Max state key size: `1024 bytes` -- Max total state size: `10 GB` -- Token expiry (need to re-init after expiry): `1 hour` -- Non supported characters for state keys are: `'/', '\', '?', '#'` - -## Adobe I/O State Store Consistency Guarantees - -### Consistency across State Instances - -Operations across multiple State instances (returned by `stateLib.init()`) are **eventually consistent**. For example, let's consider two state instances `a` and `b` initialized with the same credentials, then - -```javascript -const a = await state.init() -const b = await state.init() -await a.put('food', 'beans') -await b.put('food', 'carrots') -console.log(await a.get('food')) -``` - -might log either `beans` or `carrots` but eventually `a.get('food')` will always return `carrots`. - -Operations within a single instance however are guaranteed to be **strongly consistent**. - -Note that atomicity is ensured, i.e. `a.get('food')` will never return something like `beacarronsts`. - -### Adobe I/O Runtime considerations - -State lib is expected to be used in Adobe I/O Runtime serverless actions. A new State instance can be created on every new invocation inside the main function of the serverless action as follows: - -```javascript -const State = require('@adobe/aio-sdk').State - -function main (params) { - const state = await State.init() - // do operations on state -``` - -It's important to understand that in this case, on every invocation a new State instance is created, meaning that operations will be only **eventually consistent** across invocations but **strongly consistent** within an invocation. - -Also note that reusing the State instance by storing it in a global variable outside of the main function would not ensure **strong consistency** across all invocations as the action could be executed in a separate Docker container. - -Here is an example showcasing two invocations of the same action with an initial state `{ key: 'hello'}`: - -Invocation A | Invocation B | -| :---------------------------------- | ----------------------------------: | -`state = State.init()` | | -`state.get(key)` => returns hello | | -`state.put(key, 'bonjour')` | | -`state.get(key)` => returns bonjour | | -| | `state = State.init()` | -| | `state.get(key)` => hello OR bonjour | -| | `state.put(key, 'bonjour')` | -| | `state.get(key)` => returns bonjour | - -Because of **eventual consistency** across State instances, in invocation B, the first `state.get(key)` might return an older value although invocation A has updated the value already. +- Namespace must be in valid AppBuilder format: `amsorg-project(-workspace)?` +- Max state value size: `1MB`. +- Max state key size: `1024 bytes`. +- Supported characters are alphanumeric and `-`,`_`,`.` +- Max-supported TTL is 365 days. +- Default TTL is 1 day. ## Troubleshooting diff --git a/doc/api.md b/doc/api.md index 17ce2f3..693b322 100644 --- a/doc/api.md +++ b/doc/api.md @@ -57,6 +57,7 @@ Cloud State Management * *[.delete(key)](#AdobeState+delete) ⇒ Promise.<string>* * *[.deleteAll()](#AdobeState+deleteAll) ⇒ Promise.<boolean>* * *[.any()](#AdobeState+any) ⇒ Promise.<boolean>* + * *[.stats()](#AdobeState+stats) ⇒ Promise.<boolean>* @@ -109,6 +110,13 @@ Deletes all key-values ### *adobeState.any() ⇒ Promise.<boolean>* There exists key-values. +**Kind**: instance method of [AdobeState](#AdobeState) +**Returns**: Promise.<boolean> - true if exists, false if not + + +### *adobeState.stats() ⇒ Promise.<boolean>* +Get stats. + **Kind**: instance method of [AdobeState](#AdobeState) **Returns**: Promise.<boolean> - true if exists, false if not diff --git a/e2e/.env.example b/e2e/.env.example index 1f72db1..2b23112 100644 --- a/e2e/.env.example +++ b/e2e/.env.example @@ -1,4 +1,5 @@ -TEST_NAMESPACE_1= -TEST_AUTH_1= -TEST_NAMESPACE_2= -TEST_AUTH_2= +TEST_NAMESPACE_1='11111-test' +TEST_AUTH_1='testauth' +TEST_AUTH_2='testauth2' +TEST_NAMESPACE_2='12345-test-stage' +ADOBE_STATE_STORE_ENDPOINT_PROD='127.0.0.1:8080' diff --git a/e2e/e2e.js b/e2e/e2e.js index 475cf39..372828d 100644 --- a/e2e/e2e.js +++ b/e2e/e2e.js @@ -22,6 +22,7 @@ const { MAX_TTL_SECONDS } = require('../lib/constants') const stateLib = require('../index') const testKey = 'e2e_test_state_key' +const testKey2 = 'e2e_test_state_key2' jest.setTimeout(30000) // thirty seconds per test @@ -31,8 +32,8 @@ const initStateEnv = async (n = 1) => { process.env.__OW_API_KEY = process.env[`TEST_AUTH_${n}`] process.env.__OW_NAMESPACE = process.env[`TEST_NAMESPACE_${n}`] const state = await stateLib.init() - // make sure we delete the testKey, note that delete might fail as it is an op under test - await state.delete(testKey) + // make sure we cleanup the namespace, note that delete might fail as it is an op under test + await state.deleteAll() return state } @@ -74,10 +75,13 @@ describe('e2e tests using OpenWhisk credentials (as env vars)', () => { expect(await state.get(testKey)).toEqual(undefined) expect(await state.any()).toEqual(false) expect(await state.put(testKey, testValue)).toEqual(testKey) + expect(await state.put(testKey2, testValue)).toEqual(testKey2) expect(await state.any()).toEqual(true) + expect(await state.stats()).toEqual({ bytesKeys: testKey.length + testKey2.length, bytesValues: testValue.length * 2, keys: 2 }) expect(await state.deleteAll()).toEqual(true) expect(await state.get(testKey)).toEqual(undefined) expect(await state.any()).toEqual(false) + expect(await state.stats()).toEqual(false) }) test('time-to-live tests: write w/o ttl, get default ttl, write with ttl, get, get after ttl', async () => { @@ -116,10 +120,10 @@ describe('e2e tests using OpenWhisk credentials (as env vars)', () => { }) test('throw error when get/put with invalid keys', async () => { - const invalidKey = 'some/invalid/key' + const invalidKey = 'some/invalid:key' const state = await initStateEnv() - await expect(state.put(invalidKey, 'testValue')).rejects.toThrow('[AdobeStateLib:ERROR_BAD_ARGUMENT] /key must match pattern "^[a-zA-Z0-9-_-]{1,1024}$"') - await expect(state.get(invalidKey)).rejects.toThrow('[AdobeStateLib:ERROR_BAD_ARGUMENT] /key must match pattern "^[a-zA-Z0-9-_-]{1,1024}$"') + await expect(state.put(invalidKey, 'testValue')).rejects.toThrow('[AdobeStateLib:ERROR_BAD_ARGUMENT] /key must match pattern "^[a-zA-Z0-9-_.]{1,1024}$"') + await expect(state.get(invalidKey)).rejects.toThrow('[AdobeStateLib:ERROR_BAD_ARGUMENT] /key must match pattern "^[a-zA-Z0-9-_.]{1,1024}$"') }) test('isolation tests: get, write, delete on same key for two namespaces do not interfere', async () => { @@ -145,9 +149,9 @@ describe('e2e tests using OpenWhisk credentials (as env vars)', () => { await state2.delete(testKey) }) - test('error value bigger than 2MB test', async () => { + test('error value bigger than 1MB test', async () => { const state = await initStateEnv() - const bigValue = ('a').repeat(1024 * 1024 * 2 + 1) + const bigValue = ('a').repeat(1024 * 1024 + 1) let expectedError try { diff --git a/e2e/e2e.md b/e2e/e2e.md index fb5f4bf..5ae5f21 100644 --- a/e2e/e2e.md +++ b/e2e/e2e.md @@ -2,6 +2,8 @@ ## Requirements +**NOTE**: running the e2e tests will delete all keys in the provided namespaces, use with care! + - To run the test you'll need two OpenWhisk namespaces. Please set the credentials for those in the following env variables in an .env file: - `TEST_NAMESPACE_1, TEST_AUTH_1, TEST_NAMESPACE_2, TEST_AUTH_2` diff --git a/lib/AdobeState.js b/lib/AdobeState.js index 6db9c37..3b17b30 100644 --- a/lib/AdobeState.js +++ b/lib/AdobeState.js @@ -98,7 +98,8 @@ async function _wrap (promise, params) { case 429: return logAndThrow(new codes.ERROR_REQUEST_RATE_TOO_HIGH({ sdkDetails: copyParams })) default: - return logAndThrow(new codes.ERROR_INTERNAL({ messageValues: [`unexpected response from provider with status: ${status}`], sdkDetails: { ...cloneDeep(params), _internal: e.internal } })) + // NOTE: we should throw a different error if its not a response error + return logAndThrow(new codes.ERROR_INTERNAL({ messageValues: [`unexpected response from provider with status: ${status}`], sdkDetails: { ...cloneDeep(params), _internal: e.internal || e.message } })) } } return response @@ -126,6 +127,8 @@ class AdobeState { /** @private */ this.apikey = apikey /** @private */ + this.basicAuthHeader = `Basic ${Buffer.from(apikey).toString('base64')}` + /** @private */ this.region = region /** @private */ this.endpoint = ADOBE_STATE_STORE_ENDPOINT[getCliEnv()] @@ -168,7 +171,7 @@ class AdobeState { */ getAuthorizationHeaders () { return { - Authorization: `Basic ${this.apikey}` + Authorization: this.basicAuthHeader } } @@ -385,6 +388,27 @@ class AdobeState { const response = await _wrap(promise, {}) return response !== null } + + /** + * Get stats. + * + * @returns {Promise} true if exists, false if not + * @memberof AdobeState + */ + async stats () { + const requestOptions = { + method: 'GET', + headers: { + ...this.getAuthorizationHeaders() + } + } + + logger.debug('any', requestOptions) + + const promise = this.fetchRetry.exponentialBackoff(this.createRequestUrl(), requestOptions) + const response = await _wrap(promise, {}) + return !!response && response.json() + } } module.exports = { AdobeState } diff --git a/lib/constants.js b/lib/constants.js index 70b1823..763ca74 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -38,8 +38,8 @@ const MAX_TTL_SECONDS = 60 * 60 * 24 * 365 // 365 days const HEADER_KEY_EXPIRES = 'x-key-expires-ms' const REGEX_PATTERN_STORE_NAMESPACE = '^(development-)?([0-9]{3,10})-([a-z0-9]{1,20})(-([a-z0-9]{1,20}))?$' -// The regex for keys, allowed chars are alphanumerical with _ and - -const REGEX_PATTERN_STORE_KEY = `^[a-zA-Z0-9-_-]{1,${MAX_KEY_SIZE}}$` +// The regex for keys, allowed chars are alphanumerical with _ - . +const REGEX_PATTERN_STORE_KEY = `^[a-zA-Z0-9-_.]{1,${MAX_KEY_SIZE}}$` module.exports = { ADOBE_STATE_STORE_REGIONS, diff --git a/test/AdobeState.test.js b/test/AdobeState.test.js index fb8dfe9..79efdc7 100644 --- a/test/AdobeState.test.js +++ b/test/AdobeState.test.js @@ -50,7 +50,8 @@ const wrapInFetchResponse = (body, options = {}) => { headers: { get: headersGet }, - text: async () => body + text: async () => body, + json: async () => JSON.parse(body) } } @@ -161,7 +162,7 @@ describe('get', () => { test('invalid key', async () => { const key = 'bad/key' - await expect(store.get(key)).rejects.toThrow('[AdobeStateLib:ERROR_BAD_ARGUMENT] /key must match pattern "^[a-zA-Z0-9-_-]{1,1024}$"') + await expect(store.get(key)).rejects.toThrow('[AdobeStateLib:ERROR_BAD_ARGUMENT] /key must match pattern "^[a-zA-Z0-9-_.]{1,1024}$"') }) test('not found', async () => { @@ -193,7 +194,7 @@ describe('put', () => { }) test('success (string value) with ttl', async () => { - const key = 'valid-key' + const key = 'valid.for-those_chars' const value = 'some-value' const fetchResponseJson = {} @@ -207,7 +208,7 @@ describe('put', () => { const key = 'invalid/key' const value = 'some-value' - await expect(store.put(key, value)).rejects.toThrow('[AdobeStateLib:ERROR_BAD_ARGUMENT] /key must match pattern "^[a-zA-Z0-9-_-]{1,1024}$"') + await expect(store.put(key, value)).rejects.toThrow('[AdobeStateLib:ERROR_BAD_ARGUMENT] /key must match pattern "^[a-zA-Z0-9-_.]{1,1024}$"') }) test('failure (binary value)', async () => { @@ -325,6 +326,29 @@ describe('any', () => { store = await AdobeState.init(fakeCredentials) }) + test('success', async () => { + const fetchResponseJson = JSON.stringify({}) + mockExponentialBackoff.mockResolvedValue(wrapInFetchResponse(fetchResponseJson)) + + const value = await store.stats() + expect(value).toEqual({}) + }) + + test('not found', async () => { + mockExponentialBackoff.mockResolvedValue(wrapInFetchError(404)) + + const value = await store.stats() + expect(value).toEqual(false) + }) +}) + +describe('stats()', () => { + let store + + beforeEach(async () => { + store = await AdobeState.init(fakeCredentials) + }) + test('success', async () => { const fetchResponseJson = {} mockExponentialBackoff.mockResolvedValue(wrapInFetchResponse(fetchResponseJson)) @@ -346,7 +370,7 @@ describe('private methods', () => { test('getAuthorizationHeaders (private)', async () => { const expectedHeaders = { - Authorization: `Basic ${fakeCredentials.apikey}` + Authorization: `Basic ${Buffer.from(fakeCredentials.apikey).toString('base64')}` } const store = await AdobeState.init(fakeCredentials) diff --git a/types.d.ts b/types.d.ts index 36ed61c..aa9878f 100644 --- a/types.d.ts +++ b/types.d.ts @@ -72,6 +72,11 @@ export class AdobeState { * @returns true if exists, false if not */ any(): Promise; + /** + * Get stats. + * @returns true if exists, false if not + */ + stats(): Promise; } /**