fix(api): get env by secretKey

.env.example

@@ -73,3 +73,9 @@ DEFAULT_GITHUB_CLIENT_SECRET=
 # Hosted Auth Configuration
 WORKOS_API_KEY=
 WORKOS_CLIENT_ID=
+
+# Encryption key for secrets / records stored in database
+NANGO_ENCRYPTION_KEY=
+
+# Enable or disable environment keys cache
+NANGO_CACHE_ENV_KEYS=

packages/server/lib/controllers/access.middleware.ts

@@ -8,7 +8,11 @@ import {
     setEnvironmentId,
     isBasicAuthEnabled,
     errorManager,
-    userService
+    userService,
+    logger,
+    stringifyError,
+    telemetry,
+    MetricTypes
 } from '@nangohq/shared';
 import tracer from 'dd-trace';
 
@@ -32,12 +36,16 @@ export class AccessMiddleware {
 
         let accountId: number | null;
         let environmentId: number | null;
+        const start = Date.now();
         try {
             const result = await environmentService.getAccountIdAndEnvironmentIdBySecretKey(secret);
             accountId = result?.accountId as number;
             environmentId = result?.environmentId as number;
-        } catch (_) {
+        } catch (err) {
+            logger.error(`failed_get_env_by_secret_key ${stringifyError(err)}`);
             return errorManager.errRes(res, 'malformed_auth_header');
+        } finally {
+            telemetry.duration(MetricTypes.AUTH_GET_ENV_BY_SECRET_KEY, Date.now() - start);
         }
 
         if (accountId == null) {

packages/shared/lib/db/migrations/20240326141028_hash-key.cjs

@@ -0,0 +1,36 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+const utils = require('node:util');
+const crypto = require('node:crypto');
+
+const pbkdf2 = utils.promisify(crypto.pbkdf2);
+const ENCRYPTION_KEY = process.env['NANGO_ENCRYPTION_KEY'];
+
+function decrypt(enc, iv, authTag) {
+    const decipher = crypto.createDecipheriv('aes-256-gcm', Buffer.from(ENCRYPTION_KEY, 'base64'), Buffer.from(iv, 'base64'));
+    decipher.setAuthTag(Buffer.from(authTag, 'base64'));
+    let str = decipher.update(enc, 'base64', 'utf8');
+    str += decipher.final('utf8');
+    return str;
+}
+
+exports.up = async function (knex) {
+    await knex.schema.raw('ALTER TABLE "_nango_environments" ADD COLUMN IF NOT EXISTS "secret_key_hashed" varchar(64)');
+
+    if (!ENCRYPTION_KEY) {
+        return;
+    }
+
+    const envs = await knex.select('*').from(`_nango_environments`).where({ secret_key_hashed: null });
+
+    for (const env of envs) {
+        const decrypted = env.secret_key_iv ? decrypt(env.secret_key, env.secret_key_iv, env.secret_key_tag) : env.secret_key;
+        await knex
+            .from(`_nango_environments`)
+            .where({ id: env.id })
+            .update({ secret_key_hashed: (await pbkdf2(decrypted, ENCRYPTION_KEY, 310000, 32, 'sha256')).toString('base64') });
+    }
+};
+
+exports.down = async function (knex) {
+    await knex.schema.raw('ALTER TABLE "_nango_environments" DROP COLUMN IF EXISTS "secret_key_hashed"');
+};

packages/shared/lib/db/seeders/account.seeder.ts

@@ -0,0 +1,12 @@
+import { v4 as uuid } from 'uuid';
+import type { Account } from '../../models';
+
+import accountService from '../../services/account.service';
+
+export async function createAccount(): Promise<Account> {
+    const acc = await accountService.createAccount(uuid());
+    if (!acc) {
+        throw new Error('failed_to_create_account');
+    }
+    return acc;
+}

packages/shared/lib/models/Environment.ts

@@ -9,6 +9,7 @@ export interface Environment extends Timestamps {
     public_key: string;
     secret_key_iv?: string | null;
     secret_key_tag?: string | null;
+    secret_key_hashed?: string | null;
     callback_url: string | null;
     webhook_url: string | null;
     websockets_path?: string | null;

packages/shared/lib/services/environment.service.integration.test.ts

@@ -0,0 +1,83 @@
+import { expect, describe, it, beforeAll } from 'vitest';
+import environmentService, { hashSecretKey } from './environment.service.js';
+import { v4 as uuid } from 'uuid';
+import { multipleMigrations } from '../db/database.js';
+import { createAccount } from '../db/seeders/account.seeder.js';
+
+describe('Environment service', () => {
+    beforeAll(async () => {
+        await multipleMigrations();
+    });
+
+    it('should create a service with secrets', async () => {
+        const account = await createAccount();
+        const envName = uuid();
+        const env = await environmentService.createEnvironment(account.id, envName);
+        if (!env) {
+            throw new Error('failed_to_create_env');
+        }
+
+        expect(env).toStrictEqual({
+            account_id: account.id,
+            always_send_webhook: false,
+            callback_url: null,
+            created_at: expect.toBeIsoDate(),
+            hmac_enabled: false,
+            hmac_key: null,
+            id: expect.any(Number),
+            name: envName,
+            pending_public_key: null,
+            pending_secret_key: null,
+            pending_secret_key_iv: null,
+            pending_secret_key_tag: null,
+            public_key: expect.any(String),
+            secret_key: expect.any(String),
+            secret_key_hashed: expect.any(String),
+            secret_key_iv: expect.any(String),
+            secret_key_tag: expect.any(String),
+            send_auth_webhook: false,
+            slack_notifications: false,
+            updated_at: expect.toBeIsoDate(),
+            uuid: expect.any(String),
+            webhook_url: null
+        });
+
+        expect(env.secret_key).not.toEqual(env.secret_key_hashed);
+    });
+
+    it('should retrieve env by secretKey', async () => {
+        const account = await createAccount();
+        const env = await environmentService.createEnvironment(account.id, uuid());
+
+        const get = await environmentService.getAccountIdAndEnvironmentIdBySecretKey(env!.secret_key);
+
+        expect(get).toStrictEqual({
+            accountId: account.id,
+            environmentId: env!.id
+        });
+    });
+
+    it('should rotate secretKey', async () => {
+        const account = await createAccount();
+        const env = (await environmentService.createEnvironment(account.id, uuid()))!;
+        expect(env.secret_key).toBeUUID();
+
+        // Rotate
+        await environmentService.rotateSecretKey(env.id);
+
+        const env2 = (await environmentService.getById(env.id))!;
+        expect(env2.pending_secret_key).not.toBeNull();
+        expect(env2.pending_secret_key).not.toEqual(env2.secret_key);
+        expect(env2.secret_key_hashed).not.toEqual(env.secret_key);
+        expect(env2.secret_key_hashed).toEqual(await hashSecretKey(env.secret_key));
+
+        // Activate
+        await environmentService.activateSecretKey(env.id);
+
+        const env3 = (await environmentService.getById(env.id))!;
+        expect(env3.secret_key).toBeUUID();
+        expect(env3.pending_secret_key).toBeNull();
+        expect(env3.secret_key).toEqual(env2.pending_secret_key);
+        expect(env3.secret_key_hashed).toEqual(await hashSecretKey(env3.secret_key));
+    });
+});

packages/shared/lib/services/environment.service.ts

@@ -1,6 +1,6 @@
 import * as uuid from 'uuid';
-import db, { schema } from '../db/database.js';
-import encryptionManager from '../utils/encryption.manager.js';
+import db from '../db/database.js';
+import encryptionManager, { ENCRYPTION_KEY, pbkdf2 } from '../utils/encryption.manager.js';
 import type { Environment } from '../models/Environment.js';
 import type { EnvironmentVariable } from '../models/EnvironmentVariable.js';
 import type { Account } from '../models/Admin.js';
@@ -20,11 +20,16 @@ interface EnvironmentAccount {
 type EnvironmentAccountSecrets = Record<string, EnvironmentAccount>;
 
 export const defaultEnvironments = ['prod', 'dev'];
+const CACHE_ENABLED = !(process.env['NANGO_CACHE_ENV_KEYS'] === 'false');
 
 class EnvironmentService {
     private environmentAccountSecrets: EnvironmentAccountSecrets = {} as EnvironmentAccountSecrets;
 
     async cacheSecrets(): Promise<void> {
+        if (!CACHE_ENABLED) {
+            return;
+        }
+
         const environmentAccounts = await db.knex.select('*').from<Environment>(TABLE);
 
         const environmentAccountSecrets: EnvironmentAccountSecrets = {};
@@ -45,6 +50,10 @@ class EnvironmentService {
     }
 
     private addToEnvironmentSecretCache(accountEnvironment: Environment) {
+        if (!CACHE_ENABLED) {
+            return;
+        }
+
         this.environmentAccountSecrets[accountEnvironment.secret_key] = {
             accountId: accountEnvironment.account_id,
             environmentId: accountEnvironment.id,
@@ -96,11 +105,15 @@ class EnvironmentService {
 
         if (!this.environmentAccountSecrets[secretKey]) {
             // If the secret key is not in the cache, try to get it from the database
-            const fromDb = await db.knex.select('*').from<Environment>(TABLE).where({ secret_key: secretKey }).first();
+            const hashed = await hashSecretKey(secretKey);
+            const fromDb = await db.knex.select('*').from<Environment>(TABLE).where({ secret_key_hashed: hashed }).first();
             if (!fromDb) {
                 return null;
             }
-            this.addToEnvironmentSecretCache(fromDb);
+            if (!CACHE_ENABLED) {
+                return { accountId: fromDb.account_id, environmentId: fromDb.id };
+            }
+            this.addToEnvironmentSecretCache(encryptionManager.decryptEnvironment(fromDb)!);
         }
 
         const { accountId, environmentId } = this.environmentAccountSecrets[secretKey] as EnvironmentAccount;
@@ -177,29 +190,6 @@ class EnvironmentService {
         return { accountId: result.account_id, environmentId: result.id };
     }
 
-    async getByAccountIdAndEnvironment(id: number): Promise<Environment | null> {
-        try {
-            const result = await db.knex.select('*').from<Environment>(TABLE).where({ id });
-
-            if (result == null || result.length == 0 || result[0] == null) {
-                return null;
-            }
-
-            return encryptionManager.decryptEnvironment(result[0]);
-        } catch (e) {
-            await errorManager.report(e, {
-                environmentId: id,
-                source: ErrorSourceEnum.PLATFORM,
-                operation: LogActionEnum.DATABASE,
-                metadata: {
-                    id
-                }
-            });
-
-            return null;
-        }
-    }
-
     async getAccountAndEnvironmentById(account_id: number, environment: string): Promise<{ account: Account | null; environment: Environment | null }> {
         const account = await db.knex.select('*').from<Account>(`_nango_accounts`).where({ id: account_id });
 
@@ -228,7 +218,7 @@ class EnvironmentService {
 
     async getById(id: number): Promise<Environment | null> {
         try {
-            const result = (await schema().select('*').from<Environment>(TABLE).where({ id })) as unknown as Environment[];
+            const result = await db.knex.select('*').from<Environment>(TABLE).where({ id });
 
             if (result == null || result.length == 0 || result[0] == null) {
                 return null;
@@ -281,18 +271,24 @@ class EnvironmentService {
     }
 
     async createEnvironment(accountId: number, environment: string): Promise<Environment | null> {
-        const result: void | Pick<Environment, 'id'> = await schema().from<Environment>(TABLE).insert({ account_id: accountId, name: environment }, ['id']);
+        const result = await db.knex.from<Environment>(TABLE).insert({ account_id: accountId, name: environment }).returning('id');
 
-        if (Array.isArray(result) && result.length === 1 && result[0] != null && 'id' in result[0]) {
+        if (Array.isArray(result) && result.length === 1 && result[0] && 'id' in result[0]) {
             const environmentId = result[0]['id'];
             const environment = await this.getById(environmentId);
-
-            if (environment != null) {
-                const encryptedEnvironment = encryptionManager.encryptEnvironment(environment);
-                await schema().from<Environment>(TABLE).where({ id: environmentId }).update(encryptedEnvironment);
-                this.addToEnvironmentSecretCache(environment);
-                return encryptedEnvironment;
+            if (!environment) {
+                return null;
             }
+
+            const encryptedEnvironment = encryptionManager.encryptEnvironment({
+                ...environment,
+                secret_key_hashed: await hashSecretKey(environment.secret_key)
+            });
+            await db.knex.from<Environment>(TABLE).where({ id: environmentId }).update(encryptedEnvironment);
+
+            const env = encryptionManager.decryptEnvironment(encryptedEnvironment)!;
+            this.addToEnvironmentSecretCache(env);
+            return env;
         }
 
         return null;
@@ -500,21 +496,23 @@ class EnvironmentService {
             return false;
         }
 
+        const decrypted = encryptionManager.decryptEnvironment(environment)!;
         await db.knex
             .from<Environment>(TABLE)
             .where({ id })
             .update({
                 secret_key: environment.pending_secret_key as string,
                 secret_key_iv: environment.pending_secret_key_iv as string,
                 secret_key_tag: environment.pending_secret_key_tag as string,
+                secret_key_hashed: await hashSecretKey(decrypted.pending_secret_key!),
                 pending_secret_key: null,
                 pending_secret_key_iv: null,
                 pending_secret_key_tag: null
             });
 
-        if (this.environmentAccountSecrets[environment.secret_key]) {
+        if (this.environmentAccountSecrets[decrypted.secret_key]) {
             // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
-            delete this.environmentAccountSecrets[environment.secret_key];
+            delete this.environmentAccountSecrets[decrypted.secret_key];
         }
 
         const updatedEnvironment = await this.getById(id);
@@ -547,4 +545,12 @@ class EnvironmentService {
     }
 }
 
+export async function hashSecretKey(key: string) {
+    if (!ENCRYPTION_KEY) {
+        return key;
+    }
+
+    return (await pbkdf2(key, ENCRYPTION_KEY, 310000, 32, 'sha256')).toString('base64');
+}
+
 export default new EnvironmentService();