(fix) [NAN-644] refactor logger and some environment detection logic …

…out of (#1913) ## Describe your changes Add ~`internals`~ `utils` package and move logger and some environment detection logic there ## Issue ticket number and link NAN-644 ## Checklist before requesting a review (skip if just adding/editing APIs & templates) - [ ] I added tests, otherwise the reason is: - [ ] I added observability, otherwise the reason is: - [ ] I added analytics, otherwise the reason is:
NangoHQ · Mar 27, 2024 · 1f4a7e2 · 1f4a7e2
1 parent 674a854
commit 1f4a7e2
Show file tree

Hide file tree

Showing 89 changed files with 1,811 additions and 1,439 deletions.
diff --git a/.github/workflows/integration-flows.yaml b/.github/workflows/integration-flows.yaml
@@ -28,7 +28,7 @@ jobs:
               run: |
                   # Build, install CLI and verify it can run
                   npm install
-                  npm run build -w @nangohq/node -w @nangohq/shared -w nango
+                  npm run build -w @nangohq/node -w @nangohq/utils -w @nangohq/shared -w nango
                   npm install -g ./packages/cli
                   NANGO_CLI_UPGRADE_MODE=ignore nango version --debug
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -10,7 +10,8 @@
         "packages/runner",
         "packages/persist",
         "packages/jobs",
-        "packages/webapp"
+        "packages/webapp",
+        "packages/utils"
     ],
     "scripts": {
         "create:migration": "cd packages/shared/lib/db && knex migrate:make $1 --esm --knexfile ./knexfile.cjs",

diff --git a/packages/jobs/Dockerfile b/packages/jobs/Dockerfile
@@ -10,6 +10,7 @@ WORKDIR /nango
 
 COPY packages/node-client/ packages/node-client/
 COPY packages/shared/ packages/shared/
+COPY packages/utils/ packages/utils/
 COPY packages/jobs/ packages/jobs/
 COPY packages/runner/ packages/runner/
 COPY package*.json ./

diff --git a/packages/jobs/lib/activities.ts b/packages/jobs/lib/activities.ts
@@ -20,12 +20,14 @@ import {
     LogTypes,
     isInitialSyncStillRunning,
     getSyncByIdAndName,
-    logger,
     getLastSyncDate
 } from '@nangohq/shared';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
 import integrationService from './integration.service.js';
 import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs, WebhookArgs } from './models/worker';
 
+const logger = getLogger('Jobs');
+
 export async function routeSync(args: InitialSyncArgs): Promise<boolean | object | null> {
     const { syncId, syncJobId, syncName, nangoConnection, debug } = args;
     let environmentId = nangoConnection?.environment_id;

diff --git a/packages/jobs/lib/app.ts b/packages/jobs/lib/app.ts
@@ -4,12 +4,14 @@ import { server } from './server.js';
 import { cronAutoIdleDemo } from './crons/autoIdleDemo.js';
 import { deleteOldActivityLogs } from './crons/deleteOldActivities.js';
 import { deleteSyncsData } from './crons/deleteSyncsData.js';
-import { logger } from '@nangohq/shared';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
+
+const logger = getLogger('Jobs');
 
 try {
     const port = parseInt(process.env['NANGO_JOBS_PORT'] || '') || 3005;
     server.listen(port);
-    logger.info(`🚀 Jobs service ready at http://localhost:${port}`);
+    logger.info(`🚀 service ready at http://localhost:${port}`);
     const temporalNs = process.env['TEMPORAL_NAMESPACE'] || 'default';
     const temporal = new Temporal(temporalNs);
 
@@ -29,6 +31,6 @@ try {
         });
     });
 } catch (err) {
-    logger.error(`[JOBS]: ${JSON.stringify(err)}`);
+    logger.error(`${JSON.stringify(err)}`);
     process.exit(1);
 }
diff --git a/packages/jobs/lib/crons/autoIdleDemo.ts b/packages/jobs/lib/crons/autoIdleDemo.ts
@@ -11,11 +11,13 @@ import {
     updateScheduleStatus,
     isErr,
     findPausableDemoSyncs,
-    logger,
     SpanTypes
 } from '@nangohq/shared';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
 import tracer from 'dd-trace';
 
+const logger = getLogger('Jobs');
+
 export function cronAutoIdleDemo(): void {
     schedule('1 * * * *', () => {
         const span = tracer.startSpan(SpanTypes.JOBS_IDLE_DEMO);

diff --git a/packages/jobs/lib/crons/deleteOldActivities.ts b/packages/jobs/lib/crons/deleteOldActivities.ts
@@ -1,8 +1,11 @@
 import * as cron from 'node-cron';
-import { deleteLog, deleteLogsMessages, errorManager, ErrorSourceEnum, findOldActivities, logger, MetricTypes, telemetry } from '@nangohq/shared';
+import { deleteLog, deleteLogsMessages, errorManager, ErrorSourceEnum, findOldActivities, MetricTypes, telemetry } from '@nangohq/shared';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
 import tracer from 'dd-trace';
 import { setTimeout } from 'node:timers/promises';
 
+const logger = getLogger('Jobs');
+
 // Retention in days
 const retention = parseInt(process.env['NANGO_CLEAR_ACTIVITIES_RETENTION'] || '', 10) || 15;
 const limitLog = parseInt(process.env['NANGO_CLEAR_ACTIVITIES_LIMIT'] || '', 10) || 2000;

diff --git a/packages/jobs/lib/crons/deleteSyncsData.ts b/packages/jobs/lib/crons/deleteSyncsData.ts
@@ -2,7 +2,6 @@ import * as cron from 'node-cron';
 import {
     errorManager,
     ErrorSourceEnum,
-    logger,
     MetricTypes,
     softDeleteSchedules,
     telemetry,
@@ -11,8 +10,11 @@ import {
     db,
     findRecentlyDeletedSync
 } from '@nangohq/shared';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
 import tracer from 'dd-trace';
 
+const logger = getLogger('Jobs');
+
 const limitJobs = 1000;
 const limitSchedules = 1000;
 const limitRecords = 1000;

diff --git a/packages/jobs/lib/integration.service.ts b/packages/jobs/lib/integration.service.ts
@@ -1,16 +1,7 @@
 import type { Context } from '@temporalio/activity';
 import type { IntegrationServiceInterface, NangoIntegrationData, NangoProps, ServiceResponse } from '@nangohq/shared';
-import {
-    createActivityLogMessage,
-    localFileService,
-    remoteFileService,
-    isCloud,
-    isProd,
-    integrationFilesAreRemote,
-    NangoError,
-    formatScriptError,
-    isOk
-} from '@nangohq/shared';
+import { integrationFilesAreRemote, isCloud, isProd } from '@nangohq/utils/dist/environment/detection.js';
+import { createActivityLogMessage, localFileService, remoteFileService, NangoError, formatScriptError, isOk } from '@nangohq/shared';
 import type { Runner } from './runner/runner.js';
 import { getOrStartRunner, getRunnerId } from './runner/runner.js';
 import tracer from 'dd-trace';
@@ -82,7 +73,7 @@ class IntegrationService implements IntegrationServiceInterface {
             .setTag('syncName', syncName);
         try {
             const script: string | null =
-                (isCloud() || integrationFilesAreRemote()) && !optionalLoadLocation
+                (isCloud || integrationFilesAreRemote) && !optionalLoadLocation
                     ? await remoteFileService.getFile(integrationData.fileLocation as string, environmentId)
                     : localFileService.getIntegrationFile(syncName, optionalLoadLocation);
 
@@ -120,7 +111,7 @@ class IntegrationService implements IntegrationServiceInterface {
 
             const accountId = nangoProps.accountId;
             // a runner per account in prod only
-            const runnerId = isProd() ? getRunnerId(`${accountId}`) : getRunnerId('default');
+            const runnerId = isProd ? getRunnerId(`${accountId}`) : getRunnerId('default');
             // fallback to default runner if account runner isn't ready yet
             const runner = await getOrStartRunner(runnerId).catch(() => getOrStartRunner(getRunnerId('default')));
 

diff --git a/packages/jobs/lib/runner/local.runner.ts b/packages/jobs/lib/runner/local.runner.ts
@@ -3,7 +3,9 @@ import { RunnerType } from './runner.js';
 import type { ChildProcess } from 'child_process';
 import { execSync, spawn } from 'child_process';
 import { getRunnerClient } from '@nangohq/nango-runner';
-import { logger } from '@nangohq/shared';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
+
+const logger = getLogger('Jobs');
 
 export class LocalRunner implements Runner {
     public client: any;

diff --git a/packages/jobs/lib/runner/remote.runner.ts b/packages/jobs/lib/runner/remote.runner.ts
@@ -1,8 +1,10 @@
-import { logger } from '@nangohq/shared';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
 import type { Runner } from './runner.js';
 import { RunnerType } from './runner.js';
 import { getRunnerClient } from '@nangohq/nango-runner';
 
+const logger = getLogger('Jobs');
+
 export class RemoteRunner implements Runner {
     public client: any;
     public runnerType: RunnerType = RunnerType.Remote;

diff --git a/packages/jobs/lib/runner/render.runner.ts b/packages/jobs/lib/runner/render.runner.ts
@@ -2,7 +2,8 @@ import type { Runner } from './runner.js';
 import { RunnerType } from './runner.js';
 import type { ProxyAppRouter } from '@nangohq/nango-runner';
 import { getRunnerClient } from '@nangohq/nango-runner';
-import { NodeEnv, getEnv, getPersistAPIUrl } from '@nangohq/shared';
+import { env } from '@nangohq/utils/dist/environment/detection.js';
+import { NodeEnv, getPersistAPIUrl } from '@nangohq/shared';
 import { RenderAPI } from './render.api.js';
 import tracer from 'dd-trace';
 
@@ -56,7 +57,7 @@ export class RenderRunner implements Runner {
             if (res.data.length > 0) {
                 svc = res.data[0].service;
             } else {
-                const imageTag = getEnv();
+                const imageTag = env;
                 const ownerId = process.env['RUNNER_OWNER_ID'];
                 if (!ownerId) {
                     throw new Error('RUNNER_OWNER_ID is not set');

diff --git a/packages/jobs/lib/runner/runner.ts b/packages/jobs/lib/runner/runner.ts
@@ -2,8 +2,12 @@ import type { KVStore } from '@nangohq/shared/lib/utils/kvstore/KVStore.js';
 import { LocalRunner } from './local.runner.js';
 import { RenderRunner } from './render.runner.js';
 import { RemoteRunner } from './remote.runner.js';
-import { getEnv, getRedisUrl, InMemoryKVStore, RedisKVStore, isEnterprise, logger } from '@nangohq/shared';
+import { isEnterprise, env } from '@nangohq/utils/dist/environment/detection.js';
+import { getRedisUrl, InMemoryKVStore, RedisKVStore } from '@nangohq/shared';
 import type { ProxyAppRouter } from '@nangohq/nango-runner';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
+
+const logger = getLogger('Runner');
 
 export enum RunnerType {
     Local = 'local',
@@ -21,7 +25,7 @@ export interface Runner {
 }
 
 export function getRunnerId(suffix: string): string {
-    return `${getEnv()}-runner-account-${suffix}`;
+    return `${env}-runner-account-${suffix}`;
 }
 
 export async function getOrStartRunner(runnerId: string): Promise<Runner> {
@@ -53,7 +57,7 @@ export async function getOrStartRunner(runnerId: string): Promise<Runner> {
     }
     const isRender = process.env['IS_RENDER'] === 'true';
     let runner: Runner;
-    if (isEnterprise()) {
+    if (isEnterprise) {
         runner = await RemoteRunner.getOrStart(runnerId);
     } else if (isRender) {
         runner = await RenderRunner.getOrStart(runnerId);

diff --git a/packages/jobs/lib/server.ts b/packages/jobs/lib/server.ts
@@ -3,7 +3,9 @@ import { createHTTPServer } from '@trpc/server/adapters/standalone';
 import superjson from 'superjson';
 import { z } from 'zod';
 import { suspendRunner } from './runner/runner.js';
-import { logger } from '@nangohq/shared';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
+
+const logger = getLogger('Jobs');
 
 export const t = initTRPC.create({
     transformer: superjson

diff --git a/packages/jobs/lib/temporal.ts b/packages/jobs/lib/temporal.ts
@@ -3,7 +3,11 @@ import fs from 'fs-extra';
 import * as dotenv from 'dotenv';
 import { createRequire } from 'module';
 import * as activities from './activities.js';
-import { SYNC_TASK_QUEUE, WEBHOOK_TASK_QUEUE, isProd, isEnterprise, logger } from '@nangohq/shared';
+import { SYNC_TASK_QUEUE, WEBHOOK_TASK_QUEUE } from '@nangohq/shared';
+import { isProd, isEnterprise } from '@nangohq/utils/dist/environment/detection.js';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
+
+const logger = getLogger('Jobs.Temporal');
 
 const TEMPORAL_WORKER_MAX_CONCURRENCY = parseInt(process.env['TEMPORAL_WORKER_MAX_CONCURRENCY'] || '0') || 500;
 
@@ -26,7 +30,7 @@ export class Temporal {
         let crt: Buffer | null = null;
         let key: Buffer | null = null;
 
-        if (isProd() || isEnterprise()) {
+        if (isProd || isEnterprise) {
             crt = await fs.readFile(`/etc/secrets/${this.namespace}.crt`);
             key = await fs.readFile(`/etc/secrets/${this.namespace}.key`);
         }
@@ -35,7 +39,7 @@ export class Temporal {
             const connection = await NativeConnection.connect({
                 address: process.env['TEMPORAL_ADDRESS'] || 'localhost:7233',
                 tls:
-                    !isProd() && !isEnterprise()
+                    !isProd && !isEnterprise
                         ? false
                         : {
                               clientCertPair: {

diff --git a/packages/jobs/nodemon.json b/packages/jobs/nodemon.json
@@ -1,7 +1,7 @@
 {
-  "watch": ["lib", "../shared/lib", "../../.env"],
-  "ext": "ts,json",
-  "ignore": ["lib/**/*.spec.ts"],
-  "exec": "tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env",
-  "signal": "SIGTERM"
-}
+    "watch": ["lib", "../shared/dist", "../utils/dist", "../../.env"],
+    "ext": "ts,json",
+    "ignore": ["lib/**/*.spec.ts"],
+    "exec": "tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env",
+    "signal": "SIGTERM"
+}
diff --git a/packages/jobs/package.json b/packages/jobs/package.json
@@ -17,6 +17,7 @@
     "dependencies": {
         "@nangohq/nango-runner": "^1.0.0",
         "@nangohq/shared": "^0.39.8",
+        "@nangohq/utils": "file:../utils",
         "@temporalio/activity": "^1.9.1",
         "@temporalio/client": "^1.9.1",
         "@temporalio/worker": "^1.9.1",

diff --git a/packages/jobs/tsconfig.json b/packages/jobs/tsconfig.json
@@ -10,9 +10,10 @@
         },
         {
             "path": "../runner"
+        },
+        {
+            "path": "../utils"
         }
     ],
-    "include": [
-        "lib/**/*"
-    ]
+    "include": ["lib/**/*"]
 }
diff --git a/packages/persist/Dockerfile b/packages/persist/Dockerfile
@@ -11,6 +11,7 @@ WORKDIR /nango
 COPY packages/node-client/ packages/node-client/
 COPY packages/shared/ packages/shared/
 COPY packages/persist/ packages/persist/
+COPY packages/utils/ packages/utils/
 COPY package*.json ./
 
 RUN npm pkg delete scripts.prepare

diff --git a/packages/persist/lib/app.ts b/packages/persist/lib/app.ts
@@ -1,10 +1,13 @@
 import './tracer.js';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
 import { server } from './server.js';
 
+const logger = getLogger('Persist');
+
 try {
     const port = parseInt(process.env['NANGO_PERSIST_PORT'] || '') || 3007;
     server.listen(port, () => {
-        console.log(`🚀 Persist API ready at http://localhost:${port}`);
+        logger.info(`🚀 API ready at http://localhost:${port}`);
     });
 } catch (err) {
     console.error(`Persist API error: ${err}`);

diff --git a/packages/persist/lib/server.ts b/packages/persist/lib/server.ts
@@ -2,8 +2,11 @@ import express from 'express';
 import type { Request, Response, NextFunction } from 'express';
 import { validateRequest } from 'zod-express';
 import { z } from 'zod';
+import { getLogger } from '@nangohq/utils/dist/logger.js';
 import persistController from './controllers/persist.controller.js';
-import { logLevelValues, logger } from '@nangohq/shared';
+import { logLevelValues } from '@nangohq/shared';
+
+const logger = getLogger('Persist');
 
 export const server = express();
 server.use(express.json({ limit: '100mb' }));
@@ -12,14 +15,14 @@ server.use((req: Request, res: Response, next: NextFunction) => {
     const originalSend = res.send;
     res.send = function (body: any) {
         if (res.statusCode >= 400) {
-            logger.info(`[Persist] [Error] ${req.method} ${req.path} ${res.statusCode} '${JSON.stringify(body)}'`);
+            logger.info(`[Error] ${req.method} ${req.path} ${res.statusCode} '${JSON.stringify(body)}'`);
         }
         originalSend.call(this, body) as any;
         return this;
     };
     next();
     if (res.statusCode < 400) {
-        logger.info(`[Persist] ${req.method} ${req.path} ${res.statusCode}`);
+        logger.info(`${req.method} ${req.path} ${res.statusCode}`);
     }
 });
 

diff --git a/packages/persist/nodemon.json b/packages/persist/nodemon.json
@@ -1,6 +1,6 @@
 {
-  "watch": ["lib", "../shared/lib", "../../.env"],
-  "ext": "ts,json",
-  "ignore": ["lib/**/*.test.ts"],
-  "exec": "tsc && tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env"
-}
+    "watch": ["lib", "../shared/dist", "../utils/dist", "../../.env"],
+    "ext": "ts,json",
+    "ignore": ["lib/**/*.test.ts"],
+    "exec": "tsc && tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env"
+}
diff --git a/packages/persist/package.json b/packages/persist/package.json
@@ -18,6 +18,7 @@
     },
     "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY",
     "dependencies": {
+        "@nangohq/utils": "file:../utils",
         "@nangohq/shared": "^0.39.8",
         "dd-trace": "5.2.0",
         "express": "^4.18.2",

diff --git a/packages/persist/tsconfig.json b/packages/persist/tsconfig.json
@@ -4,6 +4,6 @@
         "rootDir": "lib",
         "outDir": "dist"
     },
-    "references": [{ "path": "../shared" }],
+    "references": [{ "path": "../shared" }, { "path": "../utils" }],
     "include": ["lib/**/*"]
-}
+}
diff --git a/packages/runner/Dockerfile b/packages/runner/Dockerfile
@@ -10,6 +10,7 @@ WORKDIR /nango
 
 COPY packages/node-client/ packages/node-client/
 COPY packages/shared/ packages/shared/
+COPY packages/utils/ packages/utils/
 COPY packages/runner/ packages/runner/
 COPY package*.json ./