diff --git a/.env.example b/.env.example index 021e720735..8537ed2099 100644 --- a/.env.example +++ b/.env.example @@ -74,6 +74,9 @@ DEFAULT_GITHUB_CLIENT_SECRET= WORKOS_API_KEY= WORKOS_CLIENT_ID= +# Google Cloud Configuration +GOOGLE_APPLICATION_CREDENTIALS= + # Encryption key for secrets / records stored in database NANGO_ENCRYPTION_KEY= diff --git a/Dockerfile b/Dockerfile index dde677e3d3..f55b6f8bc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ COPY packages/runner/package.json ./packages/runner/package.json COPY packages/server/package.json ./packages/server/package.json COPY packages/shared/package.json ./packages/shared/package.json COPY packages/webapp/package.json ./packages/webapp/package.json +COPY packages/data-ingestion/package.json ./packages/data-ingestion/package.json COPY package*.json ./ # Install every dependencies diff --git a/package-lock.json b/package-lock.json index 2ad68d10eb..5ae669a396 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,8 @@ "packages/persist", "packages/jobs", "packages/webapp", - "packages/utils" + "packages/utils", + "packages/data-ingestion" ], "dependencies": { "@babel/parser": "^7.22.5", @@ -5682,6 +5683,94 @@ "react": ">=16.13.0" } }, + "node_modules/@google-cloud/bigquery": { + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.5.1.tgz", + "integrity": "sha512-ocye5Bt2eNQMoLKy814TVTp9XrXLoyD18mGwsjmZR3mEHv5m9oxycjG4P77c+hbN9YcKxg+EOtAclULKB9pOVg==", + "dependencies": { + "@google-cloud/common": "^5.0.0", + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/precise-date": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/bigquery/node_modules/big.js": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.1.tgz", + "integrity": "sha512-bCtHMwL9LeDIozFn+oNhhFoq+yQ3BNdnsLSASUxLciOb1vgvpHsIO1dsENiGMgbb4SkP5TrzWzRiLddn8ahVOQ==", + "engines": { + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" + } + }, + "node_modules/@google-cloud/common": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.1.tgz", + "integrity": "sha512-7NBC5vD0au75nkctVs2vEGpdUPFs1BaHTMpeI+RVEgQSMe5/wEU6dx9p0fmZA0bj4HgdpobMKeegOcLUiEoxng==", + "dependencies": { + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^9.0.0", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.0.tgz", + "integrity": "sha512-87aeg6QQcEPxGCOthnpUjvw4xAZ57G7pL8FS0C4e/81fr3FjkpUpibf1s2v5XGyGhUVGF4Jfg7yEcxqn2iUw1w==", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/precise-date": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-4.0.0.tgz", + "integrity": "sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "engines": { + "node": ">=14" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.7.3", "license": "Apache-2.0", @@ -6578,6 +6667,10 @@ "react": ">=16.8.0" } }, + "node_modules/@nangohq/data-ingestion": { + "resolved": "packages/data-ingestion", + "link": true + }, "node_modules/@nangohq/frontend": { "resolved": "packages/frontend", "link": true @@ -8293,6 +8386,11 @@ "@types/node": "*" } }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" + }, "node_modules/@types/chai": { "version": "4.3.11", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.11.tgz", @@ -8747,6 +8845,30 @@ "@types/node": "*" } }, + "node_modules/@types/request": { + "version": "2.48.12", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", + "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + } + }, + "node_modules/@types/request/node_modules/form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, "node_modules/@types/resolve": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", @@ -8827,6 +8949,11 @@ "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" + }, "node_modules/@types/triple-beam": { "version": "1.3.2", "license": "MIT" @@ -9855,7 +9982,6 @@ }, "node_modules/agent-base": { "version": "6.0.2", - "dev": true, "license": "MIT", "dependencies": { "debug": "4" @@ -10301,6 +10427,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "engines": { + "node": ">=8" + } + }, "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", @@ -10772,6 +10906,14 @@ "node": "*" } }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "license": "MIT", @@ -13468,6 +13610,17 @@ "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", "dev": true }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "license": "MIT" @@ -13562,7 +13715,6 @@ }, "node_modules/end-of-stream": { "version": "1.4.4", - "dev": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -13580,6 +13732,11 @@ "node": ">=10.13.0" } }, + "node_modules/ent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", + "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" + }, "node_modules/entities": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", @@ -15165,6 +15322,11 @@ "version": "2.0.0", "license": "MIT" }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "node_modules/external-editor": { "version": "3.1.0", "license": "MIT", @@ -15848,6 +16010,55 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.3.0.tgz", + "integrity": "sha512-p+ggrQw3fBwH2F5N/PAI4k/G/y1art5OxKpb2J2chwNNHM4hHuAOtivjPuirMF4KNKwTTUal/lPfL2+7h2mEcg==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/gaxios/node_modules/https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/generic-pool": { "version": "3.9.0", "license": "MIT", @@ -16114,6 +16325,41 @@ "delegate": "^3.1.2" } }, + "node_modules/google-auth-library": { + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.7.0.tgz", + "integrity": "sha512-I/AvzBiUXDzLOy4iIZ2W+Zq33W4lcukQv1nl7C8WUA6SQwyQwUwu3waNmWNAvzds//FG8SZ+DnKnW/2k6mQS8A==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-auth-library/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/google-auth-library/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -16135,6 +16381,37 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/gtoken/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/gtoken/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/gzip-size": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", @@ -16573,7 +16850,6 @@ }, "node_modules/https-proxy-agent": { "version": "5.0.1", - "dev": true, "license": "MIT", "dependencies": { "agent-base": "6", @@ -16823,6 +17099,14 @@ "node": ">= 0.10" } }, + "node_modules/is": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", + "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==", + "engines": { + "node": "*" + } + }, "node_modules/is-alphabetical": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz", @@ -18972,6 +19256,14 @@ "node": ">=4" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "license": "MIT" @@ -23805,6 +24097,19 @@ "node": ">= 4" } }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/reusify": { "version": "1.0.4", "dev": true, @@ -24842,6 +25147,19 @@ "node": ">= 0.4" } }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -25077,6 +25395,11 @@ "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-1.2.5.tgz", "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==" }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" + }, "node_modules/style-loader": { "version": "3.3.4", "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-3.3.4.tgz", @@ -25604,6 +25927,42 @@ "node": ">=8.0.0" } }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "engines": { + "node": ">= 10" + } + }, + "node_modules/teeny-request/node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/temp-dir": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-2.0.0.tgz", @@ -29208,6 +29567,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "packages/data-ingestion": { + "name": "@nangohq/data-ingestion", + "version": "1.0.0", + "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", + "dependencies": { + "@google-cloud/bigquery": "7.5.1", + "@nangohq/utils": "file:../utils" + }, + "devDependencies": {} + }, "packages/frontend": { "name": "@nangohq/frontend", "version": "0.39.13", @@ -29227,6 +29596,7 @@ "name": "@nangohq/nango-jobs", "version": "1.0.0", "dependencies": { + "@nangohq/data-ingestion": "file:../data-ingestion", "@nangohq/nango-runner": "^1.0.0", "@nangohq/shared": "^0.39.13", "@nangohq/utils": "file:../utils", diff --git a/package.json b/package.json index e474d2827b..194e7fa800 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "packages/persist", "packages/jobs", "packages/webapp", - "packages/utils" + "packages/utils", + "packages/data-ingestion" ], "scripts": { "create:migration": "cd packages/shared/lib/db && knex migrate:make $1 --esm --knexfile ./knexfile.cjs", diff --git a/packages/data-ingestion/.gitignore b/packages/data-ingestion/.gitignore new file mode 100644 index 0000000000..8c3aba0aad --- /dev/null +++ b/packages/data-ingestion/.gitignore @@ -0,0 +1,3 @@ +tsconfig.tsbuildinfo +dist/* +node_modules diff --git a/packages/data-ingestion/lib/index.ts b/packages/data-ingestion/lib/index.ts new file mode 100644 index 0000000000..85f7951ecf --- /dev/null +++ b/packages/data-ingestion/lib/index.ts @@ -0,0 +1,101 @@ +import { BigQuery } from '@google-cloud/bigquery'; +import type { BigQuery as BigQueryType } from '@google-cloud/bigquery'; +import { getLogger } from '@nangohq/utils/dist/logger.js'; +import { isCloud } from '@nangohq/utils/dist/environment/detection.js'; + +const logger = getLogger('BigQueryClient'); + +interface RunScriptRow { + executionType: string; + internalConnectionId: number | undefined; + connectionId: string; + accountId: number | undefined; + scriptName: string; + scriptType: string; + environmentId: number; + providerConfigKey: string; + status: string; + syncId: string; + content: string; + runTimeInSeconds: number; + createdAt: number; +} + +class BigQueryClient { + private client: BigQuery; + private datasetName: string; + private tableName: string; + + constructor({ datasetName, tableName }: { datasetName: string; tableName: string }) { + this.client = new BigQuery(); + this.tableName = tableName; + this.datasetName = datasetName; + } + + static async createInstance({ datasetName, tableName }: { datasetName?: string; tableName: string }) { + const instance = new BigQueryClient({ + datasetName: datasetName || 'raw', + tableName + }); + await instance.initialize(); + return instance; + } + + private async initialize() { + try { + if (isCloud) { + await this.createDataSet(); + await this.createTable(); + } + } catch (e) { + logger.error('Error initializing', e); + } + } + + private async createDataSet() { + const dataset = this.client.dataset(this.datasetName); + const [exists] = await dataset.exists(); + if (!exists) { + await this.client.createDataset(this.datasetName); + } + } + + private async createTable() { + const table = this.client.dataset(this.datasetName).table(this.tableName); + const [exists] = await table.exists(); + if (!exists) { + await table.create({ + schema: { + fields: [ + { name: 'executionType', type: 'STRING' }, + { name: 'internalConnectionId', type: 'INTEGER' }, + { name: 'connectionId', type: 'STRING' }, + { name: 'accountId', type: 'INTEGER' }, + { name: 'scriptName', type: 'STRING' }, + { name: 'scriptType', type: 'STRING' }, + { name: 'environmentId', type: 'INTEGER' }, + { name: 'providerConfigKey', type: 'STRING' }, + { name: 'status', type: 'STRING' }, + { name: 'syncId', type: 'STRING' }, + { name: 'content', type: 'STRING' }, + { name: 'runTimeInSeconds', type: 'FLOAT' }, + { name: 'createdAt', type: 'INTEGER' } + ] + } + }); + } + } + + public async insert(data: RunScriptRow, tableName?: string) { + const table = tableName || this.tableName; + try { + if (isCloud) { + await this.client.dataset(this.datasetName).table(table).insert(data); + } + } catch (e) { + logger.error('Error inserting into BigQuery', e); + } + } +} + +export { BigQueryClient, BigQueryType }; diff --git a/packages/data-ingestion/package.json b/packages/data-ingestion/package.json new file mode 100644 index 0000000000..f97596b8d8 --- /dev/null +++ b/packages/data-ingestion/package.json @@ -0,0 +1,23 @@ +{ + "name": "@nangohq/data-ingestion", + "version": "1.0.0", + "description": "Package to ingest Nango data for analytics", + "type": "module", + "main": "dist/index.js", + "typings": "dist/index.d.ts", + "scripts": { + "build": "rimraf ./dist && tsc" + }, + "keywords": [], + "repository": { + "type": "git", + "url": "git+https://github.com/NangoHQ/nango.git", + "directory": "packages/utils" + }, + "license": "SEE LICENSE IN LICENSE FILE IN GIT REPOSITORY", + "dependencies": { + "@google-cloud/bigquery": "7.5.1", + "@nangohq/utils": "file:../utils" + }, + "devDependencies": {} +} diff --git a/packages/data-ingestion/tsconfig.json b/packages/data-ingestion/tsconfig.json new file mode 100644 index 0000000000..c903a0a19b --- /dev/null +++ b/packages/data-ingestion/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "rootDir": "lib", + "outDir": "dist" + }, + "include": ["lib/**/*"], + "references": [ + { + "path": "../utils" + } + ] +} diff --git a/packages/jobs/Dockerfile b/packages/jobs/Dockerfile index 21364186de..f4f973270c 100644 --- a/packages/jobs/Dockerfile +++ b/packages/jobs/Dockerfile @@ -11,6 +11,7 @@ WORKDIR /nango COPY packages/node-client/ packages/node-client/ COPY packages/shared/ packages/shared/ COPY packages/utils/ packages/utils/ +COPY packages/data-ingestion/ packages/data-ingestion/ COPY packages/jobs/ packages/jobs/ COPY packages/runner/ packages/runner/ COPY package*.json ./ diff --git a/packages/jobs/lib/activities.ts b/packages/jobs/lib/activities.ts index 1622b66151..3e19e55807 100644 --- a/packages/jobs/lib/activities.ts +++ b/packages/jobs/lib/activities.ts @@ -23,11 +23,18 @@ import { getLastSyncDate } from '@nangohq/shared'; import { getLogger } from '@nangohq/utils/dist/logger.js'; +import { BigQueryClient } from '@nangohq/data-ingestion/dist/index.js'; +import { env } from '@nangohq/utils/dist/environment/detection.js'; import integrationService from './integration.service.js'; import type { ContinuousSyncArgs, InitialSyncArgs, ActionArgs, WebhookArgs } from './models/worker'; const logger = getLogger('Jobs'); +const bigQueryClient = await BigQueryClient.createInstance({ + datasetName: 'raw', + tableName: `${env}_script_runs` +}); + export async function routeSync(args: InitialSyncArgs): Promise { const { syncId, syncJobId, syncName, nangoConnection, debug } = args; let environmentId = nangoConnection?.environment_id; @@ -53,6 +60,7 @@ export async function runAction(args: ActionArgs): Promise { const context: Context = Context.current(); const syncRun = new syncRunService({ + bigQueryClient, integrationService, writeToDb: true, nangoConnection, @@ -225,6 +233,7 @@ export async function syncProvider( } const syncRun = new syncRunService({ + bigQueryClient, integrationService, writeToDb: true, syncId, @@ -312,6 +321,7 @@ export async function runWebhook(args: WebhookArgs): Promise { ); const syncRun = new syncRunService({ + bigQueryClient, integrationService, writeToDb: true, nangoConnection, @@ -404,6 +414,7 @@ export async function cancelActivity(workflowArguments: InitialSyncArgs | Contin const syncType = lastSyncDate ? SyncType.INCREMENTAL : SyncType.INITIAL; const syncRun = new syncRunService({ + bigQueryClient, integrationService, writeToDb: true, syncId, diff --git a/packages/jobs/nodemon.json b/packages/jobs/nodemon.json index bcc60becec..ccb17f5064 100644 --- a/packages/jobs/nodemon.json +++ b/packages/jobs/nodemon.json @@ -1,6 +1,6 @@ { - "watch": ["lib", "../shared/dist", "../utils/dist", "../../.env"], - "ext": "ts,json", + "watch": ["lib", "../shared/dist", "../utils/dist", "../data-ingestion/dist", "../../.env"], + "ext": "js,ts,json", "ignore": ["lib/**/*.spec.ts"], "exec": "tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env", "signal": "SIGTERM" diff --git a/packages/jobs/package.json b/packages/jobs/package.json index 7598b230ec..8b96247341 100644 --- a/packages/jobs/package.json +++ b/packages/jobs/package.json @@ -15,6 +15,7 @@ "directory": "packages/jobs" }, "dependencies": { + "@nangohq/data-ingestion": "file:../data-ingestion", "@nangohq/nango-runner": "^1.0.0", "@nangohq/shared": "^0.39.13", "@nangohq/utils": "file:../utils", diff --git a/packages/jobs/tsconfig.json b/packages/jobs/tsconfig.json index 3072de4f5e..6914d3cd65 100644 --- a/packages/jobs/tsconfig.json +++ b/packages/jobs/tsconfig.json @@ -13,6 +13,9 @@ }, { "path": "../utils" + }, + { + "path": "../data-ingestion" } ], "include": ["lib/**/*"] diff --git a/packages/persist/nodemon.json b/packages/persist/nodemon.json index c71f2b06ec..adef9585e5 100644 --- a/packages/persist/nodemon.json +++ b/packages/persist/nodemon.json @@ -1,6 +1,6 @@ { "watch": ["lib", "../shared/dist", "../utils/dist", "../../.env"], - "ext": "ts,json", + "ext": "js,ts,json", "ignore": ["lib/**/*.test.ts"], "exec": "tsc && tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env" } diff --git a/packages/runner/nodemon.json b/packages/runner/nodemon.json index c71f2b06ec..adef9585e5 100644 --- a/packages/runner/nodemon.json +++ b/packages/runner/nodemon.json @@ -1,6 +1,6 @@ { "watch": ["lib", "../shared/dist", "../utils/dist", "../../.env"], - "ext": "ts,json", + "ext": "js,ts,json", "ignore": ["lib/**/*.test.ts"], "exec": "tsc && tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env" } diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json index 5292530ee5..1461aa051b 100644 --- a/packages/server/nodemon.json +++ b/packages/server/nodemon.json @@ -1,6 +1,6 @@ { "watch": ["lib", "../shared/dist", "../utils/dist", "../../.env", "../shared/providers.yaml"], - "ext": "ts,json", + "ext": "js,ts,json", "ignore": ["src/**/*.spec.ts"], "exec": "tsx -r dotenv/config lib/server.ts Dotenv_config_path=./../../.env" } diff --git a/packages/shared/lib/services/sync/config/config.service.ts b/packages/shared/lib/services/sync/config/config.service.ts index 0501c2f369..9387a9589a 100644 --- a/packages/shared/lib/services/sync/config/config.service.ts +++ b/packages/shared/lib/services/sync/config/config.service.ts @@ -738,6 +738,7 @@ export async function getConfigWithEndpointsByProviderConfigKey(environment_id: `${TABLE}.sync_type`, `${TABLE}.track_deletes`, `${TABLE}.auto_start`, + `${TABLE}.webhook_subscriptions`, '_nango_configs.unique_key', '_nango_configs.provider', db.knex.raw( diff --git a/packages/shared/lib/services/sync/run.service.integration.test.ts b/packages/shared/lib/services/sync/run.service.integration.test.ts index ea9bf96f9e..5909ace6a5 100644 --- a/packages/shared/lib/services/sync/run.service.integration.test.ts +++ b/packages/shared/lib/services/sync/run.service.integration.test.ts @@ -345,7 +345,7 @@ const runJob = async ( }; await jobService.updateSyncJobResult(syncJob.id, updatedResults, model); // finish the sync - await syncRun.finishSync([model], new Date(), `v1`, 10, trackDeletes); + await syncRun.finishFlow([model], new Date(), `v1`, 10, trackDeletes); const syncJobResult = await jobService.getLatestSyncJob(sync.id); return { diff --git a/packages/shared/lib/services/sync/run.service.ts b/packages/shared/lib/services/sync/run.service.ts index c2da7d2b2a..60d41de711 100644 --- a/packages/shared/lib/services/sync/run.service.ts +++ b/packages/shared/lib/services/sync/run.service.ts @@ -24,7 +24,28 @@ import type { Environment } from '../../models/Environment'; import type { Metadata } from '../../models/Connection'; import * as recordsService from './data/records.service.js'; +interface BigQueryClientInterface { + insert(row: RunScriptRow): void; +} + +interface RunScriptRow { + executionType: string; + internalConnectionId: number | undefined; + connectionId: string; + accountId: number | undefined; + scriptName: string; + scriptType: string; + environmentId: number; + providerConfigKey: string; + status: string; + syncId: string; + content: string; + runTimeInSeconds: number; + createdAt: number; +} + interface SyncRunConfig { + bigQueryClient?: BigQueryClientInterface; integrationService: IntegrationServiceInterface; writeToDb: boolean; isAction?: boolean; @@ -50,6 +71,7 @@ interface SyncRunConfig { } export default class SyncRun { + bigQueryClient?: BigQueryClientInterface; integrationService: IntegrationServiceInterface; writeToDb: boolean; isAction: boolean; @@ -77,6 +99,9 @@ export default class SyncRun { constructor(config: SyncRunConfig) { this.integrationService = config.integrationService; + if (config.bigQueryClient) { + this.bigQueryClient = config.bigQueryClient; + } this.writeToDb = config.writeToDb; this.isAction = config.isAction || false; this.isWebhook = config.isWebhook || false; @@ -159,7 +184,7 @@ export default class SyncRun { if (!nangoConfig) { const message = `No ${this.isAction ? 'action' : 'sync'} configuration was found for ${this.syncName}.`; if (this.activityLogId) { - await this.reportFailureForResults(message); + await this.reportFailureForResults({ content: message, runTime: 0 }); } else { console.error(message); } @@ -188,7 +213,7 @@ export default class SyncRun { if (!environment && !bypassEnvironment) { const message = `No environment was found for ${this.nangoConnection.environment_id}. The sync cannot continue without a valid environment`; - await this.reportFailureForResults(message); + await this.reportFailureForResults({ content: message, runTime: 0 }); const errorType = this.determineErrorType(); return { success: false, error: new NangoError(errorType, message, 404), response: false }; } @@ -236,7 +261,7 @@ export default class SyncRun { ); if (!integrationFileResult) { const message = `Integration was attempted to run for ${this.syncName} but no integration file was found at ${integrationFilePath}.`; - await this.reportFailureForResults(message); + await this.reportFailureForResults({ content: message, runTime: 0 }); const errorType = this.determineErrorType(); @@ -260,7 +285,7 @@ export default class SyncRun { if (isJsOrTsType(configInput as unknown as string)) { if (typeof this.input !== (configInput as unknown as string)) { const message = `The input provided of ${this.input} for ${this.syncName} is not of type ${configInput}`; - await this.reportFailureForResults(message); + await this.reportFailureForResults({ content: message, runTime: 0 }); return { success: false, error: new NangoError('action_script_failure', message, 500), response: false }; } @@ -271,6 +296,10 @@ export default class SyncRun { } } + if (!this.nangoConnection.account_id && environment?.account_id !== null && environment?.account_id !== undefined) { + this.nangoConnection.account_id = environment.account_id; + } + const nangoProps = { host: optionalHost || getApiUrl(), accountId: environment?.account_id as number, @@ -337,10 +366,11 @@ export default class SyncRun { syncData?.version ? ` version: ${syncData.version}` : '' }`; + const runTime = (Date.now() - startTime) / 1000; if (error.type === 'script_cancelled') { - await this.reportFailureForResults(error.message); + await this.reportFailureForResults({ content: error.message, runTime }); } else { - await this.reportFailureForResults(message); + await this.reportFailureForResults({ content: message, runTime }); } return { success: false, error, response: false }; @@ -350,6 +380,8 @@ export default class SyncRun { return userDefinedResults; } + const totalRunTime = (Date.now() - startTime) / 1000; + if (this.isAction) { const content = `${this.syncName} action was run successfully and results are being sent synchronously.`; @@ -372,21 +404,23 @@ export default class SyncRun { this.provider as string ); + await this.finishFlow(models, syncStartDate, syncData.version as string, totalRunTime, trackDeletes); + return { success: true, error: null, response: userDefinedResults }; } - const totalRunTime = (Date.now() - startTime) / 1000; - await this.finishSync(models, syncStartDate, syncData.version as string, totalRunTime, trackDeletes); + await this.finishFlow(models, syncStartDate, syncData.version as string, totalRunTime, trackDeletes); return { success: true, error: null, response: true }; } catch (e) { result = false; const errorMessage = JSON.stringify(e, ['message', 'name'], 2); - await this.reportFailureForResults( - `The ${this.syncType} "${this.syncName}"${ + await this.reportFailureForResults({ + content: `The ${this.syncType} "${this.syncName}"${ syncData?.version ? ` version: ${syncData?.version}` : '' - } sync did not complete successfully and has the following error: ${errorMessage}` - ); + } sync did not complete successfully and has the following error: ${errorMessage}`, + runTime: (Date.now() - startTime) / 1000 + }); const errorType = this.determineErrorType(); @@ -402,7 +436,7 @@ export default class SyncRun { return { success: true, error: null, response: result }; } - async finishSync(models: string[], syncStartDate: Date, version: string, totalRunTime: number, trackDeletes?: boolean): Promise { + async finishFlow(models: string[], syncStartDate: Date, version: string, totalRunTime: number, trackDeletes?: boolean): Promise { let i = 0; for (const model of models) { let deletedKeys: string[] = []; @@ -418,6 +452,25 @@ export default class SyncRun { await this.reportResults(model, { addedKeys: [], updatedKeys: [], deletedKeys }, i, models.length, syncStartDate, version, totalRunTime); i++; } + + // we only want to report to bigquery once if it is a multi model sync + if (this.bigQueryClient) { + void this.bigQueryClient.insert({ + executionType: this.determineExecutionType(), + connectionId: this.nangoConnection.connection_id, + internalConnectionId: this.nangoConnection.id, + accountId: this.nangoConnection.account_id, + scriptName: this.syncName, + scriptType: this.syncType, + environmentId: this.nangoConnection.environment_id, + providerConfigKey: this.nangoConnection.provider_config_key, + status: 'success', + syncId: this.syncId as string, + content: `The ${this.syncType} "${this.syncName}" ${this.determineExecutionType()} has been completed successfully.`, + runTimeInSeconds: totalRunTime, + createdAt: Date.now() + }); + } } async reportResults( @@ -464,7 +517,10 @@ export default class SyncRun { const syncResult: SyncJob = await updateSyncJobResult(this.syncJobId, updatedResults, model); if (!syncResult) { - await this.reportFailureForResults(`The sync job ${this.syncJobId} could not be updated with the results for the model ${model}.`); + await this.reportFailureForResults({ + content: `The sync job ${this.syncJobId} could not be updated with the results for the model ${model}.`, + runTime: totalRunTime + }); return; } @@ -560,11 +616,29 @@ export default class SyncRun { ); } - async reportFailureForResults(content: string) { + async reportFailureForResults({ content, runTime }: { content: string; runTime: number }) { if (!this.writeToDb) { return; } + if (this.bigQueryClient) { + void this.bigQueryClient.insert({ + executionType: this.determineExecutionType(), + connectionId: this.nangoConnection.connection_id, + internalConnectionId: this.nangoConnection.id, + accountId: this.nangoConnection.account_id, + scriptName: this.syncName, + scriptType: this.syncType, + environmentId: this.nangoConnection.environment_id, + providerConfigKey: this.nangoConnection.provider_config_key, + status: 'failed', + syncId: this.syncId as string, + content, + runTimeInSeconds: runTime, + createdAt: Date.now() + }); + } + if (!this.isWebhook) { try { await slackNotificationService.reportFailure( @@ -641,13 +715,17 @@ export default class SyncRun { ); } - private determineErrorType(): string { + private determineExecutionType(): string { if (this.isAction) { - return 'action_script_failure'; + return 'action'; } else if (this.isWebhook) { - return 'webhook_script_failure'; + return 'webhook'; } else { - return 'sync_script_failure'; + return 'sync'; } } + + private determineErrorType(): string { + return this.determineExecutionType() + '_script_failure'; + } } diff --git a/tsconfig.build.json b/tsconfig.build.json index eaf0561edc..2a548ec6f7 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -30,6 +30,9 @@ }, { "path": "packages/webapp" + }, + { + "path": "packages/data-ingestion" } ] }