From 022dcda8a09d4b332e1dd34727e7f6a10f6d8b4a Mon Sep 17 00:00:00 2001 From: Khaliq Date: Mon, 25 Mar 2024 13:50:39 +0200 Subject: [PATCH 01/12] [nan-591] insert into bigQuery --- package-lock.json | 361 +++++++++++++++++- .../shared/lib/clients/big-query.client.ts | 68 ++++ .../shared/lib/services/sync/run.service.ts | 69 +++- packages/shared/package.json | 1 + 4 files changed, 483 insertions(+), 16 deletions(-) create mode 100644 packages/shared/lib/clients/big-query.client.ts diff --git a/package-lock.json b/package-lock.json index 6551e6e54a..229f320573 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4842,6 +4842,94 @@ "react": ">=16.13.0" } }, + "node_modules/@google-cloud/bigquery": { + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/@google-cloud/bigquery/-/bigquery-7.5.1.tgz", + "integrity": "sha512-ocye5Bt2eNQMoLKy814TVTp9XrXLoyD18mGwsjmZR3mEHv5m9oxycjG4P77c+hbN9YcKxg+EOtAclULKB9pOVg==", + "dependencies": { + "@google-cloud/common": "^5.0.0", + "@google-cloud/paginator": "^5.0.0", + "@google-cloud/precise-date": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "big.js": "^6.0.0", + "duplexify": "^4.0.0", + "extend": "^3.0.2", + "is": "^3.3.0", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/bigquery/node_modules/big.js": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-6.2.1.tgz", + "integrity": "sha512-bCtHMwL9LeDIozFn+oNhhFoq+yQ3BNdnsLSASUxLciOb1vgvpHsIO1dsENiGMgbb4SkP5TrzWzRiLddn8ahVOQ==", + "engines": { + "node": "*" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/bigjs" + } + }, + "node_modules/@google-cloud/common": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@google-cloud/common/-/common-5.0.1.tgz", + "integrity": "sha512-7NBC5vD0au75nkctVs2vEGpdUPFs1BaHTMpeI+RVEgQSMe5/wEU6dx9p0fmZA0bj4HgdpobMKeegOcLUiEoxng==", + "dependencies": { + "@google-cloud/projectify": "^4.0.0", + "@google-cloud/promisify": "^4.0.0", + "arrify": "^2.0.1", + "duplexify": "^4.1.1", + "ent": "^2.2.0", + "extend": "^3.0.2", + "google-auth-library": "^9.0.0", + "retry-request": "^7.0.0", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/paginator": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.0.tgz", + "integrity": "sha512-87aeg6QQcEPxGCOthnpUjvw4xAZ57G7pL8FS0C4e/81fr3FjkpUpibf1s2v5XGyGhUVGF4Jfg7yEcxqn2iUw1w==", + "dependencies": { + "arrify": "^2.0.0", + "extend": "^3.0.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/precise-date": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/precise-date/-/precise-date-4.0.0.tgz", + "integrity": "sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/projectify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", + "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@google-cloud/promisify": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", + "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", + "engines": { + "node": ">=14" + } + }, "node_modules/@grpc/grpc-js": { "version": "1.7.3", "license": "Apache-2.0", @@ -7421,6 +7509,11 @@ "@types/node": "*" } }, + "node_modules/@types/caseless": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", + "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" + }, "node_modules/@types/chai": { "version": "4.3.11", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.11.tgz", @@ -7875,6 +7968,30 @@ "@types/node": "*" } }, + "node_modules/@types/request": { + "version": "2.48.12", + "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", + "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", + "dependencies": { + "@types/caseless": "*", + "@types/node": "*", + "@types/tough-cookie": "*", + "form-data": "^2.5.0" + } + }, + "node_modules/@types/request/node_modules/form-data": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", + "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 0.12" + } + }, "node_modules/@types/resolve": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.17.1.tgz", @@ -7955,6 +8072,11 @@ "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", "dev": true }, + "node_modules/@types/tough-cookie": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", + "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" + }, "node_modules/@types/triple-beam": { "version": "1.3.2", "license": "MIT" @@ -8983,7 +9105,6 @@ }, "node_modules/agent-base": { "version": "6.0.2", - "dev": true, "license": "MIT", "dependencies": { "debug": "4" @@ -9429,6 +9550,14 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "engines": { + "node": ">=8" + } + }, "node_modules/asap": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", @@ -9900,6 +10029,14 @@ "node": "*" } }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "license": "MIT", @@ -12596,6 +12733,17 @@ "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", "dev": true }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "license": "MIT" @@ -12690,7 +12838,6 @@ }, "node_modules/end-of-stream": { "version": "1.4.4", - "dev": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -12708,6 +12855,11 @@ "node": ">=10.13.0" } }, + "node_modules/ent": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/ent/-/ent-2.2.0.tgz", + "integrity": "sha512-GHrMyVZQWvTIdDtpiEXdHZnFQKzeO09apj8Cbl4pKWy4i0Oprcq17usfDt5aO63swf0JOeMWjWQE/LzgSRuWpA==" + }, "node_modules/entities": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/entities/-/entities-2.2.0.tgz", @@ -14293,6 +14445,11 @@ "version": "2.0.0", "license": "MIT" }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "node_modules/external-editor": { "version": "3.1.0", "license": "MIT", @@ -14962,6 +15119,55 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gaxios": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.3.0.tgz", + "integrity": "sha512-p+ggrQw3fBwH2F5N/PAI4k/G/y1art5OxKpb2J2chwNNHM4hHuAOtivjPuirMF4KNKwTTUal/lPfL2+7h2mEcg==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.9" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/gaxios/node_modules/agent-base": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.0.tgz", + "integrity": "sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/gaxios/node_modules/https-proxy-agent": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", + "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/gcp-metadata": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", + "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", + "dependencies": { + "gaxios": "^6.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/generic-pool": { "version": "3.9.0", "license": "MIT", @@ -15228,6 +15434,41 @@ "delegate": "^3.1.2" } }, + "node_modules/google-auth-library": { + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.7.0.tgz", + "integrity": "sha512-I/AvzBiUXDzLOy4iIZ2W+Zq33W4lcukQv1nl7C8WUA6SQwyQwUwu3waNmWNAvzds//FG8SZ+DnKnW/2k6mQS8A==", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^6.1.1", + "gcp-metadata": "^6.1.0", + "gtoken": "^7.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/google-auth-library/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/google-auth-library/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -15249,6 +15490,37 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/gtoken": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", + "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", + "dependencies": { + "gaxios": "^6.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/gtoken/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/gtoken/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/gzip-size": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", @@ -15687,7 +15959,6 @@ }, "node_modules/https-proxy-agent": { "version": "5.0.1", - "dev": true, "license": "MIT", "dependencies": { "agent-base": "6", @@ -15937,6 +16208,14 @@ "node": ">= 0.10" } }, + "node_modules/is": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/is/-/is-3.3.0.tgz", + "integrity": "sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg==", + "engines": { + "node": "*" + } + }, "node_modules/is-alphabetical": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz", @@ -18086,6 +18365,14 @@ "node": ">=4" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "license": "MIT" @@ -22933,6 +23220,19 @@ "node": ">= 4" } }, + "node_modules/retry-request": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", + "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", + "dependencies": { + "@types/request": "^2.48.8", + "extend": "^3.0.2", + "teeny-request": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/reusify": { "version": "1.0.4", "dev": true, @@ -23945,6 +24245,19 @@ "node": ">= 0.4" } }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -24179,6 +24492,11 @@ "resolved": "https://registry.npmjs.org/stubborn-fs/-/stubborn-fs-1.2.5.tgz", "integrity": "sha512-H2N9c26eXjzL/S/K+i/RHHcFanE74dptvvjM8iwzwbVcWY/zjBbgRqF3K0DY4+OD+uTTASTBvDoxPDaPN02D7g==" }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" + }, "node_modules/style-loader": { "version": "3.3.4", "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-3.3.4.tgz", @@ -24706,6 +25024,42 @@ "node": ">=8.0.0" } }, + "node_modules/teeny-request": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", + "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", + "dependencies": { + "http-proxy-agent": "^5.0.0", + "https-proxy-agent": "^5.0.0", + "node-fetch": "^2.6.9", + "stream-events": "^1.0.5", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/teeny-request/node_modules/@tootallnate/once": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", + "engines": { + "node": ">= 10" + } + }, + "node_modules/teeny-request/node_modules/http-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", + "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", + "dependencies": { + "@tootallnate/once": "2", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/temp-dir": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-2.0.0.tgz", @@ -32167,6 +32521,7 @@ "dependencies": { "@aws-sdk/client-s3": "3.348.0", "@datadog/datadog-api-client": "^1.16.0", + "@google-cloud/bigquery": "^7.5.1", "@hapi/boom": "^10.0.1", "@nangohq/node": "^0.39.8", "@sentry/node": "^7.105.0", diff --git a/packages/shared/lib/clients/big-query.client.ts b/packages/shared/lib/clients/big-query.client.ts new file mode 100644 index 0000000000..49e70e8107 --- /dev/null +++ b/packages/shared/lib/clients/big-query.client.ts @@ -0,0 +1,68 @@ +import { BigQuery } from '@google-cloud/bigquery'; +import logger from '../logger/console.js'; +import { isCloud, getEnv } from '../utils/utils.js'; + +type Row = Record; + +class BigQueryClient { + private client: BigQuery; + private datasetName: string; + private tableName: string; + + constructor() { + this.client = new BigQuery(); + this.datasetName = 'usage'; + this.tableName = 'scriptRuns'; + } + + private async createDataSet() { + const dataset = this.client.dataset(this.datasetName); + const [exists] = await dataset.exists(); + if (!exists) { + await this.client.createDataset(this.datasetName); + } + } + + private async createTable() { + const table = this.client.dataset(this.datasetName).table(`${getEnv()}_${this.tableName}`); + const [exists] = await table.exists(); + if (!exists) { + await table.create({ + schema: { + fields: [ + { name: 'executionType', type: 'STRING' }, + { name: 'connectionId', type: 'STRING' }, + { name: 'accountId', type: 'INTEGER' }, + { name: 'scriptName', type: 'STRING' }, + { name: 'scriptType', type: 'STRING' }, + { name: 'environmentId', type: 'INTEGER' }, + { name: 'providerConfigKey', type: 'STRING' }, + { name: 'status', type: 'STRING' }, + { name: 'syncId', type: 'STRING' }, + { name: 'content', type: 'STRING' }, + { name: 'runTime', type: 'INTEGER' }, + { name: 'timestamp', type: 'INTEGER' } + ] + } + }); + } + } + + private async initializeTable() { + await this.createDataSet(); + await this.createTable(); + } + + public async insert(data: Row) { + try { + if (isCloud()) { + await this.initializeTable(); + await this.client.dataset(this.datasetName).table(`${getEnv()}_${this.tableName}`).insert(data); + } + } catch (e) { + logger.error('Error inserting into BigQuery', e); + } + } +} + +export default new BigQueryClient(); diff --git a/packages/shared/lib/services/sync/run.service.ts b/packages/shared/lib/services/sync/run.service.ts index 8f9930ce21..90f6da9f17 100644 --- a/packages/shared/lib/services/sync/run.service.ts +++ b/packages/shared/lib/services/sync/run.service.ts @@ -8,6 +8,7 @@ import { createActivityLogMessage, createActivityLogMessageAndEnd, updateSuccess import { addSyncConfigToJob, updateSyncJobResult, updateSyncJobStatus } from '../sync/job.service.js'; import { getSyncConfig } from './config/config.service.js'; import localFileService from '../file/local.service.js'; +import BigQueryClient from '../../clients/big-query.client.js'; import { getLastSyncDate, setLastSyncDate } from './sync.service.js'; import { getDeletedKeys, takeSnapshot, clearOldRecords, syncUpdateAtForDeletedRecords } from './data/delete.service.js'; import environmentService from '../environment.service.js'; @@ -158,7 +159,7 @@ export default class SyncRun { if (!nangoConfig) { const message = `No ${this.isAction ? 'action' : 'sync'} configuration was found for ${this.syncName}.`; if (this.activityLogId) { - await this.reportFailureForResults(message); + await this.reportFailureForResults(message, 0); } else { console.error(message); } @@ -187,7 +188,7 @@ export default class SyncRun { if (!environment && !bypassEnvironment) { const message = `No environment was found for ${this.nangoConnection.environment_id}. The sync cannot continue without a valid environment`; - await this.reportFailureForResults(message); + await this.reportFailureForResults(message, 0); const errorType = this.determineErrorType(); return { success: false, error: new NangoError(errorType, message, 404), response: false }; } @@ -235,7 +236,7 @@ export default class SyncRun { ); if (!integrationFileResult) { const message = `Integration was attempted to run for ${this.syncName} but no integration file was found at ${integrationFilePath}.`; - await this.reportFailureForResults(message); + await this.reportFailureForResults(message, 0); const errorType = this.determineErrorType(); @@ -259,7 +260,7 @@ export default class SyncRun { if (isJsOrTsType(configInput as unknown as string)) { if (typeof this.input !== (configInput as unknown as string)) { const message = `The input provided of ${this.input} for ${this.syncName} is not of type ${configInput}`; - await this.reportFailureForResults(message); + await this.reportFailureForResults(message, 0); return { success: false, error: new NangoError('action_script_failure', message, 500), response: false }; } @@ -270,6 +271,10 @@ export default class SyncRun { } } + if (!this.nangoConnection.account_id && environment?.account_id) { + this.nangoConnection.account_id = environment.account_id; + } + const nangoProps = { host: optionalHost || getApiUrl(), accountId: environment?.account_id as number, @@ -336,10 +341,11 @@ export default class SyncRun { syncData?.version ? ` version: ${syncData.version}` : '' }`; + const runTime = (Date.now() - startTime) / 1000; if (error.type === 'script_cancelled') { - await this.reportFailureForResults(error.message); + await this.reportFailureForResults(error.message, runTime); } else { - await this.reportFailureForResults(message); + await this.reportFailureForResults(message, runTime); } return { success: false, error, response: false }; @@ -384,7 +390,8 @@ export default class SyncRun { await this.reportFailureForResults( `The ${this.syncType} "${this.syncName}"${ syncData?.version ? ` version: ${syncData?.version}` : '' - } sync did not complete successfully and has the following error: ${errorMessage}` + } sync did not complete successfully and has the following error: ${errorMessage}`, + (Date.now() - startTime) / 1000 ); const errorType = this.determineErrorType(); @@ -476,7 +483,8 @@ export default class SyncRun { const syncResult: SyncJob = await updateSyncJobResult(this.syncJobId, updatedResults, model); if (!syncResult) { - this.reportFailureForResults(`The sync job ${this.syncJobId} could not be updated with the results for the model ${model}.`); + await this.reportFailureForResults(`The sync job ${this.syncJobId} could not be updated with the results for the model ${model}.`, totalRunTime); + return; } @@ -570,9 +578,24 @@ export default class SyncRun { }, `syncId:${this.syncId}` ); + + await BigQueryClient.insert({ + executionType: this.determineExecutionType(), + connectionId: this.nangoConnection.connection_id, + accountId: this.nangoConnection.account_id, + scriptName: this.syncName, + scriptType: this.syncType, + environmentId: this.nangoConnection.environment_id, + providerConfigKey: this.nangoConnection.provider_config_key, + status: 'success', + syncId: this.syncId as string, + content, + runTime: totalRunTime, + timestamp: Date.now() + }); } - async reportFailureForResults(content: string) { + async reportFailureForResults(content: string, runTime: number) { if (!this.writeToDb) { return; } @@ -587,6 +610,22 @@ export default class SyncRun { this.nangoConnection.environment_id, this.provider as string ); + + await BigQueryClient.insert({ + executionType: this.determineExecutionType(), + connectionId: this.nangoConnection.connection_id, + // internal connection id is what? + accountId: this.nangoConnection.account_id, + scriptName: this.syncName, + scriptType: this.syncType, + environmentId: this.nangoConnection.environment_id, + providerConfigKey: this.nangoConnection.provider_config_key, + status: 'failed', + syncId: this.syncId as string, + content, + runTime, + timestamp: Date.now() + }); } catch { await errorManager.report('slack notification service reported a failure', { environmentId: this.nangoConnection.environment_id, @@ -653,13 +692,17 @@ export default class SyncRun { ); } - private determineErrorType(): string { + private determineExecutionType(): string { if (this.isAction) { - return 'action_script_failure'; + return 'action'; } else if (this.isWebhook) { - return 'webhook_script_failure'; + return 'webhook'; } else { - return 'sync_script_failure'; + return 'sync'; } } + + private determineErrorType(): string { + return this.determineExecutionType() + '_script_failure'; + } } diff --git a/packages/shared/package.json b/packages/shared/package.json index 719b833253..6c2f76d04c 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -19,6 +19,7 @@ "dependencies": { "@aws-sdk/client-s3": "3.348.0", "@datadog/datadog-api-client": "^1.16.0", + "@google-cloud/bigquery": "^7.5.1", "@hapi/boom": "^10.0.1", "@nangohq/node": "^0.39.8", "@sentry/node": "^7.105.0", From 849f99c835fe8da235830e89a35ffbf0acfa4152 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Tue, 26 Mar 2024 15:02:02 +0200 Subject: [PATCH 02/12] [nan-591] add internal connection id --- packages/shared/lib/services/sync/run.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/shared/lib/services/sync/run.service.ts b/packages/shared/lib/services/sync/run.service.ts index 5e82a95ff0..4c3031c2a0 100644 --- a/packages/shared/lib/services/sync/run.service.ts +++ b/packages/shared/lib/services/sync/run.service.ts @@ -568,6 +568,7 @@ export default class SyncRun { await BigQueryClient.insert({ executionType: this.determineExecutionType(), connectionId: this.nangoConnection.connection_id, + internalConnectionId: this.nangoConnection.id, accountId: this.nangoConnection.account_id, scriptName: this.syncName, scriptType: this.syncType, @@ -600,7 +601,7 @@ export default class SyncRun { await BigQueryClient.insert({ executionType: this.determineExecutionType(), connectionId: this.nangoConnection.connection_id, - // TODO get the internal connection id + internalConnectionId: this.nangoConnection.id, accountId: this.nangoConnection.account_id, scriptName: this.syncName, scriptType: this.syncType, From 066a82e93ff25a3846cbf4ca1a3cc97179e99264 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Wed, 27 Mar 2024 13:28:59 +0200 Subject: [PATCH 03/12] [nan-591] add internalConnectionId --- packages/shared/lib/clients/big-query.client.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/shared/lib/clients/big-query.client.ts b/packages/shared/lib/clients/big-query.client.ts index 49e70e8107..a6a6a86169 100644 --- a/packages/shared/lib/clients/big-query.client.ts +++ b/packages/shared/lib/clients/big-query.client.ts @@ -31,6 +31,7 @@ class BigQueryClient { schema: { fields: [ { name: 'executionType', type: 'STRING' }, + { name: 'internalConnectionId', type: 'INTEGER' }, { name: 'connectionId', type: 'STRING' }, { name: 'accountId', type: 'INTEGER' }, { name: 'scriptName', type: 'STRING' }, From 046bb2ad75cc9f206b5b100fb8e12da4a928e025 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Wed, 27 Mar 2024 13:33:16 +0200 Subject: [PATCH 04/12] [nan-591] table name and .env example --- .env.example | 3 +++ packages/shared/lib/clients/big-query.client.ts | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index 1807ad1d00..7329a39fe7 100644 --- a/.env.example +++ b/.env.example @@ -73,3 +73,6 @@ DEFAULT_GITHUB_CLIENT_SECRET= # Hosted Auth Configuration WORKOS_API_KEY= WORKOS_CLIENT_ID= + +# Google Cloud Configuration +GOOGLE_APPLICATION_CREDENTIALS= diff --git a/packages/shared/lib/clients/big-query.client.ts b/packages/shared/lib/clients/big-query.client.ts index a6a6a86169..a021d93e2d 100644 --- a/packages/shared/lib/clients/big-query.client.ts +++ b/packages/shared/lib/clients/big-query.client.ts @@ -11,8 +11,8 @@ class BigQueryClient { constructor() { this.client = new BigQuery(); - this.datasetName = 'usage'; - this.tableName = 'scriptRuns'; + this.datasetName = 'dataset'; + this.tableName = 'script_runs'; } private async createDataSet() { From 8bad3065cc7832a8d32a028dea71283d6cfbc496 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Wed, 27 Mar 2024 13:56:46 +0200 Subject: [PATCH 05/12] [nan-591] add specifics --- packages/shared/lib/clients/big-query.client.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/shared/lib/clients/big-query.client.ts b/packages/shared/lib/clients/big-query.client.ts index a021d93e2d..06ea820423 100644 --- a/packages/shared/lib/clients/big-query.client.ts +++ b/packages/shared/lib/clients/big-query.client.ts @@ -1,6 +1,6 @@ import { BigQuery } from '@google-cloud/bigquery'; import logger from '../logger/console.js'; -import { isCloud, getEnv } from '../utils/utils.js'; +import { isCloud, isLocal, getEnv } from '../utils/utils.js'; type Row = Record; @@ -41,7 +41,7 @@ class BigQueryClient { { name: 'status', type: 'STRING' }, { name: 'syncId', type: 'STRING' }, { name: 'content', type: 'STRING' }, - { name: 'runTime', type: 'INTEGER' }, + { name: 'runTime', type: 'INTEGER' }, // in seconds { name: 'timestamp', type: 'INTEGER' } ] } @@ -60,6 +60,9 @@ class BigQueryClient { await this.initializeTable(); await this.client.dataset(this.datasetName).table(`${getEnv()}_${this.tableName}`).insert(data); } + if (isLocal()) { + logger.info(`Data would be inserted into BigQuery type ${JSON.stringify(data, null, 2)}`); + } } catch (e) { logger.error('Error inserting into BigQuery', e); } From 2b817be95276dfda39ec2ef9a4392ed427d6f467 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 28 Mar 2024 20:10:49 +0200 Subject: [PATCH 06/12] [nan-591] update types --- packages/data-ingestion/lib/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/data-ingestion/lib/index.ts b/packages/data-ingestion/lib/index.ts index c9481cbac7..f7764e130c 100644 --- a/packages/data-ingestion/lib/index.ts +++ b/packages/data-ingestion/lib/index.ts @@ -7,9 +7,9 @@ const logger = getLogger('BigQueryClient'); interface RunScriptRow { executionType: string; - internalConnectionId: number; + internalConnectionId: number | undefined; connectionId: string; - accountId: number; + accountId: number | undefined; scriptName: string; scriptType: string; environmentId: number; From 88df622776e2930b750fcb3612d38053ec4e7bf9 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 28 Mar 2024 20:16:57 +0200 Subject: [PATCH 07/12] [nan-591] remove extra env for table name --- packages/data-ingestion/lib/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/data-ingestion/lib/index.ts b/packages/data-ingestion/lib/index.ts index f7764e130c..0ee87d9881 100644 --- a/packages/data-ingestion/lib/index.ts +++ b/packages/data-ingestion/lib/index.ts @@ -1,7 +1,7 @@ import { BigQuery } from '@google-cloud/bigquery'; import type { BigQuery as BigQueryType } from '@google-cloud/bigquery'; import { getLogger } from '@nangohq/utils/dist/logger.js'; -import { env, isCloud, isLocal } from '@nangohq/utils/dist/environment/detection.js'; +import { isCloud, isLocal } from '@nangohq/utils/dist/environment/detection.js'; const logger = getLogger('BigQueryClient'); @@ -87,7 +87,7 @@ class BigQueryClient { } public async insert(data: RunScriptRow, tableName?: string) { - const table = tableName || `${env}_${this.tableName}`; + const table = tableName || this.tableName; try { if (isCloud) { await this.client.dataset(this.datasetName).table(table).insert(data); From a70eec9d3a1609a5cb44ffcb37a659722647cf31 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 28 Mar 2024 20:33:38 +0200 Subject: [PATCH 08/12] [nan-591] add package --- tsconfig.build.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tsconfig.build.json b/tsconfig.build.json index eaf0561edc..2a548ec6f7 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -30,6 +30,9 @@ }, { "path": "packages/webapp" + }, + { + "path": "packages/data-ingestion" } ] } From cc32e2b74b3b6e88bf6d57eca693f4aa3be5a005 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 28 Mar 2024 21:24:57 +0200 Subject: [PATCH 09/12] [nan-591] fixes --- packages/data-ingestion/lib/index.ts | 2 +- packages/jobs/nodemon.json | 2 +- packages/persist/nodemon.json | 2 +- packages/runner/nodemon.json | 2 +- packages/server/nodemon.json | 2 +- packages/shared/lib/services/sync/run.service.ts | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/data-ingestion/lib/index.ts b/packages/data-ingestion/lib/index.ts index 0ee87d9881..380b0a66fb 100644 --- a/packages/data-ingestion/lib/index.ts +++ b/packages/data-ingestion/lib/index.ts @@ -78,7 +78,7 @@ class BigQueryClient { { name: 'status', type: 'STRING' }, { name: 'syncId', type: 'STRING' }, { name: 'content', type: 'STRING' }, - { name: 'runTimeInSeconds', type: 'INTEGER' }, // in seconds + { name: 'runTimeInSeconds', type: 'FLOAT' }, { name: 'createdAt', type: 'INTEGER' } ] } diff --git a/packages/jobs/nodemon.json b/packages/jobs/nodemon.json index e4179ce245..ccb17f5064 100644 --- a/packages/jobs/nodemon.json +++ b/packages/jobs/nodemon.json @@ -1,6 +1,6 @@ { "watch": ["lib", "../shared/dist", "../utils/dist", "../data-ingestion/dist", "../../.env"], - "ext": "ts,json", + "ext": "js,ts,json", "ignore": ["lib/**/*.spec.ts"], "exec": "tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env", "signal": "SIGTERM" diff --git a/packages/persist/nodemon.json b/packages/persist/nodemon.json index c71f2b06ec..adef9585e5 100644 --- a/packages/persist/nodemon.json +++ b/packages/persist/nodemon.json @@ -1,6 +1,6 @@ { "watch": ["lib", "../shared/dist", "../utils/dist", "../../.env"], - "ext": "ts,json", + "ext": "js,ts,json", "ignore": ["lib/**/*.test.ts"], "exec": "tsc && tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env" } diff --git a/packages/runner/nodemon.json b/packages/runner/nodemon.json index c71f2b06ec..adef9585e5 100644 --- a/packages/runner/nodemon.json +++ b/packages/runner/nodemon.json @@ -1,6 +1,6 @@ { "watch": ["lib", "../shared/dist", "../utils/dist", "../../.env"], - "ext": "ts,json", + "ext": "js,ts,json", "ignore": ["lib/**/*.test.ts"], "exec": "tsc && tsx -r dotenv/config lib/app.ts dotenv_config_path=./../../.env" } diff --git a/packages/server/nodemon.json b/packages/server/nodemon.json index 5292530ee5..1461aa051b 100644 --- a/packages/server/nodemon.json +++ b/packages/server/nodemon.json @@ -1,6 +1,6 @@ { "watch": ["lib", "../shared/dist", "../utils/dist", "../../.env", "../shared/providers.yaml"], - "ext": "ts,json", + "ext": "js,ts,json", "ignore": ["src/**/*.spec.ts"], "exec": "tsx -r dotenv/config lib/server.ts Dotenv_config_path=./../../.env" } diff --git a/packages/shared/lib/services/sync/run.service.ts b/packages/shared/lib/services/sync/run.service.ts index 04db3d8e88..bd023747c4 100644 --- a/packages/shared/lib/services/sync/run.service.ts +++ b/packages/shared/lib/services/sync/run.service.ts @@ -294,7 +294,7 @@ export default class SyncRun { } } - if (!this.nangoConnection.account_id && environment?.account_id) { + if (!this.nangoConnection.account_id && environment?.account_id !== null && environment?.account_id !== undefined) { this.nangoConnection.account_id = environment.account_id; } From f4a70ba8d85814a7cc042ba7de7213f3390a5e7a Mon Sep 17 00:00:00 2001 From: Khaliq Date: Thu, 28 Mar 2024 22:05:20 +0200 Subject: [PATCH 10/12] [nan-591] make bigquery optional --- packages/cli/lib/services/dryrun.service.ts | 1 - packages/data-ingestion/lib/index.ts | 5 +- .../sync/run.service.integration.test.ts | 10 --- .../shared/lib/services/sync/run.service.ts | 72 ++++++++++--------- 4 files changed, 40 insertions(+), 48 deletions(-) diff --git a/packages/cli/lib/services/dryrun.service.ts b/packages/cli/lib/services/dryrun.service.ts index 3a7cceace8..efa86f03e7 100644 --- a/packages/cli/lib/services/dryrun.service.ts +++ b/packages/cli/lib/services/dryrun.service.ts @@ -149,7 +149,6 @@ class DryRunService { }; const syncRun = new syncRunService({ - bigQueryClient: { insert: async () => {} }, integrationService, writeToDb: false, nangoConnection, diff --git a/packages/data-ingestion/lib/index.ts b/packages/data-ingestion/lib/index.ts index 380b0a66fb..85f7951ecf 100644 --- a/packages/data-ingestion/lib/index.ts +++ b/packages/data-ingestion/lib/index.ts @@ -1,7 +1,7 @@ import { BigQuery } from '@google-cloud/bigquery'; import type { BigQuery as BigQueryType } from '@google-cloud/bigquery'; import { getLogger } from '@nangohq/utils/dist/logger.js'; -import { isCloud, isLocal } from '@nangohq/utils/dist/environment/detection.js'; +import { isCloud } from '@nangohq/utils/dist/environment/detection.js'; const logger = getLogger('BigQueryClient'); @@ -92,9 +92,6 @@ class BigQueryClient { if (isCloud) { await this.client.dataset(this.datasetName).table(table).insert(data); } - if (isLocal) { - logger.info(`Data would be inserted into BigQuery type ${JSON.stringify(data, null, 2)}`); - } } catch (e) { logger.error('Error inserting into BigQuery', e); } diff --git a/packages/shared/lib/services/sync/run.service.integration.test.ts b/packages/shared/lib/services/sync/run.service.integration.test.ts index 0d0557887f..ea9bf96f9e 100644 --- a/packages/shared/lib/services/sync/run.service.integration.test.ts +++ b/packages/shared/lib/services/sync/run.service.integration.test.ts @@ -26,14 +26,7 @@ class integrationServiceMock implements IntegrationServiceInterface { } } -class bigQueryClientMock { - async insert() { - return; - } -} - const integrationService = new integrationServiceMock(); -const bigQueryClient = new bigQueryClientMock(); describe('Running sync', () => { beforeAll(async () => { @@ -185,7 +178,6 @@ describe('Running sync', () => { describe('SyncRun', () => { const dryRunConfig = { - bigQueryClient, integrationService: integrationService as unknown as IntegrationServiceInterface, writeToDb: false, nangoConnection: { @@ -204,7 +196,6 @@ describe('SyncRun', () => { it('should initialize correctly', () => { const config = { integrationService: integrationService as unknown as IntegrationServiceInterface, - bigQueryClient, writeToDb: true, nangoConnection: { id: 1, @@ -326,7 +317,6 @@ const runJob = async ( } const config = { integrationService: integrationService, - bigQueryClient, writeToDb: true, nangoConnection: connection, syncName: sync.name, diff --git a/packages/shared/lib/services/sync/run.service.ts b/packages/shared/lib/services/sync/run.service.ts index bd023747c4..cdbf18a157 100644 --- a/packages/shared/lib/services/sync/run.service.ts +++ b/packages/shared/lib/services/sync/run.service.ts @@ -45,7 +45,7 @@ interface RunScriptRow { } interface SyncRunConfig { - bigQueryClient: BigQueryClientInterface; + bigQueryClient?: BigQueryClientInterface; integrationService: IntegrationServiceInterface; writeToDb: boolean; isAction?: boolean; @@ -71,7 +71,7 @@ interface SyncRunConfig { } export default class SyncRun { - bigQueryClient: BigQueryClientInterface; + bigQueryClient?: BigQueryClientInterface; integrationService: IntegrationServiceInterface; writeToDb: boolean; isAction: boolean; @@ -99,7 +99,9 @@ export default class SyncRun { constructor(config: SyncRunConfig) { this.integrationService = config.integrationService; - this.bigQueryClient = config.bigQueryClient; + if (config.bigQueryClient) { + this.bigQueryClient = config.bigQueryClient; + } this.writeToDb = config.writeToDb; this.isAction = config.isAction || false; this.isWebhook = config.isWebhook || false; @@ -591,21 +593,23 @@ export default class SyncRun { `syncId:${this.syncId}` ); - void this.bigQueryClient.insert({ - executionType: this.determineExecutionType(), - connectionId: this.nangoConnection.connection_id, - internalConnectionId: this.nangoConnection.id, - accountId: this.nangoConnection.account_id, - scriptName: this.syncName, - scriptType: this.syncType, - environmentId: this.nangoConnection.environment_id, - providerConfigKey: this.nangoConnection.provider_config_key, - status: 'success', - syncId: this.syncId as string, - content, - runTimeInSeconds: totalRunTime, - createdAt: Date.now() - }); + if (this.bigQueryClient) { + void this.bigQueryClient.insert({ + executionType: this.determineExecutionType(), + connectionId: this.nangoConnection.connection_id, + internalConnectionId: this.nangoConnection.id, + accountId: this.nangoConnection.account_id, + scriptName: this.syncName, + scriptType: this.syncType, + environmentId: this.nangoConnection.environment_id, + providerConfigKey: this.nangoConnection.provider_config_key, + status: 'success', + syncId: this.syncId as string, + content, + runTimeInSeconds: totalRunTime, + createdAt: Date.now() + }); + } } async reportFailureForResults({ content, runTime }: { content: string; runTime: number }) { @@ -624,21 +628,23 @@ export default class SyncRun { this.provider as string ); - void this.bigQueryClient.insert({ - executionType: this.determineExecutionType(), - connectionId: this.nangoConnection.connection_id, - internalConnectionId: this.nangoConnection.id, - accountId: this.nangoConnection.account_id, - scriptName: this.syncName, - scriptType: this.syncType, - environmentId: this.nangoConnection.environment_id, - providerConfigKey: this.nangoConnection.provider_config_key, - status: 'failed', - syncId: this.syncId as string, - content, - runTimeInSeconds: runTime, - createdAt: Date.now() - }); + if (this.bigQueryClient) { + void this.bigQueryClient.insert({ + executionType: this.determineExecutionType(), + connectionId: this.nangoConnection.connection_id, + internalConnectionId: this.nangoConnection.id, + accountId: this.nangoConnection.account_id, + scriptName: this.syncName, + scriptType: this.syncType, + environmentId: this.nangoConnection.environment_id, + providerConfigKey: this.nangoConnection.provider_config_key, + status: 'failed', + syncId: this.syncId as string, + content, + runTimeInSeconds: runTime, + createdAt: Date.now() + }); + } } catch { await errorManager.report('slack notification service reported a failure', { environmentId: this.nangoConnection.environment_id, From 4c7c6e648374168b8be4a4f4c945c043eb232c60 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 29 Mar 2024 09:44:39 +0300 Subject: [PATCH 11/12] [nan-591] final test tweaks --- .../services/sync/config/config.service.ts | 1 + .../sync/run.service.integration.test.ts | 2 +- .../shared/lib/services/sync/run.service.ts | 67 ++++++++++--------- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/packages/shared/lib/services/sync/config/config.service.ts b/packages/shared/lib/services/sync/config/config.service.ts index 0501c2f369..9387a9589a 100644 --- a/packages/shared/lib/services/sync/config/config.service.ts +++ b/packages/shared/lib/services/sync/config/config.service.ts @@ -738,6 +738,7 @@ export async function getConfigWithEndpointsByProviderConfigKey(environment_id: `${TABLE}.sync_type`, `${TABLE}.track_deletes`, `${TABLE}.auto_start`, + `${TABLE}.webhook_subscriptions`, '_nango_configs.unique_key', '_nango_configs.provider', db.knex.raw( diff --git a/packages/shared/lib/services/sync/run.service.integration.test.ts b/packages/shared/lib/services/sync/run.service.integration.test.ts index ea9bf96f9e..5909ace6a5 100644 --- a/packages/shared/lib/services/sync/run.service.integration.test.ts +++ b/packages/shared/lib/services/sync/run.service.integration.test.ts @@ -345,7 +345,7 @@ const runJob = async ( }; await jobService.updateSyncJobResult(syncJob.id, updatedResults, model); // finish the sync - await syncRun.finishSync([model], new Date(), `v1`, 10, trackDeletes); + await syncRun.finishFlow([model], new Date(), `v1`, 10, trackDeletes); const syncJobResult = await jobService.getLatestSyncJob(sync.id); return { diff --git a/packages/shared/lib/services/sync/run.service.ts b/packages/shared/lib/services/sync/run.service.ts index cdbf18a157..92d3fabb9c 100644 --- a/packages/shared/lib/services/sync/run.service.ts +++ b/packages/shared/lib/services/sync/run.service.ts @@ -380,6 +380,8 @@ export default class SyncRun { return userDefinedResults; } + const totalRunTime = (Date.now() - startTime) / 1000; + if (this.isAction) { const content = `${this.syncName} action was run successfully and results are being sent synchronously.`; @@ -402,11 +404,12 @@ export default class SyncRun { this.provider as string ); + await this.finishFlow(models, syncStartDate, syncData.version as string, totalRunTime, trackDeletes); + return { success: true, error: null, response: userDefinedResults }; } - const totalRunTime = (Date.now() - startTime) / 1000; - await this.finishSync(models, syncStartDate, syncData.version as string, totalRunTime, trackDeletes); + await this.finishFlow(models, syncStartDate, syncData.version as string, totalRunTime, trackDeletes); return { success: true, error: null, response: true }; } catch (e) { @@ -433,7 +436,7 @@ export default class SyncRun { return { success: true, error: null, response: result }; } - async finishSync(models: string[], syncStartDate: Date, version: string, totalRunTime: number, trackDeletes?: boolean): Promise { + async finishFlow(models: string[], syncStartDate: Date, version: string, totalRunTime: number, trackDeletes?: boolean): Promise { let i = 0; for (const model of models) { let deletedKeys: string[] = []; @@ -460,7 +463,29 @@ export default class SyncRun { version: string, totalRunTime: number ): Promise { - if (!this.writeToDb || !this.activityLogId || !this.syncJobId) { + if (!this.writeToDb) { + return; + } + + if (this.bigQueryClient) { + void this.bigQueryClient.insert({ + executionType: this.determineExecutionType(), + connectionId: this.nangoConnection.connection_id, + internalConnectionId: this.nangoConnection.id, + accountId: this.nangoConnection.account_id, + scriptName: this.syncName, + scriptType: this.syncType, + environmentId: this.nangoConnection.environment_id, + providerConfigKey: this.nangoConnection.provider_config_key, + status: 'success', + syncId: this.syncId as string, + content: `The ${this.syncType} "${this.syncName}" ${this.determineExecutionType()} has been completed successfully.`, + runTimeInSeconds: totalRunTime, + createdAt: Date.now() + }); + } + + if (!this.activityLogId || !this.syncJobId) { return; } @@ -592,6 +617,12 @@ export default class SyncRun { }, `syncId:${this.syncId}` ); + } + + async reportFailureForResults({ content, runTime }: { content: string; runTime: number }) { + if (!this.writeToDb) { + return; + } if (this.bigQueryClient) { void this.bigQueryClient.insert({ @@ -603,19 +634,13 @@ export default class SyncRun { scriptType: this.syncType, environmentId: this.nangoConnection.environment_id, providerConfigKey: this.nangoConnection.provider_config_key, - status: 'success', + status: 'failed', syncId: this.syncId as string, content, - runTimeInSeconds: totalRunTime, + runTimeInSeconds: runTime, createdAt: Date.now() }); } - } - - async reportFailureForResults({ content, runTime }: { content: string; runTime: number }) { - if (!this.writeToDb) { - return; - } if (!this.isWebhook) { try { @@ -627,24 +652,6 @@ export default class SyncRun { this.nangoConnection.environment_id, this.provider as string ); - - if (this.bigQueryClient) { - void this.bigQueryClient.insert({ - executionType: this.determineExecutionType(), - connectionId: this.nangoConnection.connection_id, - internalConnectionId: this.nangoConnection.id, - accountId: this.nangoConnection.account_id, - scriptName: this.syncName, - scriptType: this.syncType, - environmentId: this.nangoConnection.environment_id, - providerConfigKey: this.nangoConnection.provider_config_key, - status: 'failed', - syncId: this.syncId as string, - content, - runTimeInSeconds: runTime, - createdAt: Date.now() - }); - } } catch { await errorManager.report('slack notification service reported a failure', { environmentId: this.nangoConnection.environment_id, From 68ffc72044522b8027b985e6615549775b603169 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Fri, 29 Mar 2024 10:16:47 +0300 Subject: [PATCH 12/12] [nan-591] for a multi model sync only report once --- .../shared/lib/services/sync/run.service.ts | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/shared/lib/services/sync/run.service.ts b/packages/shared/lib/services/sync/run.service.ts index 92d3fabb9c..60d41de711 100644 --- a/packages/shared/lib/services/sync/run.service.ts +++ b/packages/shared/lib/services/sync/run.service.ts @@ -452,21 +452,8 @@ export default class SyncRun { await this.reportResults(model, { addedKeys: [], updatedKeys: [], deletedKeys }, i, models.length, syncStartDate, version, totalRunTime); i++; } - } - - async reportResults( - model: string, - responseResults: UpsertSummary, - index: number, - numberOfModels: number, - syncStartDate: Date, - version: string, - totalRunTime: number - ): Promise { - if (!this.writeToDb) { - return; - } + // we only want to report to bigquery once if it is a multi model sync if (this.bigQueryClient) { void this.bigQueryClient.insert({ executionType: this.determineExecutionType(), @@ -484,8 +471,18 @@ export default class SyncRun { createdAt: Date.now() }); } + } - if (!this.activityLogId || !this.syncJobId) { + async reportResults( + model: string, + responseResults: UpsertSummary, + index: number, + numberOfModels: number, + syncStartDate: Date, + version: string, + totalRunTime: number + ): Promise { + if (!this.writeToDb || !this.activityLogId || !this.syncJobId) { return; }