[gh-#1170] added,updated record filter (#1172)

NangoHQ · Oct 23, 2023 · 93cd819 · 93cd819
1 parent 52e8412
commit 93cd819
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 22 deletions.
diff --git a/docs-v2/sdks/node.mdx b/docs-v2/sdks/node.mdx
@@ -324,7 +324,7 @@ const records = await nango.getRecords<ModelName>({
     offset: <number>,                         // Optional: For pagination: The number of records to skip. If not passed, no records are skipped.
     sortBy: 'id' | 'createdAt' | 'updatedAt', // Optional: Optional: Set how the records are sorted. The default is by 'id'
     order: 'asc' | 'desc',                    // Optional: Set the order of results. The default is 'desc'
-    filter: 'added' | 'updated' | 'deleted'   // Optional: Retrieve only records that have been added, updated, or deleted since the last sync. Useful when used with the delta parameter.
+    filter: 'added' | 'updated' | 'deleted'   // Optional: Retrieve only records that have been added, updated, or deleted since the last sync. Useful when used with the delta parameter. Also accepts comma separated combinations e.g., "added,updated"
 });
 
 console.log(records);

diff --git a/packages/node-client/lib/index.ts b/packages/node-client/lib/index.ts
@@ -284,7 +284,7 @@ export class Nango {
     }
 
     public async getRecords<T = any>(config: GetRecordsRequestConfig): Promise<(T & { _nango_metadata: RecordMetadata })[]> {
-        const { connectionId, providerConfigKey, model, delta, offset, limit, includeNangoMetadata } = config;
+        const { connectionId, providerConfigKey, model, delta, offset, limit, includeNangoMetadata, filter } = config;
         validateSyncRecordConfiguration(config);
 
         const order = config?.order === 'asc' ? 'asc' : 'desc';
@@ -299,20 +299,6 @@ export class Nango {
                 break;
         }
 
-        let filter = '';
-
-        switch (config.filter) {
-            case 'deleted':
-                filter = 'deleted';
-                break;
-            case 'updated':
-                filter = 'updated';
-                break;
-            case 'added':
-                filter = 'added';
-                break;
-        }
-
         if (includeNangoMetadata) {
             console.warn(
                 `The includeNangoMetadata option will be deprecated soon and will be removed in a future release. Each record now has a _nango_metadata property which includes the same properties.`
@@ -322,7 +308,8 @@ export class Nango {
 
         const url = `${this.serverUrl}/sync/records/?model=${model}&order=${order}&delta=${delta || ''}&offset=${offset || ''}&limit=${limit || ''}&sort_by=${
             sortBy || ''
-        }&include_nango_metadata=${includeMetadata}&filter=${filter}`;
+        }&include_nango_metadata=${includeMetadata}${filter ? `&filter=${filter}` : ''}`;
+
         const headers: Record<string, string | number | boolean> = {
             'Connection-Id': connectionId,
             'Provider-Config-Key': providerConfigKey

diff --git a/packages/node-client/lib/types.ts b/packages/node-client/lib/types.ts
@@ -41,6 +41,9 @@ export interface ProxyConfiguration {
     decompress?: boolean;
 }
 
+type FilterAction = 'added' | 'updated' | 'deleted';
+type CombinedFilterAction = `${FilterAction},${FilterAction}`;
+
 export interface GetRecordsRequestConfig {
     providerConfigKey: string;
     connectionId: string;
@@ -51,7 +54,7 @@ export interface GetRecordsRequestConfig {
     sortBy?: 'updatedAt' | 'createdAt' | 'id';
     order?: 'asc' | 'desc';
     includeNangoMetadata?: boolean;
-    filter?: 'added' | 'updated' | 'deleted';
+    filter?: FilterAction | CombinedFilterAction;
 }
 
 export interface BasicApiCredentials {

diff --git a/packages/shared/lib/services/sync/data/records.service.ts b/packages/shared/lib/services/sync/data/records.service.ts
@@ -157,14 +157,29 @@ export async function getDataRecords(
     }
 
     if (filter) {
-        switch (filter) {
-            case 'added':
+        switch (true) {
+            case filter.includes('added') && filter.includes('updated'):
+                query = query.andWhere('external_deleted_at', null).andWhere(function () {
+                    this.where('created_at', '=', db.knex.raw('updated_at')).orWhere('created_at', '!=', db.knex.raw('updated_at'));
+                });
+                break;
+            case filter.includes('updated') && filter.includes('deleted'):
+                query = query.andWhere(function () {
+                    this.where('external_is_deleted', true).orWhere('external_deleted_at', null).andWhere('created_at', '!=', db.knex.raw('updated_at'));
+                });
+                break;
+            case filter.includes('added') && filter.includes('deleted'):
+                query = query.andWhere(function () {
+                    this.where('external_is_deleted', true).orWhere('external_deleted_at', null).andWhere('created_at', '=', db.knex.raw('updated_at'));
+                });
+                break;
+            case filter === 'added':
                 query = query.andWhere('external_deleted_at', null).andWhere('created_at', '=', db.knex.raw('updated_at'));
                 break;
-            case 'updated':
+            case filter === 'updated':
                 query = query.andWhere('external_deleted_at', null).andWhere('created_at', '!=', db.knex.raw('updated_at'));
                 break;
-            case 'deleted':
+            case filter === 'deleted':
                 query = query.andWhere({ external_is_deleted: true });
                 break;
         }