Merge pull request #524 from OpenFn/cursor-surveycto

surveycto: add new cursor
OpenFn · May 8, 2024 · dadff02 · dadff02
2 parents 2c7aa2f + e892ccb
commit dadff02
Show file tree

Hide file tree

Showing 43 changed files with 413 additions and 85 deletions.
diff --git a/packages/bigquery/CHANGELOG.md b/packages/bigquery/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-bigquery
 
+## 2.0.8
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 2.0.7
 
 ### Patch Changes

diff --git a/packages/bigquery/package.json b/packages/bigquery/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-bigquery",
-  "version": "2.0.7",
+  "version": "2.0.8",
   "description": "A Google BigQuery language package for use with Open Function",
   "main": "dist/index.cjs",
   "scripts": {
@@ -21,7 +21,7 @@
   ],
   "dependencies": {
     "@google-cloud/bigquery": "^5.12.0",
-    "@openfn/language-common": "workspace:1.13.2",
+    "@openfn/language-common": "workspace:1.13.3",
     "csv-parse": "^4.16.3",
     "import": "0.0.6",
     "json2csv": "^5.0.7",

diff --git a/packages/commcare/CHANGELOG.md b/packages/commcare/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-commcare
 
+## 1.6.14
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 1.6.13
 
 ### Patch Changes

diff --git a/packages/commcare/package.json b/packages/commcare/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-commcare",
-  "version": "1.6.13",
+  "version": "1.6.14",
   "description": "Commcare Language Pack for OpenFn",
   "homepage": "https://docs.openfn.org",
   "repository": {
@@ -25,7 +25,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "workspace:1.13.2",
+    "@openfn/language-common": "workspace:1.13.3",
     "@openfn/language-http": "workspace:^6.0.0",
     "JSONPath": "^0.10.0",
     "form-data": "^4.0.0",

diff --git a/packages/common/CHANGELOG.md b/packages/common/CHANGELOG.md
@@ -1,5 +1,11 @@
 v0.4.0
 
+## 1.13.3
+
+### Patch Changes
+
+- 88f99a8f: cursor: support format option
+
 ## 1.13.2
 
 ### Patch Changes

diff --git a/packages/common/package.json b/packages/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-common",
-  "version": "1.13.2",
+  "version": "1.13.3",
   "description": "Common Expressions for OpenFn",
   "homepage": "https://docs.openfn.org",
   "repository": {

diff --git a/packages/common/src/Adaptor.js b/packages/common/src/Adaptor.js
@@ -6,7 +6,7 @@ import { parse } from 'csv-parse';
 import { Readable } from 'node:stream';
 
 import { request } from 'undici';
-import { format } from 'date-fns';
+import dateFns from 'date-fns';
 
 import { expandReferences as newExpandReferences, parseDate } from './util';
 
@@ -787,6 +787,8 @@ let cursorKey = 'cursor';
  * Supports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,
  * which will be converted relative to the environment (ie, the Lightning or CLI locale). Custom timezones 
  * are not yet supported.
+ * You can provide a formatter to customise the final cursor value, which is useful for normalising
+ * different inputs. The custom formatter runs after natural language date conversion.
  * See the usage guide at {@link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}
  * @public
  * @example <caption>Use a cursor from state if present, or else use the default value</caption>
@@ -798,15 +800,18 @@ let cursorKey = 'cursor';
  * @param {object} options - options to control the cursor.
  * @param {string} options.key - set the cursor key. Will persist through the whole run.
  * @param {any} options.defaultValue - the value to use if value is falsy
+ * @param {Function} options.format - custom formatter for the final cursor value
  * @returns {Operation}
  */
 export function cursor(value, options = {}) {
   return (state) => {
-    const [resolvedValue, resolvedOptions] = newExpandReferences(state, value, options);
+    const { format, ...optionsWithoutFormat } = options;
+    const [resolvedValue, resolvedOptions] = newExpandReferences(state, value, optionsWithoutFormat);
 
     const {
       defaultValue, // if there is no cursor on state, this will be used
       key, // the key to use on state
+      // format // pulled out before reference resolution else or it'll be treated as a ref!
     } = resolvedOptions;
 
     if (key) {
@@ -821,16 +826,21 @@ export function cursor(value, options = {}) {
     if (typeof cursor === 'string') {
       const date = parseDate(cursor, cursorStart)
       if (date instanceof Date && date.toString !== "Invalid Date") {
-        state[cursorKey] = date.toISOString();
-        // Log the converted date in a very international, human-friendly format
-        // See https://date-fns.org/v3.6.0/docs/format
-        const formatted = format(date, 'HH:MM d MMM yyyy (OOO)')
+        state[cursorKey] = format?.(date) ?? date.toISOString();
+
+        const formatted = format
+          ? state[cursorKey]
+          // If no custom formatter is provided,
+          // Log the converted date in a very international, human-friendly format
+          // See https://date-fns.org/v3.6.0/docs/format
+          : dateFns.format(date, 'HH:MM d MMM yyyy (OOO)');
+
         console.log(`Setting cursor "${cursor}" to: ${formatted}`);
         return state;
       }
     }
-    state[cursorKey] = cursor;
-    console.log('Setting cursor to:', cursor);
+    state[cursorKey] = format?.(cursor) ?? cursor;
+    console.log('Setting cursor to:', state[cursorKey]);
 
     return state;
   }

diff --git a/packages/common/test/index.test.js b/packages/common/test/index.test.js
@@ -932,4 +932,29 @@ describe('cursor', () => {
     expect(result.cursor).to.eql(c)
   });
 
+  it('should apply a custom formatter', () => {
+    const state = {};
+    const result = cursor('abc', {
+      format: (c) => c.toUpperCase()
+    })(state);
+    expect(result.cursor).to.eql('ABC')
+  })
+
+  it('should format "today"', () => {
+    const state = {};
+    const date = new Date().toDateString();
+    const result = cursor('today', {
+      format: (c) => c.toDateString()
+    })(state);
+    expect(result.cursor).to.eql(date)
+  })
+
+  it('should format a number to an arbitrary object', () => {
+    const state = {};
+    const result = cursor(3, {
+      format: (c) => ({ page: c })
+    })(state);
+    expect(result.cursor).to.eql({ page: 3 })
+  })
+
 });
diff --git a/packages/dynamics/CHANGELOG.md b/packages/dynamics/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-dynamics
 
+## 0.4.12
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 0.4.11
 
 ### Patch Changes

diff --git a/packages/dynamics/package.json b/packages/dynamics/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-dynamics",
-  "version": "0.4.11",
+  "version": "0.4.12",
   "description": "A Microsoft Dynamics Language Pack for OpenFn",
   "main": "dist/index.cjs",
   "scripts": {
@@ -20,7 +20,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.13.2",
+    "@openfn/language-common": "1.13.3",
     "request": "^2.72.0"
   },
   "devDependencies": {

diff --git a/packages/msgraph/CHANGELOG.md b/packages/msgraph/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-msgraph
 
+## 0.5.3
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 0.5.2
 
 ### Patch Changes

diff --git a/packages/msgraph/package.json b/packages/msgraph/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-msgraph",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "description": "Microsoft Graph Language Pack for OpenFn",
   "type": "module",
   "exports": {

diff --git a/packages/mssql/CHANGELOG.md b/packages/mssql/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-mssql
 
+## 4.2.2
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 4.2.1
 
 ### Patch Changes

diff --git a/packages/mssql/package.json b/packages/mssql/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-mssql",
-  "version": "4.2.1",
+  "version": "4.2.2",
   "description": "A Microsoft SQL language pack for OpenFn",
   "exports": {
     ".": {
@@ -31,7 +31,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "workspace:1.13.2",
+    "@openfn/language-common": "workspace:1.13.3",
     "tedious": "15.1.0"
   },
   "devDependencies": {

diff --git a/packages/mysql/CHANGELOG.md b/packages/mysql/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-mysql
 
+## 1.4.13
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 1.4.12
 
 ### Patch Changes

diff --git a/packages/mysql/package.json b/packages/mysql/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-mysql",
-  "version": "1.4.12",
+  "version": "1.4.13",
   "description": "A MySQL Language Pack for OpenFn",
   "homepage": "https://docs.openfn.org",
   "main": "dist/index.cjs",
@@ -25,7 +25,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.13.2",
+    "@openfn/language-common": "1.13.3",
     "json-sql": "^0.3.10",
     "mysql": "^2.13.0",
     "squel": "^5.8.0",

diff --git a/packages/nexmo/CHANGELOG.md b/packages/nexmo/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-nexmo
 
+## 0.4.7
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 0.4.6
 
 ### Patch Changes

diff --git a/packages/nexmo/package.json b/packages/nexmo/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-nexmo",
-  "version": "0.4.6",
+  "version": "0.4.7",
   "description": "An Language Package for Nexmo",
   "main": "dist/index.cjs",
   "scripts": {

diff --git a/packages/ocl/CHANGELOG.md b/packages/ocl/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-ocl
 
+## 1.1.11
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 1.1.10
 
 ### Patch Changes

diff --git a/packages/ocl/package.json b/packages/ocl/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-ocl",
-  "version": "1.1.10",
+  "version": "1.1.11",
   "description": "An OCL language package for use with Open Function",
   "main": "dist/index.cjs",
   "scripts": {
@@ -20,7 +20,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.13.2"
+    "@openfn/language-common": "1.13.3"
   },
   "devDependencies": {
     "@openfn/simple-ast": "^0.4.1",

diff --git a/packages/openfn/CHANGELOG.md b/packages/openfn/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-openfn
 
+## 1.3.12
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 1.3.11
 
 ### Patch Changes

diff --git a/packages/openfn/package.json b/packages/openfn/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-openfn",
-  "version": "1.3.11",
+  "version": "1.3.12",
   "description": "An (experimental) adaptor for accessing the OpenFn web API",
   "homepage": "https://docs.openfn.org",
   "repository": {
@@ -25,7 +25,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.13.2",
+    "@openfn/language-common": "1.13.3",
     "axios": "^0.21.1"
   },
   "devDependencies": {

diff --git a/packages/openmrs/CHANGELOG.md b/packages/openmrs/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-openmrs
 
+## 3.0.4
+
+### Patch Changes
+
+- Updated dependencies [88f99a8f]
+  - @openfn/language-common@1.13.3
+
 ## 3.0.3
 
 ### Patch Changes

diff --git a/packages/openmrs/package.json b/packages/openmrs/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-openmrs",
-  "version": "3.0.3",
+  "version": "3.0.4",
   "description": "OpenMRS Language Pack for OpenFn",
   "homepage": "https://docs.openfn.org",
   "repository": {
@@ -25,7 +25,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.13.2"
+    "@openfn/language-common": "1.13.3"
   },
   "devDependencies": {
     "@openfn/simple-ast": "^0.4.1",