surveycto: add new cursor

.changeset/purple-schools-switch.md

@@ -0,0 +1,5 @@
+---
+'@openfn/language-surveycto': minor
+---
+
+Extend the cursor function to support surveyCTO string format dates

.changeset/wet-squids-drop.md

@@ -0,0 +1,5 @@
+---
+'@openfn/language-common': patch
+---
+
+cursor: support format option

packages/common/src/Adaptor.js

@@ -6,7 +6,7 @@ import { parse } from 'csv-parse';
 import { Readable } from 'node:stream';
 
 import { request } from 'undici';
-import { format } from 'date-fns';
+import dateFns from 'date-fns';
 
 import { expandReferences as newExpandReferences, parseDate } from './util';
 
@@ -787,6 +787,8 @@ let cursorKey = 'cursor';
  * Supports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,
  * which will be converted relative to the environment (ie, the Lightning or CLI locale). Custom timezones 
  * are not yet supported.
+ * You can provide a formatter to customise the final cursor value, which is useful for normalising
+ * different inputs. The custom formatter runs after natural language date conversion.
  * See the usage guide at {@link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}
  * @public
  * @example <caption>Use a cursor from state if present, or else use the default value</caption>
@@ -798,15 +800,18 @@ let cursorKey = 'cursor';
  * @param {object} options - options to control the cursor.
  * @param {string} options.key - set the cursor key. Will persist through the whole run.
  * @param {any} options.defaultValue - the value to use if value is falsy
+ * @param {Function} options.format - custom formatter for the final cursor value
  * @returns {Operation}
  */
 export function cursor(value, options = {}) {
   return (state) => {
-    const [resolvedValue, resolvedOptions] = newExpandReferences(state, value, options);
+    const { format, ...optionsWithoutFormat } = options;
+    const [resolvedValue, resolvedOptions] = newExpandReferences(state, value, optionsWithoutFormat);
 
     const {
       defaultValue, // if there is no cursor on state, this will be used
       key, // the key to use on state
+      // format // pulled out before reference resolution else or it'll be treated as a ref!
     } = resolvedOptions;
 
     if (key) {
@@ -821,16 +826,21 @@ export function cursor(value, options = {}) {
     if (typeof cursor === 'string') {
       const date = parseDate(cursor, cursorStart)
       if (date instanceof Date && date.toString !== "Invalid Date") {
-        state[cursorKey] = date.toISOString();
-        // Log the converted date in a very international, human-friendly format
-        // See https://date-fns.org/v3.6.0/docs/format
-        const formatted = format(date, 'HH:MM d MMM yyyy (OOO)')
+        state[cursorKey] = format?.(date) ?? date.toISOString();
+
+        const formatted = format
+          ? state[cursorKey]
+          // If no custom formatter is provided,
+          // Log the converted date in a very international, human-friendly format
+          // See https://date-fns.org/v3.6.0/docs/format
+          : dateFns.format(date, 'HH:MM d MMM yyyy (OOO)');
+
         console.log(`Setting cursor "${cursor}" to: ${formatted}`);
         return state;
       }
     }
-    state[cursorKey] = cursor;
-    console.log('Setting cursor to:', cursor);
+    state[cursorKey] = format?.(cursor) ?? cursor;
+    console.log('Setting cursor to:', state[cursorKey]);
 
     return state;
   }

packages/common/test/index.test.js

@@ -932,4 +932,29 @@ describe('cursor', () => {
     expect(result.cursor).to.eql(c)
   });
 
+  it('should apply a custom formatter', () => {
+    const state = {};
+    const result = cursor('abc', {
+      format: (c) => c.toUpperCase()
+    })(state);
+    expect(result.cursor).to.eql('ABC')
+  })
+
+  it('should format "today"', () => {
+    const state = {};
+    const date = new Date().toDateString();
+    const result = cursor('today', {
+      format: (c) => c.toDateString()
+    })(state);
+    expect(result.cursor).to.eql(date)
+  })
+
+  it('should format a number to an arbitrary object', () => {
+    const state = {};
+    const result = cursor(3, {
+      format: (c) => ({ page: c })
+    })(state);
+    expect(result.cursor).to.eql({ page: 3 })
+  })
+
 });

packages/postgresql/src/Adaptor.js

@@ -165,6 +165,8 @@ function queryHandler(state, query, options, callback) {
  * @param {function} callback - (Optional) callback function
  * @returns {Operation}
  */
+
+// TODO this function is not properly async
 export function sql(sqlQuery, options, callback) {
   return state => {
     let { client } = state;

packages/surveycto/package.json

@@ -20,7 +20,8 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "workspace:^1.12.0"
+    "@openfn/language-common": "workspace:^1.12.0",
+    "date-fns-tz": "^3.1.3"
   },
   "devDependencies": {
     "@openfn/simple-ast": "0.4.1",

packages/surveycto/src/Adaptor.js

@@ -1,6 +1,9 @@
-import { execute as commonExecute } from '@openfn/language-common';
+import {
+  execute as commonExecute,
+  cursor as commonCorsor,
+} from '@openfn/language-common';
 import { expandReferences } from '@openfn/language-common/util';
-import { requestHelper } from './Utils';
+import { convertDate, requestHelper } from './Utils';
 
 /**
  * Execute a sequence of operations.
@@ -26,43 +29,45 @@ export function execute(...operations) {
 }
 
 /**
- * Options provided to the HTTP request
- * @typedef {Object} FormSubmissionOptions
- * @property {string} [date=0] - A timestamp in seconds or millseconds or in `MMM dd, yyyy h:mm:ss a` format. Defaults to `0` which will request all data
- * @property {string} [format='json'] - Format the submission data typee, It can be in `csv` or `json`. Defaults to `json` (JSON response)
- * @property {string} status - (Opt)Review status. Can be either, `approved`, `rejected`, `pending` or combine eg `approved|rejected`.
+ * Options provided to `fetchSubmissions()`
+ * @typedef {Object} FetchSubmissionOptions
+ * @property {string} [date=0] - Fetches only submissions from this timestamp.
+ * All values will be converted to surveyCTO `MMM dd, yyy h:mm:ss` format (in UTC time) in the request.
+ * Unix and Epoch timestamps are supported, as well as ISO date representatons.
+ * If set to 0, all submissions will be retrieved.
+ * @property {string} [format=json] - Format the submission data type as  `csv` or `json`.
+ * @property {string} [status] - Review status. Can be either, `approved`, `rejected`, `pending` or combine eg `approved|rejected`.
  */
 
 /**
  * Fetch form submissions
  * @example <caption>Fetch all form submissions</caption>
  * fetchSubmissions('test');
- * @example <caption> With `MMM dd, yyyy h:mm:ss a` date format</caption>
+ * @example <caption> With SurveyCTO date format (UTC)</caption>
  * fetchSubmissions('test', { date: 'Apr 18, 2024 6:26:21 AM' });
- * @example <caption> With `unix timestamp` date format</caption>
- * fetchSubmissions('test', { date: '1444694400000' });
+ * @example <caption>Using a rolling cursor </caption>
+ * cursor((state) => state.cursor, { defaultValue: 'today' });
+ * fetchSubmissions('test', { date: (state) => state.cursor, format: 'csv' });
+ * cursor('now');
  * @example <caption> Formatting the results to CSV String</caption>
- * fetchSubmissions('test', { date: '1444694400000', format: 'csv' });
+ * fetchSubmissions('test', { format: 'csv' });
  * @example <caption> With reviewStatus filter</caption>
- * fetchSubmissions('test', {
- *   date: '1444694400',
- *   status: 'approved|rejected',
- * });
- * @example <caption> With access to the callback</caption>
+ * fetchSubmissions('test', { status: 'approved|rejected' });
+ * @example <caption> With a callback function</caption>
  * fetchSubmissions(
  *   'test',
  *   {
  *     date: 'Apr 18, 2024 6:26:21 AM',
- *     status: 'approved|rejected',
  *   },
  *   state => {
  *     console.log('Hello from the callback!');
  *     return state;
  *   }
  * );
+ * @public
  * @function
  * @param {string} formId - Form id
- * @param {FormSubmissionOptions} options - Form submission date, format, status parameters
+ * @param {FetchSubmissionOptions} options - Form submission date, format, status parameters
  * @param {function} callback - (Optional) Callback function
  * @returns {Operation}
  */
@@ -74,10 +79,12 @@ export function fetchSubmissions(formId, options, callback = s => s) {
       options
     );
 
-    const { date, format, status } = {
-      ...{ date: 0, format: 'json' },
-      ...resolvedOptions,
-    };
+    let { date = 0, format = 'json', status } = resolvedOptions;
+
+    if (date) {
+      // Ensure the incoming date is in surveyCTO `MMM dd, yyy h:mm:ss a`
+      date = convertDate(date);
+    }
 
     const path =
       format === 'csv'
@@ -107,13 +114,14 @@ export function fetchSubmissions(formId, options, callback = s => s) {
 }
 
 /**
- * Options provided to the SurveyCTO API request
+ * Options provided to request()
  * @typedef {Object} RequestOptions
- * @property {object} headers - An object of headers parameters.
- * @property {object} body - Body data to append to the request.
- * @property {object} query - An object of query parameters to be encoded into the URL.
- * @property {string} [method = GET] - The HTTP method to use. Defaults to `GET`
+ * @property {object} [headers] - An object of headers parameters.
+ * @property {object} [body] - Body data to append to the request.
+ * @property {object} [query] - An object of query parameters to be encoded into the URL.
+ * @property {string} [method = GET] - The HTTP method to use.
  */
+
 /**
  * Make a request in SurveyCTO API
  * @public
@@ -140,13 +148,37 @@ export function request(path, params, callback = s => s) {
   };
 }
 
+/**
+ * Sets `state.cursor` to a SurveyCTO `MMM dd, yyy h:mm:ss a` timestamp string.
+ * Supports natural language dates like `now`, `today`, `yesterday`, `n hours ago`, `n days ago`, and `start`,
+ * which will be converted into timestamp strings.
+ * See the usage guide at {@link https://docs.openfn.org/documentation/jobs/job-writing-guide#using-cursors}
+ * @public
+ * @example <caption>Use a cursor from state if present, or else use the default value</caption>
+ * cursor('today')
+ * fetchSubmissions('test', { date: $.cursor });
+ * @function
+ * @param {any} value - the cursor value. Usually an ISO date, natural language date, or page number
+ * @param {object} options - options to control the cursor.
+ * @param {string} options.key - set the cursor key. Will persist through the whole run.
+ * @param {any} options.defaultValue - the value to use if value is falsy
+ * @param {Function} options.format - custom formatter for the final cursor value
+ * @returns {Operation}
+ */
+export function cursor(value, options) {
+  const opts = {
+    format: convertDate,
+    ...options,
+  };
+  return commonCursor(value, opts);
+}
+
 export {
   fn,
   chunk,
   merge,
   field,
   fields,
-  cursor,
   dateFns,
   dataPath,
   parseCsv,

packages/surveycto/src/Utils.js

@@ -4,6 +4,7 @@ import {
   logResponse,
   makeBasicAuthHeader,
 } from '@openfn/language-common/util';
+import { formatInTimeZone } from 'date-fns-tz';
 
 const addBasicAuth = (configuration = {}, headers) => {
   const { username, password } = configuration;
@@ -50,3 +51,39 @@ export const requestHelper = (state, path, params, callback = s => s) => {
       throw err;
     });
 };
+
+export const dateRegex = /^(\w{3} \d{2}, \d{4} \d{2}:\d{2}(:\d{2})? (PM|AM))$/;
+
+/**
+ * This function will attempt to convert any date representation into
+ * a surveyCTO `MMM dd, yyy h:mm:ss a` string.
+ * Strings already in this format will be ignored, other strings will be parsed
+ * by the Date constructor.
+ * Number values should be epoch or unix timestamps and will be converted to strings
+ * @param {*} date a date in a string, number or Date format
+ * @returns
+ */
+export const convertDate = date => {
+  // If it's already in the right format, return it
+  if (typeof date === 'string' && dateRegex.test(date)) {
+    return date;
+  }
+
+  // Otherwise parse the input into a new date object
+  let dateObj = date;
+  if (typeof date === 'string' || typeof date === 'number') {
+    if (/^\d{10}$/.test(date)) {
+      // If the incoming date is a unit timestamp, just return it
+      dateObj = new Date(date * 1000);
+    } else {
+      dateObj = new Date(date);
+    }
+  }
+
+  // And return in the correct formatting (utc time)
+  return formatInTimeZone(
+    dateObj.toISOString(),
+    'UTC',
+    'MMM dd, yyy h:mm:ss a'
+  );
+};