Add audit page, SQLite followup fixes, fixed side navigation, fix std…

…out/stderr (#208)

* Fix for insert

* Fixes

* Add navigation

* Add all routes

* Clean up navigation

* Final fixes

* Try reoderd

* Drop settings from desktop menu

* Move to nanoid

* Work on positions

* Move to nanoid

* Break out editor page

* More progress on state tests

* More progress on views and panelPositions

* Hook up eslint

* Fin for eslint

* Fixes for file panel

* Some stuff working

* Fix for reorder test

* Drop console logs, update dark mode

* Fixes for ca certs

* Add new panel dropdown

* Fix graph/table shortcut button spacing

* No more usedebouncedstate

* Fix for onblur

* Fixes for format

* Fix for gofmt

* Switch to useDebouncedCallback

* Fix for eslint

* Add dashboard/export header

* Make navigation a function of input pages

* History basic showing

* Fixes for eslint

* Fix for fmt

* Eval tests are failing

* Fixes for tests

* Start react ee tests

* Bump coverage

* Start react ee tests

* Rename file

* Fix for copyright

* Fix for some more tests

* Fix for test

* Fixes for in-memory mode

* Fix for localstorage store

* Fix for new project

* Attempt at supporting resize on create

* http.location

* Fixes for tsc

* Add notification icon to stdout/stderr when present

* Smaller

* Delete results file before evalling

* Always close file

* No need for deleting results

* Fixes for errors

* Try using apckage managers for go

* Fix for mac

* Fix for error check

* Continue manual install of go on mac

* Fix for EOL on windows

* Fix for new project on desktop

* Its about the pathname

* Set loading

.eslintignore

@@ -0,0 +1,2 @@
+ui/scripts/*
+ui/state.test.js

.eslintrc

@@ -0,0 +1,28 @@
+{
+    "root": true,
+    "parser": "@typescript-eslint/parser",
+    "plugins": [
+	"@typescript-eslint",
+	"jest"
+    ],
+    "extends": [
+	"eslint:recommended",
+	"plugin:react/recommended",
+	"plugin:@typescript-eslint/eslint-recommended",
+	"plugin:@typescript-eslint/recommended",
+	"plugin:react-hooks/recommended"
+    ],
+    "rules": {
+    "react/jsx-no-target-blank": 'off',
+    "react/no-children-prop": 'off',
+	"@typescript-eslint/no-explicit-any": 'off',
+	'@typescript-eslint/no-unused-vars': [
+	    'error',
+	    { 
+		argsIgnorePattern: '^_',
+		varsIgnorePattern: '^_',
+		caughtErrorsIgnorePattern: '^_',
+	    },
+	]
+    }
+}

.github/workflows/pull_requests.yml

@@ -28,6 +28,7 @@ jobs:
 
     - run: ./scripts/ci/prepare_linux.sh --integration-tests
     - run: yarn format
+    - run: yarn eslint ui
     - run: yarn tsc
     - run: cd runner && gofmt -w .
     - run: yarn build-language-definitions

GOOD_FIRST_PROJECTS.md

@@ -21,10 +21,9 @@ and dsq for these tasks to make sense.
   * Make sure there’s a library for every language
   * Figure out how to embed the library inside DataStation
 * More databases
-  * IBM DB2, Neo4j, Apache Presto/Trino, Meilisearch, Apache Hive, Apache Druid, Apache Pinot
+  * IBM DB2, Neo4j, Apache Presto/Trino, Meilisearch, Apache Hive, Apache Druid, Apache Pinot, Quickwit
 * Add a new supported log format
   * Example: logfmt
-* Fakegen help text
 
 ## Medium
 

desktop/bridge.ts

@@ -0,0 +1,53 @@
+import { contextBridge, ipcRenderer, IpcRendererEvent } from 'electron';
+import { RPC_ASYNC_REQUEST, RPC_ASYNC_RESPONSE } from '../shared/constants';
+import log from '../shared/log';
+import { Endpoint, IPCRendererResponse, WindowAsyncRPC } from '../shared/rpc';
+
+let messageNumber = -1;
+
+export function bridgeAsyncRPC() {
+  const asyncRPC: WindowAsyncRPC = async function <
+    Request,
+    Response = void,
+    EndpointT extends string = Endpoint
+  >(resource: EndpointT, projectId: string, body: Request): Promise<Response> {
+    const payload = {
+      // Assign a new message number
+      messageNumber: ++messageNumber,
+      resource,
+      body,
+      projectId,
+    };
+    ipcRenderer.send(RPC_ASYNC_REQUEST, payload);
+
+    const result = await new Promise<IPCRendererResponse<Response>>(
+      (resolve, reject) => {
+        try {
+          ipcRenderer.once(
+            `${RPC_ASYNC_RESPONSE}:${payload.messageNumber}`,
+            (e: IpcRendererEvent, response: IPCRendererResponse<Response>) =>
+              resolve(response)
+          );
+        } catch (e) {
+          reject(e);
+        }
+      }
+    );
+
+    if (result.kind === 'error') {
+      try {
+        throw result.error;
+      } catch (e) {
+        // The result.error object isn't a real Error at this point with
+        // prototype after going through serialization. So throw it to get
+        // a real Error instance that has full info for logs.
+        log.error(e);
+        throw e;
+      }
+    }
+
+    return result.body;
+  };
+
+  contextBridge.exposeInMainWorld('asyncRPC', asyncRPC);
+}

desktop/panel/eval.ts

@@ -1,18 +1,20 @@
 import { execFile } from 'child_process';
 import fs from 'fs';
 import jsesc from 'jsesc';
+import circularSafeStringify from 'json-stringify-safe';
 import { EOL } from 'os';
 import path from 'path';
 import { preview } from 'preview';
 import { shape, Shape } from 'shape';
 import { file as makeTmpFile } from 'tmp-promise';
-import * as uuid from 'uuid';
 import {
   Cancelled,
   EVAL_ERRORS,
   InvalidDependentPanelError,
+  NoResultError,
 } from '../../shared/errors';
 import log from '../../shared/log';
+import { newId } from '../../shared/object';
 import { PanelBody } from '../../shared/rpc';
 import {
   ConnectorInfo,
@@ -131,9 +133,7 @@ export async function evalInSubprocess(
       ensureFile(path.join(CODE_ROOT, 'coverage', 'fake.cov'));
       args.unshift('-test.run');
       args.unshift('^TestRunMain$');
-      args.unshift(
-        '-test.coverprofile=coverage/gorunner.' + uuid.v4() + '.cov'
-      );
+      args.unshift('-test.coverprofile=coverage/gorunner.' + newId() + '.cov');
     }
 
     log.info(`Launching "${base} ${args.join(' ')}"`);
@@ -189,20 +189,31 @@ export async function evalInSubprocess(
       }
     });
 
-    const resultMeta = fs.readFileSync(tmp.path).toString();
-    let parsePartial = !resultMeta;
+    const resultMeta = JSON.parse(fs.readFileSync(tmp.path).toString());
+    let parsePartial = typeof resultMeta.preview === 'undefined';
     if (!parsePartial) {
-      const rm: Partial<PanelResult> = JSON.parse(resultMeta);
+      const rm: Partial<PanelResult> = resultMeta;
       // Case of existing Node.js runner
       return [rm, stderr];
     }
 
     // Case of new Go runner
     const projectResultsFile = getProjectResultsFile(projectName);
-    const rm: Partial<PanelResult> = parsePartialJSONFile(
-      projectResultsFile + panel.id
-    );
-    return [rm, stderr];
+    let rm: Partial<PanelResult>;
+    try {
+      rm = parsePartialJSONFile(projectResultsFile + panel.id);
+    } catch (e) {
+      if (!e.stdout) {
+        e.stdout = resultMeta.stdout;
+      }
+
+      if (e instanceof NoResultError && resultMeta.exception) {
+        // do nothing. The underlying exception is more interesting
+      } else {
+        throw e;
+      }
+    }
+    return [{ ...rm, ...resultMeta }, stderr];
   } finally {
     try {
       if (pid) {
@@ -352,20 +363,36 @@ export const makeEvalHandler = (subprocessEval?: {
         dispatch,
         subprocessEval
       );
-
-      // This is to handle "exceptions" within the runner.
-      if (res.exception) {
-        throw res.exception;
-      }
-      // The outer try-catch is to handle exceptions within this Node code
     } catch (e) {
       log.error(e);
       res.exception = e;
-      if (!EVAL_ERRORS.find((ee) => ee.name === e.name) && stderr) {
-        // Just a generic exception, we already caught all info in `stderr`, so just throw that.
-        res.exception = stderr;
+      if (res.exception.stdout) {
+        res.stdout = res.exception.stdout;
+        delete res.exception.stdout;
       }
     }
+
+    if (
+      res.exception &&
+      !EVAL_ERRORS.find((ee) => ee.name === res.exception.name) &&
+      stderr
+    ) {
+      // Just a generic exception, we already caught all info in `stderr`, so just store that.
+      res.exception = new Error(res.stdout || stderr);
+    }
+
+    // I'm not really sure why this is necessary but sometimes the
+    // exception comes out in an object that can't be stringified. And
+    // yet I don't believe there's any throwing of errors from the Go
+    // code.
+    if (res.exception && circularSafeStringify(res.exception) === '{}') {
+      res.exception = {
+        name: res.exception.name,
+        message: res.exception.message,
+        ...res.exception,
+      };
+    }
+
     res.lastRun = start;
     res.loading = false;
     res.elapsed = new Date().valueOf() - start.valueOf();

desktop/panel/program.test.js

@@ -1,4 +1,4 @@
-const path = require('path');
+const os = require('os');
 const { LANGUAGES } = require('../../shared/languages');
 const {
   InvalidDependentPanelError,
@@ -7,9 +7,6 @@ const {
 const { getProjectResultsFile } = require('../store');
 const fs = require('fs');
 const { LiteralPanelInfo, ProgramPanelInfo } = require('../../shared/state');
-const { updateProjectHandler } = require('../store');
-const { CODE_ROOT } = require('../constants');
-const { makeEvalHandler } = require('./eval');
 const { inPath, withSavedPanels, RUNNERS, VERBOSE } = require('./testutil');
 
 const TESTS = [
@@ -485,4 +482,84 @@ for (const subprocessName of RUNNERS) {
       }
     });
   });
+
+  describe('stdout/stderr capture', function () {
+    test('it captures print in successful program', async () => {
+      const pp = new ProgramPanelInfo(null, {
+        type: 'python',
+        content: 'print(1002)\nDM_setPanel(1)',
+      });
+
+      let finished = false;
+      const panels = [pp];
+      await withSavedPanels(
+        panels,
+        async (project, dispatch) => {
+          const savedProject = await dispatch({
+            resource: 'getProject',
+            projectId: project.projectName,
+            body: {
+              projectId: project.projectName,
+            },
+          });
+          expect(savedProject.pages[0].panels[0].resultMeta.stdout).toBe(
+            '1002' + os.EOL
+          );
+          finished = true;
+        },
+        {
+          evalPanels: true,
+          subprocessName,
+        }
+      );
+
+      if (!finished) {
+        throw new Error('Callback did not finish');
+      }
+    });
+
+    test('it captures print in unsuccessful program', async () => {
+      const pp = new ProgramPanelInfo(null, {
+        type: 'python',
+        content: 'print("hey there")\nraise Exception(1)',
+      });
+
+      let finished = false;
+      const panels = [pp];
+      try {
+        await withSavedPanels(
+          panels,
+          async (project, dispatch) => {
+            finished = true;
+          },
+          {
+            evalPanels: true,
+            subprocessName,
+          }
+        );
+      } catch (e) {
+        const savedProject = await e.dispatch({
+          resource: 'getProject',
+          projectId: e.project.projectName,
+          body: {
+            projectId: e.project.projectName,
+          },
+        });
+        expect(
+          savedProject.pages[0].panels[0].resultMeta.stdout.startsWith(
+            'hey there' + os.EOL + 'Traceback'
+          )
+        ).toBe(true);
+        if (e instanceof Error || e.name === 'Error') {
+          finished = true;
+        } else {
+          throw e;
+        }
+      }
+
+      if (!finished) {
+        throw new Error('Callback did not finish');
+      }
+    });
+  });
 }

desktop/panel/testutil.js

@@ -63,6 +63,7 @@ exports.updateProject = async function (project, opts) {
         body: {
           data: page,
           position: i,
+          insert: true,
         },
       },
       true
@@ -78,6 +79,7 @@ exports.updateProject = async function (project, opts) {
           body: {
             data: panels[j],
             position: j,
+            insert: true,
           },
         },
         true
@@ -94,6 +96,7 @@ exports.updateProject = async function (project, opts) {
         body: {
           data: server,
           position: i,
+          insert: true,
         },
       },
       true
@@ -109,6 +112,7 @@ exports.updateProject = async function (project, opts) {
         body: {
           data: connector,
           position: i,
+          insert: true,
         },
       },
       true
@@ -191,6 +195,9 @@ exports.withSavedPanels = async function (
           dispatch
         );
         if (res.exception) {
+          // So that callers can get access to project data if needed
+          res.exception.project = project;
+          res.exception.dispatch = dispatch;
           throw res.exception;
         }