More reasonably buffer writes (#15)

* More reasonably buffer writes * Fix DM_getPanel for postgres * Misc fixes * Quote version in release script * Switch to ace editor * Fix sidebar open/close icon * Fix for formatting
multiprocessio · Jun 30, 2021 · 68756e4 · 68756e4
1 parent c123d68
commit 68756e4
Show file tree

Hide file tree

Showing 17 changed files with 177 additions and 78 deletions.
diff --git a/desktop/constants.ts b/desktop/constants.ts
@@ -4,6 +4,8 @@ import path from 'path';
 export const DISK_ROOT = path.join(os.homedir(), 'DataStationProjects');
 
 export const PROJECT_EXTENSION = 'dsproj';
-export const RESULTS_FILE = '.results';
+export const RESULTS_FILE = path.join(DISK_ROOT, '.results');
 
 export const DSPROJ_FLAG = '--dsproj';
+
+export const SYNC_PERIOD = 3000; // seconds
diff --git a/desktop/scripts/release.build b/desktop/scripts/release.build
@@ -1,8 +1,8 @@
 git clean -xid
 yarn
 yarn build-ui
-prepend "window.DS_CONFIG_DEBUG = false; window.DS_CONFIG_VERSION={arg0};" build/ui.js
+prepend "window.DS_CONFIG_VERSION='{arg0}';" build/ui.js
 yarn build-desktop
-prepend "global.DS_CONFIG_DEBUG = false; global.DS_CONFIG_VERSION={arg0};" build/desktop.js
+prepend "global.DS_CONFIG_VERSION='{arg0}';" build/desktop.js
 yarn electron-packager --overwrite --out=releases --build-version={arg0} .
 zip -r releases/{os}-{arch}-{arg0}.zip "releases/DataStation Community Edition-{os}-{arch}"
diff --git a/desktop/settings.ts b/desktop/settings.ts
@@ -45,7 +45,6 @@ export class Settings {
   }
 
   save() {
-    console.log('Saving settings');
     return fs.writeFile(this.file, JSON.stringify(this));
   }
 

diff --git a/desktop/store.ts b/desktop/store.ts
@@ -1,17 +1,62 @@
-import fs from 'fs/promises';
+import fs from 'fs';
+import fsPromises from 'fs/promises';
 import path from 'path';
 
-import { ProjectState } from '../shared/state';
+import { IDDict, ProjectState } from '../shared/state';
 
-import { DISK_ROOT, PROJECT_EXTENSION, RESULTS_FILE } from './constants';
+import {
+  DISK_ROOT,
+  PROJECT_EXTENSION,
+  RESULTS_FILE,
+  SYNC_PERIOD,
+} from './constants';
+
+const buffers: IDDict<{
+  contents: string;
+  timeout: ReturnType<typeof setTimeout>;
+}> = {};
+export function writeFileBuffered(name: string, contents: string) {
+  if (buffers[name]) {
+    clearTimeout(buffers[name].timeout);
+  }
+  buffers[name] = {
+    contents,
+    timeout: null,
+  };
+  buffers[name].timeout = setTimeout(() => {
+    fs.writeFileSync(name, contents);
+    console.info('Wrote ' + name);
+    delete buffers[name];
+  }, SYNC_PERIOD);
+}
+
+function flushUnwritten(...args: any[]) {
+  if (args && args.length > 1) {
+    // Otherwise errors are masked
+    console.error(args);
+  }
+
+  Object.keys(buffers).map((fileName: string) => {
+    clearTimeout(buffers[fileName].timeout);
+    // Must be a synchronous write in this 'exit' context
+    // https://nodejs.org/api/process.html#process_event_exit
+    fs.writeFileSync(fileName, buffers[fileName].contents);
+    console.trace('Flushed ' + fileName);
+    delete buffers[fileName];
+  });
+}
+// There doesn't seem to be a catchall signal
+['exit', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'SIGINT'].map((sig) =>
+  process.on(sig, flushUnwritten)
+);
 
 export const storeHandlers = [
   {
     resource: 'getProjectState',
     handler: async (projectId: string) => {
       const fileName = await ensureProjectFile(projectId);
       try {
-        const f = await fs.readFile(fileName);
+        const f = await fsPromises.readFile(fileName);
         return JSON.parse(f.toString());
       } catch (e) {
         console.error(e);
@@ -23,7 +68,7 @@ export const storeHandlers = [
     resource: 'updateProjectState',
     handler: async (projectId: string, newState: ProjectState) => {
       const fileName = await ensureProjectFile(projectId);
-      return fs.writeFile(fileName, JSON.stringify(newState));
+      return writeFileBuffered(fileName, JSON.stringify(newState));
     },
   },
   {
@@ -33,8 +78,9 @@ export const storeHandlers = [
         return;
       }
       const fileName = await ensureFile(RESULTS_FILE);
-      await fs.writeFile(fileName, JSON.stringify(results));
-      console.log('Project synced');
+      // Don't use buffered write
+      await fsPromises.writeFile(fileName, JSON.stringify(results));
+      console.info('Project synced');
     },
   },
 ];
@@ -49,9 +95,7 @@ export function ensureProjectFile(projectId: string) {
 }
 
 export async function ensureFile(f: string) {
-  if (path.isAbsolute(f)) {
-    return f;
-  }
-  await fs.mkdir(DISK_ROOT, { recursive: true });
-  return path.join(DISK_ROOT, f);
+  let root = path.isAbsolute(f) ? path.dirname(f) : DISK_ROOT;
+  await fsPromises.mkdir(root, { recursive: true });
+  return path.isAbsolute(f) ? f : path.join(DISK_ROOT, f);
 }
diff --git a/package.json b/package.json
@@ -16,9 +16,9 @@
     "test-licenses": "yarn license-checker --production --onlyAllow 'MIT;Apache-2.0;ISC;BSD-3-Clause;MIT*;BSD-2-Clause'"
   },
   "dependencies": {
+    "ace-builds": "^1.4.12",
     "chart.js": "^3.3.2",
     "json-stringify-safe": "^5.0.1",
-    "lodash.throttle": "^4.1.1",
     "mysql2": "^2.2.5",
     "node-fetch": "^2.6.1",
     "pako": "^2.0.3",
@@ -27,6 +27,7 @@
     "pg": "^8.6.0",
     "prismjs": "^1.24.0",
     "react": "^17.0.2",
+    "react-ace": "^9.4.1",
     "react-dom": "^17.0.2",
     "react-simple-code-editor": "^0.11.0",
     "sql.js": "^1.5.0",

diff --git a/type-overrides/lodash.d.ts b/type-overrides/lodash.d.ts
diff --git a/ui/Connectors.tsx b/ui/Connectors.tsx
@@ -19,6 +19,7 @@ export function Connectors({
     <div className="connectors">
       {state.connectors?.map((dc: ConnectorInfo, i: number) => (
         <Connector
+          key={dc.id}
           connector={dc}
           updateConnector={(dc: ConnectorInfo) => updateConnector(i, dc)}
           deleteConnector={deleteConnector.bind(null, i)}

diff --git a/ui/GraphPanel.tsx b/ui/GraphPanel.tsx
@@ -155,6 +155,7 @@ export function GraphPanelDetails({
       </div>
       <div className="form-row">
         <PanelSourcePicker
+          currentPanel={panel.id}
           panels={panels}
           value={panel.graph.panelSource}
           onChange={(value: number) => {

diff --git a/ui/Panel.tsx b/ui/Panel.tsx
@@ -411,6 +411,7 @@ export function Panel({
                   body
                 ) : (
                   <CodeEditor
+                    id={panel.id}
                     onKeyDown={keyboardShortcuts}
                     value={panel.content}
                     onChange={(value: string) => {

diff --git a/ui/PanelSourcePicker.tsx b/ui/PanelSourcePicker.tsx
@@ -8,22 +8,28 @@ export function PanelSourcePicker({
   value,
   onChange,
   panels,
+  currentPanel,
 }: {
   value: number;
   onChange: (n: number) => void;
   panels: Array<PanelInfo>;
+  currentPanel: string;
 }) {
   return (
     <Select
       label="Panel Source"
       onChange={(value: string) => onChange(+value)}
       value={value.toString()}
     >
-      {panels.map((panel, i) => (
-        <option key={panel.id} value={i.toString()}>
-          [#{i}] {panel.name}
-        </option>
-      ))}
+      {panels
+        .map((panel, i) =>
+          panel.id === currentPanel ? null : (
+            <option key={panel.id} value={i.toString()}>
+              [#{i}] {panel.name}
+            </option>
+          )
+        )
+        .filter(Boolean)}
     </Select>
   );
 }
diff --git a/ui/ProjectStore.ts b/ui/ProjectStore.ts
@@ -1,5 +1,4 @@
 import { IpcRenderer } from 'electron';
-import throttle from 'lodash.throttle';
 import * as React from 'react';
 
 import { ProjectState, DEFAULT_PROJECT, PanelResult } from '../shared/state';
@@ -46,19 +45,12 @@ export interface ProjectStore {
   get: (projectId: string) => Promise<ProjectState>;
 }
 
-export function makeStore(mode: string, syncMillis: number = 2000) {
+export function makeStore(mode: string) {
   const storeClass = {
     desktop: DesktopIPCStore,
     browser: LocalStorageStore,
   }[mode];
-  const store = new storeClass();
-  if (syncMillis) {
-    store.update = throttle<[string, ProjectState], Promise<void>>(
-      store.update.bind(store),
-      syncMillis
-    );
-  }
-  return store;
+  return new storeClass();
 }
 
 export const ProjectContext =

diff --git a/ui/SQLPanel.tsx b/ui/SQLPanel.tsx
@@ -30,10 +30,13 @@ export async function evalSQLPanel(
       return `t${replacements.length - 1}`;
     });
 
+    const valueQuote = panel.sql.type === 'in-memory' ? "'" : '"';
+    const columnQuote = panel.sql.type === 'mysql' ? '`' : '"';
+
     let ctePrefix = '';
     replacements.forEach((panelIndex: number, tableIndex: number) => {
       const results = panelResults[panelIndex];
-      if (results.exception || results.value.length === 0) {
+      if (!results || results.exception || results.value.length === 0) {
         // TODO: figure out how to query empty panels. (How to resolve column names for SELECT?)
         throw new Error('Cannot query empty results in panel ${panelIndex}');
       }
@@ -56,18 +59,27 @@ export async function evalSQLPanel(
               return 'null';
             }
 
-            // Replace double quoted strings with single quoted
+            // Replace double quoted strings with correct quote type
             if (stringified[0] === '"') {
               stringified = stringified.substring(1, stringified.length - 1);
             }
 
-            // Make sure to escape embedded single quotes
-            return `'${stringified.replace("'", "''")}'`;
+            // Make sure to escape embedded quotes
+            return (
+              valueQuote +
+              stringified.replace(valueQuote, valueQuote + valueQuote) +
+              valueQuote
+            );
           });
         }
       );
       const quotedColumns = columns
-        .map((c: string) => `'${c.replace("'", "''")}'`)
+        .map(
+          (c: string) =>
+            columnQuote +
+            c.replace(columnQuote, columnQuote + columnQuote) +
+            columnQuote
+        )
         .join(', ');
       const values = valuesAsSQLStrings
         .map((v: Array<string>) => `(${v.join(', ')})`)
@@ -76,7 +88,7 @@ export async function evalSQLPanel(
       if (tableIndex === 0) {
         prefix = 'WITH ';
       }
-      ctePrefix = `${ctePrefix}${prefix}t${tableIndex}(${quotedColumns}) AS (SELECT * FROM (VALUES ${values}))`;
+      ctePrefix = `${ctePrefix}${prefix}t${tableIndex}(${quotedColumns}) AS (SELECT * FROM (VALUES ${values}) t${tableIndex})`;
     });
     content = ctePrefix + ' ' + content;
 

diff --git a/ui/Sidebar.tsx b/ui/Sidebar.tsx
@@ -8,13 +8,13 @@ export function Sidebar({ children }: { children: React.ReactNode }) {
   return (
     <div className={`section sidebar ${!expanded ? 'sidebar--collapsed' : ''}`}>
       <div className="title vertical-align-center">
-        <span className="material-icons-outlined">manage_search</span>;
+        <span className="material-icons-outlined">manage_search</span>
         <Button
           icon
           className="flex-right"
           onClick={() => setExpanded(!expanded)}
         >
-          keyboard_arrow_right
+          {expanded ? 'keyboard_arrow_left' : 'keyboard_arrow_right'}
         </Button>
       </div>
       {expanded && children}

diff --git a/ui/TablePanel.tsx b/ui/TablePanel.tsx
@@ -26,6 +26,7 @@ export function TablePanelDetails({
     <React.Fragment>
       <div className="form-row">
         <PanelSourcePicker
+          currentPanel={panel.id}
           panels={panels}
           value={panel.table.panelSource}
           onChange={(value: number) => {

diff --git a/ui/component-library/CodeEditor.tsx b/ui/component-library/CodeEditor.tsx
@@ -1,10 +1,9 @@
 import * as React from 'react';
-import Editor from 'react-simple-code-editor';
-import { highlight, languages, Language } from 'prismjs/components/prism-core';
-import 'prismjs/components/prism-python';
-import 'prismjs/components/prism-sql';
-import 'prismjs/components/prism-clike';
-import 'prismjs/components/prism-javascript';
+import AceEditor from 'react-ace';
+import 'ace-builds/src-noconflict/mode-sql';
+import 'ace-builds/src-noconflict/mode-javascript';
+import 'ace-builds/src-noconflict/mode-python';
+import 'ace-builds/src-noconflict/theme-github';
 
 export function CodeEditor({
   value,
@@ -13,31 +12,43 @@ export function CodeEditor({
   disabled,
   onKeyDown,
   language,
+  id,
 }: {
   value: string;
   onChange: (value: string) => void;
   className?: string;
   disabled?: boolean;
   onKeyDown?: (e: React.KeyboardEvent) => void;
   language: string;
+  id: string;
 }) {
-  const highlightWithLineNumbers = (input: string, language: Language) =>
-    highlight(input, language)
-      .split('\n')
-      .map((line, i) => `<span class='editorLineNumber'>${i + 1}</span>${line}`)
-      .join('\n');
   return (
     <div className="editor-container">
-      <Editor
+      <AceEditor
+        mode={language}
+        theme="github"
+        onChange={onChange}
+        name={id}
         value={value}
-        onValueChange={onChange}
-        highlight={(code) =>
-          highlightWithLineNumbers(code, languages[language])
-        }
-        onKeyDown={onKeyDown}
         className={className}
-        disabled={disabled}
-        textareaId="codeArea"
+        readOnly={disabled}
+        width="100%"
+        fontSize="1rem"
+        commands={[
+          // AceEditor wants commands in this way but outside here we
+          // only support onKeyDown so doing this funky translation.
+          {
+            name: 'ctrl-enter',
+            bindKey: { win: 'Ctrl-Enter', mac: 'Ctrl-Enter' },
+            exec: () =>
+              onKeyDown({
+                ctrlKey: true,
+                code: 'Enter',
+              } as React.KeyboardEvent),
+          },
+        ]}
+        showGutter={true}
+        keyboardHandler="emacs"
       />
     </div>
   );