Add back play button to visual panels (#195)

* Add back play button to visual panels

* More tests for http and macros

* complex macro and install httpmirror on every platform

* Fix for fmt

* Fixes for athena

* Add go bin to path

* Copy httpmirror on mac too

desktop/panel/http.test.js

@@ -32,30 +32,36 @@ const baseline = JSON.parse(
 
 const USERDATA_FILES = ['json', 'xlsx', 'csv', 'parquet', 'jsonl', 'cjson'];
 const PORT = '9799';
+const PORT2 = '9798';
 
-let server;
+let server, server2;
 // Kill the existing server if it wasn't killed correctly already.
 beforeAll(async () => {
   // TODO: port this logic to other platforms...
-  if (process.platform === 'linux') {
-    try {
-      cp.execSync(
-        `bash -c "ps aux | grep 'http.server ${PORT}' | grep -v grep | awk '{print \\$2}' | xargs -I {} kill {}"`
-      );
-    } catch (e) {
-      console.error(e);
+  for (const port of [PORT, PORT2]) {
+    if (process.platform === 'linux') {
+      try {
+        cp.execSync(
+          `bash -c "ps aux | grep 'http.server ${port}' | grep -v grep | awk '{print \\$2}' | xargs -I {} kill {}"`
+        );
+      } catch (e) {
+        console.error(e);
+      }
     }
   }
 
   // Start a new server for all tests
   server = spawn('python3', ['-m', 'http.server', PORT]);
+  server2 = spawn('httpmirror', [PORT2]);
 
-  server.stdout.on('data', (data) => {
-    console.log(data.toString());
-  });
+  [server, server2].forEach((server) => {
+    server.stdout.on('data', (data) => {
+      console.log(data.toString());
+    });
 
-  server.stderr.on('data', (data) => {
-    console.warn(data.toString());
+    server.stderr.on('data', (data) => {
+      console.warn(data.toString());
+    });
   });
 
   // Keep trying to connect to the server until it's ready
@@ -240,7 +246,7 @@ for (const subprocessName of RUNNERS) {
   });
 
   describe('http with macro', () => {
-    test('correct result', () => {
+    test('macro as url', () => {
       const lp = new LiteralPanelInfo({
         contentTypeInfo: { type: 'text/plain' },
         content: '/testdata/allformats/unknown',
@@ -269,6 +275,40 @@ for (const subprocessName of RUNNERS) {
         { evalPanels: true, subprocessName }
       );
     });
+
+    test('macro as body', () => {
+      const lp = new LiteralPanelInfo({
+        contentTypeInfo: { type: 'application/json' },
+        content: '[{"name": "jim", "age": 12}]',
+        name: 'Raw Data',
+      });
+
+      const hp = new HTTPPanelInfo(
+        '',
+        new HTTPConnectorInfo('', 'http://localhost:9798')
+      );
+      hp.http.http.method = 'POST';
+      hp.content = '{{ DM_getPanel("0") | json }}';
+      hp.http.http.contentTypeInfo.type = 'text/plain';
+
+      const panels = [lp, hp];
+
+      return withSavedPanels(
+        panels,
+        (project) => {
+          // Grab result
+          const value = JSON.parse(
+            fs
+              .readFileSync(getProjectResultsFile(project.projectName) + hp.id)
+              .toString()
+          );
+          expect([{ name: 'jim', age: 12 }]).toStrictEqual(
+            JSON.parse(value.split('\r\n\r\n')[1])
+          );
+        },
+        { evalPanels: true, subprocessName }
+      );
+    });
   });
 
   if (process.platform === 'linux') {
@@ -313,4 +353,5 @@ for (const subprocessName of RUNNERS) {
 
 afterAll(() => {
   process.kill(server.pid);
+  process.kill(server2.pid);
 });

runner/database_athena.go

@@ -8,6 +8,7 @@ import (
 	"github.com/multiprocessio/go-json"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/athena"
 )
@@ -35,7 +36,14 @@ func mapAthenaType(value, t string) interface{} {
 }
 
 func evalAthena(panel *PanelInfo, dbInfo DatabaseConnectorInfoDatabase, w io.Writer) error {
+	secret, err := dbInfo.Password.decrypt()
+	if err != nil {
+		return err
+	}
+
 	cfg := aws.NewConfig().WithRegion(dbInfo.Extra["aws_region"])
+	cfg.Credentials = credentials.NewStaticCredentials(dbInfo.Username, secret, "")
+
 	sess := session.Must(session.NewSession(cfg))
 
 	svc := athena.New(sess, cfg)

runner/file.go

@@ -317,8 +317,6 @@ func transformGeneric(in io.Reader, out io.Writer) error {
 		return err
 	}
 
-	var prev byte = ' '
-
 	for {
 		b, err := r.ReadByte()
 		if err == io.EOF {
@@ -329,19 +327,29 @@ func transformGeneric(in io.Reader, out io.Writer) error {
 			return err
 		}
 
-		if b == '"' && prev != '\\' {
-			err = o.WriteByte('\\')
-			if err != nil {
-				return err
-			}
+		// Escape necessary characters
+		switch b {
+		case '\b':
+			_, err = o.WriteString(`\b`)
+		case '\f':
+			_, err = o.WriteString(`\f`)
+		case '\n':
+			_, err = o.WriteString(`\n`)
+		case '\r':
+			_, err = o.WriteString(`\r`)
+		case '\t':
+			_, err = o.WriteString(`\t`)
+		case '"':
+			_, err = o.WriteString(`\"`)
+		case '\\':
+			_, err = o.WriteString(`\\`)
+		default:
+			err = o.WriteByte(b)
 		}
 
-		err = o.WriteByte(b)
 		if err != nil {
 			return err
 		}
-
-		prev = b
 	}
 
 	err = o.WriteByte('"')

runner/file_test.go

@@ -192,26 +192,19 @@ func Test_transformORCFile(t *testing.T) {
 }
 
 func Test_transformGeneric(t *testing.T) {
-	tests := []struct {
-		in  string
-		out interface{}
-	}{
-		{
-			in:  `abcdef`,
-			out: `abcdef`,
-		},
-		{
-			in:  `ab""cdef`,
-			out: `ab""cdef`,
-		},
+	tests := []string{
+		`abcdef`,
+		`ab""cdef`,
+		`ab
+cdef`,
 	}
 
 	for _, test := range tests {
 		inTmp, err := ioutil.TempFile("", "")
 		defer os.Remove(inTmp.Name())
 		assert.Nil(t, err)
 
-		inTmp.WriteString(test.in)
+		inTmp.WriteString(test)
 
 		outTmp, err := ioutil.TempFile("", "")
 		defer os.Remove(outTmp.Name())
@@ -226,7 +219,7 @@ func Test_transformGeneric(t *testing.T) {
 		err = json.Unmarshal(outTmpBs, &m)
 		assert.Nil(t, err)
 
-		assert.Equal(t, test.out, m)
+		assert.Equal(t, test, m)
 	}
 }
 

runner/http.go

@@ -201,6 +201,7 @@ func (ec EvalContext) evalHTTPPanel(project *ProjectState, pageIndex int, panel
 			sendBody: panel.Content != "" &&
 				(h.Method == http.MethodPut || h.Method == http.MethodPatch || h.Method == http.MethodPost),
 			customCaCerts: customCaCerts,
+			method:        h.Method,
 		})
 		if err != nil {
 			return err

scripts/ci/prepare_linux_integration_test_setup_only.sh

@@ -9,6 +9,9 @@ set -ex
 # Set up Julia, SSH, etc.
 sudo apt-get install -y jq julia openssh-server php
 
+# Set up http helper
+go install github.com/multiprocessio/httpmirror@latest
+
 # Set up coverage tools
 go install github.com/axw/gocov/gocov@v1.0.0
 go install github.com/wadey/gocovmerge@b5bfa59

scripts/ci/prepare_macos.sh

@@ -7,5 +7,7 @@ brew uninstall go@1.15
 brew install cmake jq go r julia node@16 npm
 brew link --overwrite node@16
 go install github.com/google/go-jsonnet/cmd/jsonnet@latest
+go install github.com/multiprocessio/httpmirror@latest
+cp ~/go/bin/httpmirror /usr/local/bin/httpmirror
 npm install --global yarn
 yarn

scripts/ci/prepare_windows.ps1

@@ -4,5 +4,7 @@ Join-Path (Resolve-Path ~).Path "scoop\shims" >> $Env:GITHUB_PATH
 scoop install nodejs@16 cmake python yarn zip jq curl go r julia
 yarn
 go install github.com/google/go-jsonnet/cmd/jsonnet@latest
+go install github.com/multiprocessio/httpmirror@latest
+Join-Path (Resolve-Path ~).Path "go\bin" >> $Env:GITHUB_PATH
 
 New-Alias zip 7z

ui/Panel.tsx

@@ -314,7 +314,11 @@ export function Panel({
       className={`panel ${fullScreen === panel.id ? 'panel--fullscreen' : ''} ${
         hidden ? 'panel--hidden' : ''
       } ${
-        panelUIDetails.body === null && !results.exception ? 'panel--empty' : ''
+        (panelUIDetails.body === null ||
+          (panelUIDetails.hideBody && panelUIDetails.hideBody(panel))) &&
+        !results.exception
+          ? 'panel--empty'
+          : ''
       } ${results.loading ? 'panel--loading' : ''}`}
       tabIndex={1001}
       ref={panelRef}
@@ -431,27 +435,25 @@ export function Panel({
                   'Run to apply changes'
                 )}
               </span>
-              {!VISUAL_PANELS.includes(panel.type) ? (
-                <span
-                  title={
-                    results.loading
-                      ? killable
-                        ? 'Cancel'
-                        : 'Running'
-                      : 'Evaluate Panel (Ctrl-Enter)'
+              <span
+                title={
+                  results.loading
+                    ? killable
+                      ? 'Cancel'
+                      : 'Running'
+                    : 'Evaluate Panel (Ctrl-Enter)'
+                }
+              >
+                <Button
+                  icon
+                  onClick={() =>
+                    killable ? killProcess() : reevalPanel(panel.id)
                   }
+                  type="primary"
                 >
-                  <Button
-                    icon
-                    onClick={() =>
-                      killable ? killProcess() : reevalPanel(panel.id)
-                    }
-                    type="primary"
-                  >
-                    {results.loading ? <IconPlayerPause /> : <IconPlayerPlay />}
-                  </Button>
-                </span>
-              ) : null}
+                  {results.loading ? <IconPlayerPause /> : <IconPlayerPlay />}
+                </Button>
+              </span>
               <span title="Full screen mode">
                 <Button
                   icon
@@ -519,14 +521,17 @@ export function Panel({
             <ErrorBoundary className="panel-body">
               <div className="flex">
                 <div className="panel-body">
-                  {panelUIDetails.body && (
-                    <panelUIDetails.body
-                      panel={panel}
-                      keyboardShortcuts={keyboardShortcuts}
-                      panels={panels}
-                      updatePanel={updatePanel}
-                    />
-                  )}
+                  {panelUIDetails.body &&
+                    (panelUIDetails.hideBody
+                      ? !panelUIDetails.hideBody(panel)
+                      : true) && (
+                      <panelUIDetails.body
+                        panel={panel}
+                        keyboardShortcuts={keyboardShortcuts}
+                        panels={panels}
+                        updatePanel={updatePanel}
+                      />
+                    )}
                   {results.exception instanceof PanelPlayWarning ? (
                     <PanelPlayWarningWithLinks
                       panels={panels}

ui/components/ContentTypePicker.tsx

@@ -9,17 +9,19 @@ export function ContentTypePicker({
   onChange,
   disableAutoDetect,
   inMemoryEval,
+  label,
 }: {
   value: ContentTypeInfo;
   onChange: (v: ContentTypeInfo) => void;
   disableAutoDetect?: boolean;
   inMemoryEval: boolean;
+  label?: string;
 }) {
   return (
     <React.Fragment>
       <div className="form-row">
         <Select
-          label="Content Type"
+          label={label || 'Content Type'}
           value={value.type}
           onChange={(type: string) => {
             if (type === 'null') {