Pack program info rather than reading JSON files dynamically in Go ru…

…nner (#129)

* Pack program info

* Add basic integration tests for dsq

* Add test script

* Debug

* Create results directory if not exists

* Generate packed info for diff

* Drop all tests but dsq

* Run tests on win/mac/linux

* No integration test version

* With powershell

* Switch go

* Correct image

* Uninstall 1.15

* Drop cat

* Add ids incase

* Add more comments

* Dont fail if dir exists

* Dont fail if dir exists

* Debugging

* Run windows test in bash

* Rearrange tests

* Try with verbose

* More debugging

* Try another pipe detection

* Verbose again

* Add back old tests

* Add support for pipe fallback

* one more whitespace

* one more whitespace

* one more whitespace

* Fix for fmt

.github/workflows/pull_requests.yml

@@ -35,6 +35,7 @@ jobs:
     - run: yarn format
     - run: yarn tsc
     - run: cd runner && gofmt -w .
+    - run: ./runner/scripts/generate_program_type_info.sh
     - run: ./scripts/fail_on_diff.sh
     - run: yarn test-licenses
     # Needed so we can have ./build/desktop_runner.js and ./build/go_desktop_runner ready for tests
@@ -166,3 +167,41 @@ jobs:
     - run: cd runner && go test -cover
     - run: yarn release-desktop 0.0.0-e2etest
     - run: yarn e2e-test
+
+  dsq-tests-ubuntu:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+      with:
+        ref: ${{ github.ref }}
+
+    - run: ./scripts/ci/prepare_linux.sh
+    - run: cd runner && go build -o ../dsq cmd/dsq/main.go
+    - run: ./runner/cmd/dsq/scripts/test.sh
+
+  dsq-tests-windows:
+    runs-on: windows-latest
+
+    steps:
+    - uses: actions/checkout@master
+      with:
+        ref: ${{ github.ref }}
+
+    - run: ./scripts/ci/prepare_windows.ps1
+      shell: pwsh
+    - run: cd runner && go build -o ../dsq cmd/dsq/main.go
+    - run: ./runner/cmd/dsq/scripts/test.sh
+      shell: bash
+
+  dsq-tests-macos:
+    runs-on: macos-latest
+
+    steps:
+    - uses: actions/checkout@master
+      with:
+        ref: ${{ github.ref }}
+
+    - run: ./scripts/ci/prepare_macos.sh
+    - run: cd runner && go build -o ../dsq cmd/dsq/main.go
+    - run: ./runner/cmd/dsq/scripts/test.sh

.gitignore

@@ -12,4 +12,5 @@ TAGS
 all.out
 runner/report
 runner/runner
-runner/runner.exe
+runner/runner.exe
+dsq

runner/cmd/dsq/main.go

@@ -5,16 +5,24 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"strings"
 
 	"github.com/multiprocessio/datastation/runner"
+
+	"github.com/google/uuid"
 )
 
 func isinpipe() bool {
 	fi, _ := os.Stdin.Stat()
-	return (fi.Mode() & os.ModeCharDevice) == 0
+	if fi == nil {
+		return false
+	}
+
+	// This comes back incorrect in automated environments like Github Actions.
+	return !(fi.Mode()&os.ModeNamedPipe == 0)
 }
 
 func resolveContentType(fileExtensionOrContentType string) string {
@@ -31,20 +39,33 @@ func getResult(res interface{}) error {
 	out := bytes.NewBuffer(nil)
 	arg := firstNonFlagArg
 
-	var internalErr error
-	if isinpipe() {
-		mimetype := resolveContentType(arg)
-		if mimetype == "" {
-			return fmt.Errorf(`First argument when used in a pipe should be file extension or content type. e.g. 'cat test.csv | dsq csv "SELECT * FROM {}"'`)
+	mimetype := resolveContentType(arg)
+	if mimetype == "" {
+		return fmt.Errorf(`First argument when used in a pipe should be file extension or content type. e.g. 'cat test.csv | dsq csv "SELECT * FROM {}"'`)
+	}
+
+	cti := runner.ContentTypeInfo{Type: mimetype}
+
+	// isinpipe() is sometimes incorrect. If the first arg
+	// is a file, fall back to acting like this isn't in a
+	// pipe.
+	runAsFile := !isinpipe()
+	if !runAsFile && cti.Type == arg {
+		if _, err := os.Stat(arg); err == nil {
+			runAsFile = true
 		}
+	}
 
-		cti := runner.ContentTypeInfo{Type: mimetype}
-		internalErr = runner.TransformReader(os.Stdin, "", cti, out)
+	if !runAsFile {
+		err := runner.TransformReader(os.Stdin, "", cti, out)
+		if err != nil {
+			return err
+		}
 	} else {
-		internalErr = runner.TransformFile(arg, runner.ContentTypeInfo{}, out)
-	}
-	if internalErr != nil {
-		return internalErr
+		err := runner.TransformFile(arg, runner.ContentTypeInfo{}, out)
+		if err != nil {
+			return err
+		}
 	}
 
 	decoder := json.NewDecoder(out)
@@ -97,8 +118,17 @@ func main() {
 		ResultMeta: runner.PanelResult{
 			Shape: *shape,
 		},
+		Id:   uuid.New().String(),
+		Name: uuid.New().String(),
+	}
+
+	projectTmp, err := ioutil.TempFile("", "dsq-project")
+	if err != nil {
+		log.Fatal(err)
 	}
+	defer os.Remove(projectTmp.Name())
 	project := &runner.ProjectState{
+		Id: projectTmp.Name(),
 		Pages: []runner.ProjectPage{
 			{
 				Panels: []runner.PanelInfo{p0},
@@ -117,6 +147,8 @@ func main() {
 	panel := &runner.PanelInfo{
 		Type:    runner.DatabasePanel,
 		Content: query,
+		Id:      uuid.New().String(),
+		Name:    uuid.New().String(),
 		DatabasePanelInfo: &runner.DatabasePanelInfo{
 			Database: runner.DatabasePanelInfoDatabase{
 				ConnectorId: connector.Id,

runner/cmd/dsq/scripts/test.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -e
+
+types="csv parquet json"
+
+for t in $types; do
+    echo "Testing $t (pipe)."
+    sqlcount="$(cat ./testdata/userdata.$t | ./dsq $t 'SELECT COUNT(1) AS c FROM {}' | jq '.[0].c')"
+    if [[ "$sqlcount" != "1000" ]]; then
+	echo "Bad SQL count for $t (pipe). Expected 1000, got $sqlcount."
+	exit 1
+    else
+	echo "Pipe $t test successful."
+    fi
+
+    echo "Testing $t (file)."
+    sqlcount="$(./dsq ./testdata/userdata.$t 'SELECT COUNT(1) AS c FROM {}' | jq '.[0].c')"
+    if [[ "$sqlcount" != "1000" ]]; then
+	echo "Bad SQL count for $t (file). Expected 1000, got $sqlcount."
+	exit 1
+    else
+	echo "File $t test successful."
+    fi
+done

runner/file.go

@@ -69,6 +69,8 @@ func withJSONArrayOutWriter(w io.Writer, cb func(w *JSONArrayWriter) error) erro
 }
 
 func openTruncate(out string) (*os.File, error) {
+	base := filepath.Dir(out)
+	_ = os.Mkdir(base, os.ModePerm)
 	return os.OpenFile(out, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, os.ModePerm)
 }
 

runner/program.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path"
 	"strings"
 
 	"github.com/google/uuid"
@@ -90,7 +89,7 @@ func (ec EvalContext) evalProgramPanel(project *ProjectState, pageIndex int, pan
 	}
 
 	var p ProgramEvalInfo
-	err := readJSONFileInto(path.Join("shared", "languages", string(panel.Program.Type)+".json"), &p)
+	err := json.Unmarshal([]byte(packedProgramTypeInfo[panel.Program.Type]), &p)
 	if err != nil {
 		return err
 	}

runner/program_info.go

@@ -0,0 +1,11 @@
+package runner
+
+// GENERATED BY ./runner/scripts/generate_program_type_info.sh. DO NOT MODIFY.
+
+var packedProgramTypeInfo = map[SupportedLanguages]string{
+	"javascript": `{"id":"javascript","preamble":"function DM_getPanelFile(i){ return '$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i];}\nfunction DM_getPanel(i) {\n  const fs = require('fs');\n  return JSON.parse(fs.readFileSync('$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[i]));\n}\nfunction DM_setPanel(v) {\n  const fs = require('fs');\n  const fd = fs.openSync('$$PANEL_RESULTS_FILE$$', 'w');\n  if (Array.isArray(v)) {\n    fs.writeSync(fd, '[');\n    for (let i = 0; i < v.length; i++) {\n      const row = v[i];\n      let rowJSON = JSON.stringify(row);\n      if (i < v.length - 1) {\n        rowJSON += ',';\n      }\n      fs.writeSync(fd, rowJSON);\n    }\n    fs.writeSync(fd, ']');\n  } else {\n    fs.writeSync(fd, JSON.stringify(v));\n  }\n}","defaultPath":"node","name":"JavaScript"}`,
+	"julia":      `{"id":"julia","name":"Julia","defaultPath":"julia","preamble":"\ntry\n    import JSON\ncatch e\n    import Pkg\n    Pkg.add(\"JSON\")\n    import JSON\nend\nfunction DM_getPanel(i)\n  panelId = JSON.parse(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[string(i)]\n  JSON.parsefile(string(\"$$RESULTS_FILE$$\", panelId))\nend\nfunction DM_setPanel(v)\n  open(\"$$PANEL_RESULTS_FILE$$\", \"w\") do f\n    JSON.print(f, v)\n  end\nend\nfunction DM_getPanelFile(i)\n  string(\"$$RESULTS_FILE$$\", JSON.parse(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[string(i)])\nend"}`,
+	"python":     `{"id":"python","name":"Python","defaultPath":"python3","preamble":"def DM_getPanelFile(i):\n  return r'$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[str(i)]\ndef DM_getPanel(i):\n  import json\n  with open(r'$$RESULTS_FILE$$'+$$JSON_ID_MAP$$[str(i)]) as f:\n    return json.load(f)\ndef DM_setPanel(v):\n  import json\n  with open(r'$$PANEL_RESULTS_FILE$$', 'w') as f:\n    json.dump(v, f)"}`,
+	"r":          `{"id":"r","name":"R","defaultPath":"Rscript","preamble":"\ntryCatch(library(\"rjson\"), error=function(cond) {install.packages(\"rjson\", repos=\"https://cloud.r-project.org\")}, finally=library(\"rjson\"))\nDM_getPanel <- function(i) {\n  panelId = fromJSON(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[[toString(i)]]\n  fromJSON(file=paste(\"$$RESULTS_FILE$$\", panelId, sep=\"\"))\n}\nDM_setPanel <- function(v) {\n  write(toJSON(v), \"$$PANEL_RESULTS_FILE$$\")\n}\nDM_getPanelFile <- function(i) {\n  paste(\"$$RESULTS_FILE$$\", fromJSON(\"$$JSON_ID_MAP_QUOTE_ESCAPED$$\")[[toString(i)]], sep=\"\")\n}\n"}`,
+	"ruby":       `{"id":"ruby","name":"Ruby","defaultPath":"ruby","preamble":"\ndef DM_getPanel(i)\n  require 'json'\n  JSON.parse(File.read('$$RESULTS_FILE$$' + JSON.parse('$$JSON_ID_MAP$$')[i.to_s]))\nend\ndef DM_setPanel(v)\n  require 'json'\n  File.write('$$PANEL_RESULTS_FILE$$', v.to_json)\nend\ndef DM_getPanelFile(i)\n  require 'json'\n  '$$RESULTS_FILE$$' + JSON.parse('$$JSON_ID_MAP$$')[i.to_s]\nend\n"}`,
+}

runner/scripts/generate_program_type_info.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "package runner
+
+// GENERATED BY ./runner/scripts/generate_program_type_info.sh. DO NOT MODIFY.
+
+var packedProgramTypeInfo = map[SupportedLanguages]string{" > runner/program_info.go
+for file in $(ls ./shared/languages/*.json); do
+    echo "$(cat $file | jq '.id'): \`$(cat $file | jq -c '.')\`," >> runner/program_info.go
+done
+
+echo "}" >> runner/program_info.go
+
+gofmt -w -s .