diff --git a/desktop/scripts/desktop.build b/desktop/scripts/desktop.build index 71b3014c7..b5416c0ad 100644 --- a/desktop/scripts/desktop.build +++ b/desktop/scripts/desktop.build @@ -6,8 +6,8 @@ setenv_default VERSION "development" cd runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w -X main.VERSION={VERSION}" -o ../build/go_desktop_runner{required_ext} cmd/main.go yarn esbuild desktop/preload.ts --external:electron --sourcemap --bundle --outfile=build/preload.js -yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop_runner.js -yarn esbuild desktop/app.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop.js +yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:asar --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop_runner.js +yarn esbuild desktop/app.ts --bundle --platform=node --sourcemap --external:better-sqlite3 --external:react-native-fs --external:react-native-fetch-blob "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:asar --external:ssh2-sftp-client --external:cpu-features --external:electron --target=node10.4 --outfile=build/desktop.js rm -rf build/migrations cp -r desktop/migrations build/migrations diff --git a/desktop/store.ts b/desktop/store.ts index 1f0dcc26e..790f9771d 100644 --- a/desktop/store.ts +++ b/desktop/store.ts @@ -1,3 +1,4 @@ +import asar from 'asar'; import * as sqlite3 from 'better-sqlite3'; import { Buffer } from 'buffer'; import fs from 'fs'; @@ -7,6 +8,7 @@ import { getPath } from '../shared/object'; import { GetProjectRequest, MakeProjectRequest } from '../shared/rpc'; import { ConnectorInfo, + DatabaseConnectorInfo, DatabasePanelInfo, doOnEncryptFields, Encrypt, @@ -185,7 +187,11 @@ export class Store { const name = f.slice(0, f.length - ('.' + PROJECT_EXTENSION).length); return { createdAt, name }; }); - files.sort(); + // Sort timestamp DESC + files.sort( + (a, b) => + new Date(b.createdAt).valueOf() - new Date(a.createdAt).valueOf() + ); return files; }, }; @@ -378,17 +384,56 @@ GROUP BY panel_id }, }; + // Example: /private/var/folders/l0/51ds3d1d2214wb1y0vbtl5qr0000gn/T/AppTranslocation/6AC58880-BE22-4AB7-8006-47D9764BC590/d/DataStation Desktop CE.app/Contents/Resources/app.asar/sampledata/nginx_logs.jsonl + unmangleAsar(file: string): string { + const asarName = 'app.asar'; + if (!file.includes(asarName)) { + return file; + } + + // Since Go reads from the filesystem it doesn't look into the asar that is used in release builds. So we need to extract it if it doesn't exist. + const [asarParent, fileName] = file.split(asarName); + const asarFile = asarParent + asarName; + const newFile = path.join(asarParent, fileName); + // TODO: if these files change then checksum will be needed + if (!fs.existsSync(newFile)) { + fs.mkdirSync(path.dirname(newFile), { recursive: true }); + fs.writeFileSync( + newFile, + asar.extractFile(asarFile, fileName.slice(1) /* drop leading / */) + ); + } + + return newFile; + } + cleanupSampleProject(sampleProject: ProjectState) { for (const page of sampleProject.pages || []) { for (const panel of page.panels || []) { if (panel.type === 'file') { const fp = panel as FilePanelInfo; if (fp.file.name.startsWith('sampledata')) { - fp.file.name = path.join(CODE_ROOT, fp.file.name); + fp.file.name = this.unmangleAsar( + path.join(CODE_ROOT, fp.file.name) + ); } } } } + + for (const con of sampleProject.connectors || []) { + if (con.type === 'database') { + const dc = con as DatabaseConnectorInfo; + if ( + dc.database.type === 'sqlite' && + dc.database.database.startsWith('sampledata') + ) { + dc.database.database = this.unmangleAsar( + path.join(CODE_ROOT, dc.database.database) + ); + } + } + } } makeProjectHandler: MakeProjectHandler = { diff --git a/package.json b/package.json index f73d174c0..fb6cd85fd 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "dependencies": { "@tabler/icons": "^1.68.0", "ace-builds": "^1.5.0", + "asar": "^3.1.0", "better-sqlite3": "^7.5.3", "chart.js": "^3.7.1", "cookie-parser": "^1.4.6", diff --git a/runner/scripts/runner_test.build b/runner/scripts/runner_test.build index 8de9cd2b0..13805093a 100644 --- a/runner/scripts/runner_test.build +++ b/runner/scripts/runner_test.build @@ -1,3 +1,3 @@ -yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:oracledb "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --external:pg-native --target=node10.4 --outfile=build/desktop_runner.js +yarn esbuild desktop/runner.ts --bundle --platform=node --sourcemap --external:sqlite3 --external:asar --external:react-native-fs --external:react-native-fetch-blob --external:oracledb "--external:@elastic/elasticsearch" "--external:wasm-brotli" --external:prometheus-query --external:snowflake-sdk --external:ssh2 --external:ssh2-promise --external:ssh2-sftp-client --external:cpu-features --external:electron --external:pg-native --target=node10.4 --outfile=build/desktop_runner.js cd runner/cmd && go test -trimpath -mod=readonly -modcacherw -ldflags="-s -w" -race -o ../../build/go_desktop_runner_test{required_ext} -coverpkg="." -c -tags testrunmain \ No newline at end of file diff --git a/scripts/build.py b/scripts/build.py index 7391aa97b..39a4f8180 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -168,11 +168,15 @@ def eval_line(line): quote_line(line, show=True) if not what: return - with open(to, 'rb+') as to: - current = to.read() - to.seek(0) - to.write(what.encode()+b'\n'+current) - to.truncate() + try: + with open(to, 'rb+') as to: + current = to.read() + to.seek(0) + to.write(what.encode()+b'\n'+current) + to.truncate() + except FileNotFoundError: + with open(to, 'w') as to: + to.write(what) return quote_line(line, show=True) diff --git a/server/scripts/release.build b/server/scripts/release.build index 292f06f0b..75cb15567 100644 --- a/server/scripts/release.build +++ b/server/scripts/release.build @@ -13,9 +13,14 @@ mkdir datastation mkdir datastation/node_modules mv build datastation/ +# Bring in sampledata +cp -r sampledata datastation/sampledata + # Bring in node_modules -cp -r node_modules/electron datastation/node_modules/electron -cp -r node_modules/better-sqlite3 datastation/node_modules/better-sqlite3 +prepend "{{}}" datastation/package.json +cd datastation && yarn add asar electron better-sqlite3 +rm datastation/yarn.lock +rm datastation/package.json # Only need stubs not full chrome install rm -rf datastation/node_modules/electron/dist diff --git a/server/scripts/server.build b/server/scripts/server.build index ebb16020b..feaeab9df 100644 --- a/server/scripts/server.build +++ b/server/scripts/server.build @@ -6,8 +6,8 @@ yarn build-ui cd runner && go build -trimpath -buildmode=pie -mod=readonly -modcacherw -ldflags="-s -w" -o ../build/go_server_runner{required_ext} ./cmd/main.go -yarn esbuild server/runner.ts --sourcemap --platform=node --bundle --target=node10.4 --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server_runner.js -yarn esbuild server/index.ts --sourcemap --platform=node --bundle --external:sqlite3 --external:react-native-fs --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server.js +yarn esbuild server/runner.ts --sourcemap --platform=node --bundle --target=node10.4 --external:sqlite3 --external:asar --external:react-native-fs --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server_runner.js +yarn esbuild server/index.ts --sourcemap --platform=node --bundle --external:sqlite3 --external:react-native-fs --external:asar --external:react-native-fetch-blob --external:cpu-features --external:electron --outfile=build/server.js prepend "global.DS_UI_TITLE='DataStation Server CE';" build/server.js diff --git a/ui/style.css b/ui/style.css index 565b561ab..304a8eeb1 100644 --- a/ui/style.css +++ b/ui/style.css @@ -17,6 +17,7 @@ /* Generic settings */ --sans-serif: 'San Francisco', 'Gadugi', sans-serif; --margin-base: 8px; + --label-width: 100px; } * { @@ -201,7 +202,7 @@ label, .select-label { display: inline-block; text-align: right; - width: 100px; + width: var(--label-width); padding-right: calc(2 * var(--margin-base)); overflow-wrap: break-word; } @@ -599,6 +600,11 @@ input + input[type='file'] { .connector-body, .server-body { overflow-x: auto; + + input, + select { + max-width: 250px; + } } .radio-label, @@ -1121,6 +1127,10 @@ canvas { width: 500px; margin: 0 auto; margin-top: calc(2 * var(--margin-base)); + + .select select { + max-width: 300px !important; + } } .card--full {