Fix create page creation order, fix server zip creation, add basic te…

…sts for server zip (#240) * Drop console logs, fix page add * More build script cleanup * More renaming * More fixes to server zip * Add test script for release zip * Closer on release test script * Add systemd testing dockerfile * Fix for server release zip and systemd * Add server test to CI * Fix server name * More cleanup * Working diff test * Update gitignore * Debugging * Rename e2e files * Move server tests into separate step * Fix for fmt * Debug one job * Debug * More debug * Trap and debug * Fix bash synax * Move to tar.gz * And remove on completion * Bring .tar.gz size all the way down * Debug docker version * Try installing podman manually * sudo * No systemd test for now
multiprocessio · May 8, 2022 · 6780870 · 6780870
1 parent 9d05575
commit 6780870
Show file tree

Hide file tree

Showing 17 changed files with 261 additions and 147 deletions.
diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml
@@ -5,52 +5,137 @@ on:
     branches: [main]
 
 jobs:
-  test:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@master
-      with:
-        ref: ${{ github.ref }}
-
-    # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn
-    - name: Get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-
-    - uses: actions/cache@v2
-      id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-
-    - run: ./scripts/ci/prepare_linux.sh --integration-tests
-    - run: yarn format
-    - run: yarn eslint ui
-    - run: yarn tsc
-    - run: cd runner && gofmt -w .
-    - run: yarn build-language-definitions
-    - run: ./scripts/fail_on_diff.sh
-    - run: yarn test-licenses
-    # Needed so we can have ./build/desktop_runner.js and ./build/go_desktop_runner ready for tests
-    - run: yarn build-test-runner
-    - run: yarn test --detectOpenHandles --forceExit --verbose
-      env:
-        AIRTABLE_TOKEN: ${{ secrets.AIRTABLE_TOKEN }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        BIGQUERY_TOKEN: ${{ secrets.BIGQUERY_TOKEN }}
-        SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
-        SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
-        SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
-    - run: cd runner && go test -race -coverprofile ../coverage/gounit.cov
-    - run: ./runner/scripts/test_coverage.sh
-    # Test server builds
-    - run: yarn build-server
-
-  e2e-linux:
+  # test:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@master
+  #     with:
+  #       ref: ${{ github.ref }}
+
+  #   # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn
+  #   - name: Get yarn cache directory path
+  #     id: yarn-cache-dir-path
+  #     run: echo "::set-output name=dir::$(yarn cache dir)"
+
+  #   - uses: actions/cache@v2
+  #     id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+  #     with:
+  #       path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+  #       key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+  #       restore-keys: |
+  #         ${{ runner.os }}-yarn-
+  #   - run: ./scripts/ci/prepare_linux.sh --integration-tests
+  #   - run: yarn format
+  #   - run: yarn eslint ui
+  #   - run: yarn tsc
+  #   - run: cd runner && gofmt -w .
+  #   - run: yarn build-language-definitions
+  #   - run: ./scripts/fail_on_diff.sh
+  #   - run: yarn test-licenses
+  #   # Needed so we can have ./build/desktop_runner.js and ./build/go_desktop_runner ready for tests
+  #   - run: yarn build-test-runner
+  #   - run: yarn test --detectOpenHandles --forceExit --verbose
+  #     env:
+  #       AIRTABLE_TOKEN: ${{ secrets.AIRTABLE_TOKEN }}
+  #       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  #       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  #       BIGQUERY_TOKEN: ${{ secrets.BIGQUERY_TOKEN }}
+  #       SNOWFLAKE_USER: ${{ secrets.SNOWFLAKE_USER }}
+  #       SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+  #       SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }}
+  #   - run: cd runner && go test -race -coverprofile ../coverage/gounit.cov
+  #   - run: ./runner/scripts/test_coverage.sh
+
+  # e2e-linux:
+  #   runs-on: ubuntu-latest
+
+  #   steps:
+  #   - uses: actions/checkout@master
+  #     with:
+  #       ref: ${{ github.ref }}
+
+  #   # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn
+  #   - name: Get yarn cache directory path
+  #     id: yarn-cache-dir-path
+  #     run: echo "::set-output name=dir::$(yarn cache dir)"
+
+  #   - uses: actions/cache@v2
+  #     id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+  #     with:
+  #       path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+  #       key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+  #       restore-keys: |
+  #         ${{ runner.os }}-yarn-
+  #   - run: ./scripts/ci/prepare_linux.sh
+  #   - run: yarn release-desktop 0.0.0-e2etest
+  #   - run: git reset --hard # release blows everything up
+  #     # Set up a virtual framebuffer so Chrome will start
+  #     # https://www.electronjs.org/docs/tutorial/testing-on-headless-ci
+  #     # https://github.com/juliangruber/browser-run/issues/147
+  #   - run: xvfb-run --auto-servernum yarn e2e-test
+
+  # e2e-macos:
+  #   runs-on: macos-latest
+
+  #   steps:
+  #   - uses: actions/checkout@master
+  #     with:
+  #       ref: ${{ github.ref }}
+
+  #   # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn
+  #   - name: Get yarn cache directory path
+  #     id: yarn-cache-dir-path
+  #     run: echo "::set-output name=dir::$(yarn cache dir)"
+
+  #   - uses: actions/cache@v2
+  #     id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+  #     with:
+  #       path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+  #       key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+  #       restore-keys: |
+  #         ${{ runner.os }}-yarn-
+  #   - run: ./scripts/ci/prepare_macos.sh
+  #   # Needed so we can have ./build/desktop_runner.js ready for tests
+  #   - run: yarn build-test-runner
+  #   # https://github.com/golang/go/issues/49138 bug in macos -race detector
+  #   - run: MallocNanoZone=0 yarn test --detectOpenHandles --forceExit --verbose shared ui desktop
+  #   - run: cd runner && MallocNanoZone=0 go test -race -cover
+  #   - run: yarn release-desktop 0.0.0-e2etest
+  #   - run: git reset --hard # release blows everything up
+  #   - run: yarn e2e-test
+
+  # e2e-windows:
+  #   runs-on: windows-latest
+
+  #   steps:
+  #   - uses: actions/checkout@master
+  #     with:
+  #       ref: ${{ github.ref }}
+
+  #   # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn
+  #   - name: Get yarn cache directory path
+  #     id: yarn-cache-dir-path
+  #     run: echo "::set-output name=dir::$(yarn cache dir)"
+
+  #   - uses: actions/cache@v2
+  #     id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+  #     with:
+  #       path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+  #       key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+  #       restore-keys: |
+  #         ${{ runner.os }}-yarn-
+  #   - run: ./scripts/ci/prepare_windows.ps1
+  #     shell: pwsh
+  #   # Needed so we can have ./build/desktop_runner.js ready for tests
+  #   - run: yarn build-test-runner
+  #   - run: yarn test --detectOpenHandles --forceExit --verbose shared ui desktop
+  #   - run: cd runner && go test -race -cover
+  #   - run: yarn release-desktop 0.0.0-e2etest
+  #   - run: git reset --hard # release blows everything up
+  #   - run: yarn e2e-test
+
+  test-server:
     runs-on: ubuntu-latest
 
     steps:
@@ -72,71 +157,6 @@ jobs:
           ${{ runner.os }}-yarn-
 
     - run: ./scripts/ci/prepare_linux.sh
-    - run: yarn release-desktop 0.0.0-e2etest
-    - run: git reset --hard # release blows everything up
-      # Set up a virtual framebuffer so Chrome will start
-      # https://www.electronjs.org/docs/tutorial/testing-on-headless-ci
-      # https://github.com/juliangruber/browser-run/issues/147
-    - run: xvfb-run --auto-servernum yarn e2e-test
-
-  e2e-macos:
-    runs-on: macos-latest
-
-    steps:
-    - uses: actions/checkout@master
-      with:
-        ref: ${{ github.ref }}
-
-    # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn
-    - name: Get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-
-    - uses: actions/cache@v2
-      id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-
-    - run: ./scripts/ci/prepare_macos.sh
-    # Needed so we can have ./build/desktop_runner.js ready for tests
-    - run: yarn build-test-runner
-    # https://github.com/golang/go/issues/49138 bug in macos -race detector
-    - run: MallocNanoZone=0 yarn test --detectOpenHandles --forceExit --verbose shared ui desktop
-    - run: cd runner && MallocNanoZone=0 go test -race -cover
-    - run: yarn release-desktop 0.0.0-e2etest
-    - run: git reset --hard # release blows everything up
-    - run: yarn e2e-test
-
-  e2e-windows:
-    runs-on: windows-latest
-
-    steps:
-    - uses: actions/checkout@master
-      with:
-        ref: ${{ github.ref }}
-
-    # FROM: https://github.com/actions/cache/blob/main/examples.md#node---yarn
-    - name: Get yarn cache directory path
-      id: yarn-cache-dir-path
-      run: echo "::set-output name=dir::$(yarn cache dir)"
-
-    - uses: actions/cache@v2
-      id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
-      with:
-        path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-        key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-yarn-
-
-    - run: ./scripts/ci/prepare_windows.ps1
-      shell: pwsh
-    # Needed so we can have ./build/desktop_runner.js ready for tests
-    - run: yarn build-test-runner
-    - run: yarn test --detectOpenHandles --forceExit --verbose shared ui desktop
-    - run: cd runner && go test -race -cover
-    - run: yarn release-desktop 0.0.0-e2etest
-    - run: git reset --hard # release blows everything up
-    - run: yarn e2e-test
+    - run: yarn release-server 0.0.0-test
+    - run: sudo apt-get install -y podman
+    - run: ./server/scripts/test_release.sh datastation-server-x64-0.0.0-test.tar.gz
diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml
@@ -22,9 +22,9 @@ jobs:
       run: |
         curl --fail \
           -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-          -H "Content-Type: application/zip" \
-          --data-binary @./releases/datastation-server-x64-$GIT_TAG.zip \
-          "https://uploads.github.com/repos/multiprocessio/datastation/releases/$RELEASE_ID/assets?name=datastation-server-x64-$GIT_TAG.zip"
+          -H "Content-Type: application/tar+gzip" \
+          --data-binary @./releases/datastation-server-x64-$GIT_TAG.tar.gz \
+          "https://uploads.github.com/repos/multiprocessio/datastation/releases/$RELEASE_ID/assets?name=datastation-server-x64-$GIT_TAG.tar.gz"
 
   build-linux-desktop:
     runs-on: ubuntu-latest

diff --git a/.gitignore b/.gitignore
@@ -2,6 +2,7 @@
 node_modules
 ui/build
 build
+datastation
 yarn-error.log
 releases
 coverage

diff --git a/desktop/scripts/release.build b/desktop/scripts/release.build
@@ -40,5 +40,5 @@ rm -rf releases
 yarn electron-rebuild
 
 # Build and package
-yarn electron-packager --asar --overwrite --icon=build/icon.png --out=releases --build-version={arg0} --app-version={arg0} . "DataStation Community Edition"
-zip -9 -r releases/datastation-{os}-{arch}-{arg0}.zip "releases/DataStation Community Edition-{os}-{arch}"
+yarn electron-packager --asar --overwrite --icon=build/icon.png --out=releases --build-version={arg0} --app-version={arg0} . "DataStation Desktop CE"
+zip -9 -r releases/datastation-{os}-{arch}-{arg0}.zip "releases/DataStation Desktop CE-{os}-{arch}"
diff --git a/e2e/index.js b/e2e/index.js
@@ -29,15 +29,14 @@ async function run() {
   });
 
   const applicationPath = {
-    darwin:
-      'DataStation Community Edition.app/Contents/MacOS/DataStation Community Edition',
-    win32: 'DataStation Community Edition.exe',
-    linux: 'DataStation Community Edition',
+    darwin: 'DataStation Desktop CE.app/Contents/MacOS/DataStation Desktop CE',
+    win32: 'DataStation Desktop CE.exe',
+    linux: 'DataStation Desktop CE',
   }[process.platform];
 
   const directory = path.join(
     'releases',
-    `DataStation Community Edition-${process.platform}-${process.arch}`
+    `DataStation Desktop CE-${process.platform}-${process.arch}`
   );
 
   const driver = new webdriver.Builder()
@@ -55,7 +54,7 @@ async function run() {
   try {
     await driver.wait(async () => {
       const title = await driver.getTitle();
-      assert.equal(title, 'DataStation Community Edition');
+      assert.equal(title, 'DataStation Desktop CE');
       reached = true;
       return true;
     }, 10_000);

diff --git a/package.json b/package.json
@@ -13,7 +13,7 @@
     "build-server": "python3 ./scripts/build.py ./server/scripts/server.build",
     "docker-build-server": "bash ./server/scripts/build_image.sh",
     "build-language-definitions": "python3 ./scripts/build.py ./ui/scripts/languages.build && ./runner/scripts/generate_program_type_info.sh && yarn format",
-    "start-server": "yarn build-server && node ./build/server.js",
+    "start-server": "yarn build-server && cp node_modules/better-sqlite3/build/Release/better_sqlite3.node build && node ./build/server.js",
     "start-desktop": "yarn build-desktop && yarn electron --trace-warning --unhandled-rejection=warn build/desktop.js",
     "start-ui": "./ui/scripts/watch_and_serve.sh",
     "release-desktop": "python3 ./scripts/build.py ./desktop/scripts/release.build",

diff --git a/server/release/datastation.service b/server/release/datastation.service
@@ -9,7 +9,8 @@ Restart=always
 RestartSec=1
 User=datastation
 ExecStartPre=
-ExecStart=node /usr/share/datastation/server.js
+WorkingDirectory=/usr/share/datastation
+ExecStart=node /usr/share/datastation/build/server.js
 ExecStartPost=
 ExecStop=
 ExecReload=

diff --git a/server/release/install.sh b/server/release/install.sh
@@ -2,15 +2,15 @@
 
 set -ex
 
-ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/.."
+ROOT_DIR="$(readlink -f $(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd)/..)"
 
-sudo mkdir -p /etc/datastation /usr/share/datastation/
-sudo cp -r $ROOT_DIR/* /usr/share/datastation/
+sudo mkdir -p /etc/datastation /usr/share /etc/systemd/system /home/datastation
 sudo cp $ROOT_DIR/release/config.yaml /etc/datastation/
 sudo cp $ROOT_DIR/release/datastation.service /etc/systemd/system/
-sudo cp $ROOT_DIR/release/datastation-exporter.timer /etc/systemd/system/
+# sudo cp $ROOT_DIR/release/datastation-exporter.timer /etc/systemd/system/
+sudo mv $ROOT_DIR /usr/share/datastation
 
 sudo id -u datastation >/dev/null 2>&1 || sudo useradd -r -s /bin/false datastation
-sudo chown -R datastation:datastation /etc/datastation /usr/share/datastation
+sudo chown -R datastation:datastation /etc/datastation /usr/share/datastation /home/datastation
 
 sudo systemctl enable datastation
diff --git a/server/scripts/release.build b/server/scripts/release.build
@@ -7,10 +7,23 @@ prepend "window.DS_CONFIG_VERSION='{arg0}';" build/ui.js
 prepend "window.DS_CONFIG_VERSION='{arg0}';" build/server_runner.js
 prepend "global.DS_CONFIG_VERSION='{arg0}';" build/server.js
 
-mv node_modules build
+# Need to have a directory with a build directory and a node_modules directory
+rm -rf datastation
+mkdir datastation
+mkdir datastation/node_modules
+mv build datastation/
 
+# Bring in node_modules
+cp -r node_modules/electron datastation/node_modules/electron
+cp -r node_modules/better-sqlite3 datastation/node_modules/better-sqlite3
+# Only need stubs not full chrome install
+rm -rf datastation/node_modules/electron/dist
+
+cp node_modules/better-sqlite3/build/Release/better_sqlite3.node datastation/build/
 # Copy in install script and default configs
-cp -r server/release build/release
+cp -r server/release  datastation/release
+
+mkdir releases
+tar -zcvf releases/datastation-server-{arch}-{arg0}.tar.gz datastation
 
-mkdir -p releases
-zip -9 -r releases/datastation-server-{arch}-{arg0}.zip build
+rm -rf datastation