[Feat] Kepler-Jupyter 0.3.4 with kepler v3 (#2565)

.github/workflows/build-publish-pypi.yml

@@ -16,10 +16,10 @@ jobs:
           node-version: 18.x
           registry-url: https://registry.npmjs.org/
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Install dependencies
         run: |

.gitignore

@@ -18,6 +18,7 @@ package-lock.json
 .DS_Store
 .idea
 .vscode/
+.venv/
 
 /bundle*.js
 /favicon.png
@@ -33,4 +34,4 @@ npm-debug.log
 !.yarn/plugins
 !.yarn/releases
 !.yarn/sdks
-!.yarn/versions
+!.yarn/versions

bindings/kepler.gl-jupyter/README.md

@@ -47,7 +47,7 @@ jupyter nbextension enable --py --sys-prefix keplergl # can be skipped for noteb
 
 ### 2. For Google Colab:
 
-`keplergl` (>0.3.0) works with Google Colab. You can install it using pip. 
+`keplergl` (>0.3.0) works with Google Colab. You can install it using pip.
 
 ```python
 # Install keplergl (>0.3.0)
@@ -58,7 +58,7 @@ jupyter nbextension enable --py --sys-prefix keplergl # can be skipped for noteb
 
 #### JupyterLab 3
 
-NOTE: `keplergl` <=0.3.0 doesn't work with JupyterLab 3. You need to make sure the python package `keplergl` > 0.3.0 is installed. 
+NOTE: `keplergl` <=0.3.0 doesn't work with JupyterLab 3. You need to make sure the python package `keplergl` > 0.3.0 is installed.
 
 Installation using pip:
 ```shell
@@ -73,7 +73,7 @@ There is no need to use `jupyter labextension install` for `keplergl` > 0.3.0 wi
 
 #### JupyterLab 1
 
-For JupyterLab1, you need to install `keplergl-jupyter` labextension from NPM registery. There is no need to install `keplergl` python package. 
+For JupyterLab1, you need to install `keplergl-jupyter` labextension from NPM registery. There is no need to install `keplergl` python package.
 
 First, install `jupyterlab-manager` for JupyterLab1:
 ```shell
@@ -91,7 +91,7 @@ jupyter labextension install keplergl-jupyter
 
 #### JupyterLab 2
 
-For JupyterLab2, you need to install `keplergl-jupyter` labextension from NPM registery. There is no need to install `keplergl` python package. 
+For JupyterLab2, you need to install `keplergl-jupyter` labextension from NPM registery. There is no need to install `keplergl` python package.
 
 First, install `jupyterlab-manager` for JupyterLab2:
 ```shell
@@ -127,10 +127,12 @@ map_1 = KeplerGl(height=400)
 map_1
 
 # Load kepler.gl with map data and config
+# Since keplergl 0.3.4, you can pass `use_arrow=True` to load and render data faster using GeoArrow, e.g. `KeplerGl(data={'data_1': df}, config=config, use_arrow=True)`
 map_2 = KeplerGl(height=400, data={'data_1': df}, config=config)
 map_2
 
 # Add data to map
+# Since keplergl 0.3.4, you can pass `use_arrow=True` to load and render data faster using GeoArrow, e.g. `map_1.add_data(df, 'data_1', use_arrow=True)`
 map_1.add_data(df, 'data_1')
 
 # Apply config
@@ -209,17 +211,19 @@ You will need to install node, yarn and Jupyter Notebook.
 Install [node](https://nodejs.org/en/download/package-manager/#macos) `> 12`, and [yarn](https://yarnpkg.com/en/docs/install#mac-stable). Use [nvm](https://github.com/creationix/nvm) for better node version management e.g. `nvm install 12`.
 
 
-#### 2. Install Jupyter 
+#### 2. Install Jupyter
 
 - Using conda
 ```shell
 conda install jupyter
+conda install notebook 6.0.1
 ```
 
 - Using pip
 
 ```shell
 pip install jupyter
+pip install notebook==6.0.1
 ```
 
 #### 3. Install GeoPandas
@@ -281,7 +285,7 @@ jupyter nbextension install --py --symlink --sys-prefix keplergl
 jupyter nbextension enable --py --sys-prefix keplergl
 ```
 
-NOTE: The above command `jupyter nbextension install -py --symlink --sys-prefix keplergl` is trying to create a symoblic link of the folder `bindings/kepler.gl-jupyter/keplergl/static` under the jupyter's folder `nbextensions`. Please check if there is already a folder "nbextensions/kepler-jupyter" existed, and you might need to remove it first. 
+NOTE: The above command `jupyter nbextension install -py --symlink --sys-prefix keplergl` is trying to create a symoblic link of the folder `bindings/kepler.gl-jupyter/keplergl/static` under the jupyter's folder `nbextensions`. Please check if there is already a folder "nbextensions/kepler-jupyter" existed, and you might need to remove it first.
 
 To find the location of `nbextensions` folder, you can use the following command:
 ```shell

bindings/kepler.gl-jupyter/RELEASE.md

@@ -8,7 +8,7 @@ NOTE: __Version number of the js module **`kelergl-jupyter`** and the python mod
 
 ### Step1:
 
-Update `version_info` in keplergl/_version.py in bindings/kepler.gl-jupyter folder. 
+Update `version_info` in keplergl/_version.py in bindings/kepler.gl-jupyter folder.
 Update `"version": "0.x.x"` to match the version info in js/package.json in bindings/kepler.gl-jupyter folder.
 Update `EXTENSION_SPEC_VERSION` to match the js module version. Update `version` in js/package
 
@@ -21,7 +21,7 @@ git commit -am "keplergl==<version>"
 
 ### Step2:
 
-Create a tag: `<version>-jupyter` e.g. v0.3.2-jupyter
+Create a tag: `<version>-jupyter` e.g. v0.3.4-jupyter
 
 ```
 git tag -a <version>-jupyter -m "<version>-jupyter"
@@ -38,7 +38,7 @@ The new version should be automatically picked and built from PyPi by conda-forg
 
 Edit `meta.yaml` under directory `recipes/`:
 
-* Update the version number 
+* Update the version number
 
 ```python
 {% set version = "0.3.0" %}

bindings/kepler.gl-jupyter/js/babel.config.js

@@ -6,7 +6,7 @@ const KeplerPackage = require('./package');
 module.exports = function babel(api) {
   api.cache(true);
 
-  const presets = ['@babel/preset-env', '@babel/preset-react'];
+  const presets = ['@babel/preset-env', '@babel/preset-react', '@babel/preset-typescript'];
   const plugins = [
     '@babel/plugin-proposal-class-properties',
     'transform-inline-environment-variables',

bindings/kepler.gl-jupyter/js/lib/keplergl/components/app.js

@@ -12,7 +12,7 @@ import {
   PanelHeaderFactory,
   CustomPanelsFactory,
   injectComponents
-} from 'kepler.gl/components';
+} from '@kepler.gl/components';
 
 import CustomPanelHeaderFactory from './panel-header';
 import CustomSidebarFactory from './side-bar';

bindings/kepler.gl-jupyter/js/lib/keplergl/components/config-panel.js

@@ -3,9 +3,9 @@
 
 import React, {useState} from 'react';
 
-import {Button, Icons, TextArea, withState} from 'kepler.gl/components';
-import {KeplerGlSchema} from 'kepler.gl/schemas';
-import {visStateLens, mapStateLens, mapStyleLens} from 'kepler.gl/reducers';
+import {Button, Icons, TextArea, withState} from '@kepler.gl/components';
+import {KeplerGlSchema} from '@kepler.gl/schemas';
+import {visStateLens, mapStateLens, mapStyleLens} from '@kepler.gl/reducers';
 import {CopyToClipboard} from 'react-copy-to-clipboard';
 import styled from 'styled-components';
 

bindings/kepler.gl-jupyter/js/lib/keplergl/components/panel-header.js

@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
-import {PanelHeaderFactory, Icons, withState} from 'kepler.gl/components';
-import {toggleModal} from 'kepler.gl/actions';
+import {PanelHeaderFactory, Icons, withState} from '@kepler.gl/components';
+import {toggleModal} from '@kepler.gl/actions';
 import React from 'react';
 import {IntlProvider} from 'react-intl';
 

bindings/kepler.gl-jupyter/js/lib/keplergl/components/side-bar.js

@@ -2,7 +2,7 @@
 // Copyright contributors to the kepler.gl project
 
 import React from 'react';
-import {SidebarFactory, CollapseButtonFactory} from 'kepler.gl/components';
+import {SidebarFactory, CollapseButtonFactory} from '@kepler.gl/components';
 import styled from 'styled-components';
 
 const StyledSideBarContainer = styled.div`

bindings/kepler.gl-jupyter/js/lib/keplergl/kepler.gl.js

@@ -1,8 +1,8 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
-import {addDataToMap, ActionTypes} from 'kepler.gl/actions';
-import {KeplerGlSchema} from 'kepler.gl/schemas';
+import {addDataToMap, ActionTypes} from '@kepler.gl/actions';
+import {KeplerGlSchema} from '@kepler.gl/schemas';
 import document from 'global/document';
 
 import renderRoot from './components/root';
@@ -142,7 +142,8 @@ class KeplerGlJupyter {
           },
           data: {
             fields: d.fields,
-            rows: d.allData
+            // rows: d.allData
+            ...(d.dataContainer instanceof ArrowDataContainer ? {cols: d.dataContainer._cols} : {rows: d.allData})
           }
         })),
         config,

bindings/kepler.gl-jupyter/js/lib/keplergl/store.js

@@ -3,10 +3,7 @@
 
 import {applyMiddleware, compose, createStore} from 'redux';
 import {combineReducers} from 'redux';
-import {keplerGlReducer} from 'kepler.gl/reducers';
-
-// TODO: remove this after added middleware to files
-import {enhanceReduxMiddleware} from 'kepler.gl/middleware';
+import {keplerGlReducer, enhanceReduxMiddleware} from '@kepler.gl/reducers';
 
 const customizedKeplerGlReducer = keplerGlReducer.initialState({
   uiState: {

bindings/kepler.gl-jupyter/js/lib/keplergl/utils.js

@@ -1,7 +1,8 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
-import {processCsvData, processGeojson} from 'kepler.gl/processors';
+import {tableFromIPC} from 'apache-arrow';
+import {processCsvData, processGeojson, processArrowBatches} from '@kepler.gl/processors';
 import log from '../log';
 import console from 'global/console';
 
@@ -23,11 +24,22 @@ function handleJuptyerDataFormat(dataEntry) {
       type = 'json';
     }
   } else if (typeof data === 'string') {
+    // check if js string is json string
     try {
       parsed = JSON.parse(data);
       type = 'json';
     } catch (e) {
-      // assume it is csv
+      // assume it is base64 string represents arrow table
+      try {
+        console.log('parse base64string arrow tabl');
+        // convert arrowTable from base64 string to ArrayBuffer
+        const arrowTableBuffer = Buffer.from(data, 'base64').buffer;
+        // create arrow table from ArrayBuffer
+        parsed = tableFromIPC([new Uint8Array(arrowTableBuffer)]);
+        type = 'arrow';
+      } catch (e) {
+        // now we can assume it is csv
+      }
     }
   }
 
@@ -47,6 +59,8 @@ function processReceivedData({data, info}) {
         ? processGeojson(data)
         : info.queryType === 'df'
         ? processDataFrame(data)
+        : info.queryType === 'arrow'
+        ? processArrowBatches(data.batches)
         : null;
   } catch (e) {
     console.log(

bindings/kepler.gl-jupyter/js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "keplergl-jupyter",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "This is a simple jupyter widget for kepler.gl, an advanced geo-spatial visualization tool, to render large-scale interactive maps.",
   "author": "Shan He",
   "license": "MIT",
@@ -26,19 +26,21 @@
     "start": "NODE_ENV=development webpack --config ./webpack/dev.js --mode development --watch --progress",
     "clean": "rimraf dist/ && rimraf ../keplergl/static/",
     "cleanall": "npm run clean && rimraf node_modules/",
-    "prepublish": "yarn build && yarn build:lab",
-    "build": "npm run clean && npm run build:lab && webpack --config ./webpack/build.js && jupyter labextension build .",
-    "build:lab": "rimraf babel/ && mkdir babel && babel lib --out-dir babel",
+    "prepublish": "NODE_OPTIONS=--openssl-legacy-provider yarn build && yarn build:lab",
+    "build": "NODE_OPTIONS=--openssl-legacy-provider npm run clean && npm run build:lab && webpack --config ./webpack/build.js && jupyter labextension build .",
+    "build:lab": "NODE_OPTIONS=--openssl-legacy-provider rimraf babel/ && mkdir babel && babel lib --out-dir babel",
     "test": "echo \"Error: no test specified\" && exit 1",
     "lint": "eslint lib webpack --fix",
     "prettier": "prettier --config ./.prettierrc --print-width 80 --single-quote --write lib/**/*.js"
   },
   "devDependencies": {
+    "apache-arrow": "^13.0.0",
     "@babel/cli": "7.4.4",
     "@babel/core": "7.4.5",
     "@babel/plugin-proposal-class-properties": "^7.3.0",
     "@babel/preset-env": "^7.0.0",
     "@babel/preset-react": "^7.0.0",
+    "@babel/preset-typescript": "^7.16.7",
     "@jupyterlab/builder": "^4.0.0",
     "babel-eslint": "^9.0.0",
     "babel-loader": "^8.0.0",
@@ -55,7 +57,7 @@
     "react-dev-utils": "^10.2.1",
     "rimraf": "^2.6.1",
     "webpack": "^4.29.0",
-    "webpack-cli": "4.10.x",
+    "webpack-cli": "^3.2.1",
     "webpack-dev-middleware": "^3.5.1",
     "webpack-dev-server": "^3.1.14",
     "webpack-hot-middleware": "^2.24.3",
@@ -64,8 +66,17 @@
   "dependencies": {
     "@jupyter-widgets/base": "^2 || ^3 || ^4.0.0",
     "@jupyterlab/builder": "^4.0.0",
+    "@kepler.gl/actions": "^3.0.0",
+    "@kepler.gl/components": "^3.0.0",
+    "@kepler.gl/processors": "^3.0.0",
+    "@kepler.gl/reducers": "^3.0.0",
+    "@kepler.gl/styles": "^3.0.0",
+    "@kepler.gl/schemas": "^3.0.0",
+    "@loaders.gl/arrow": "^4.1.0",
+    "@loaders.gl/core": "^4.1.0",
+    "@loaders.gl/csv": "^4.1.0",
+    "@loaders.gl/json": "^4.1.0",
     "global": "^4.3.0",
-    "kepler.gl": "^2.5.5",
     "node-polyfill-webpack-plugin": "^1.1.2",
     "querystring": "0.2.1",
     "react": "^18.2.0",
@@ -79,8 +90,11 @@
     "styled-components": "^4.1.3"
   },
   "resolutions": {
-    "react-vis": "1.11.7",
-    "webpack-cli": "4.10.x"
+    "@deck.gl/core": "^8.9.27",
+    "@deck.gl/react": "^8.9.27",
+    "@deck.gl/aggregation-layers": "^8.9.27",
+    "@deck.gl/geo-layers": "^8.9.27",
+    "@deck.gl/layers": "^8.9.27"
   },
   "jupyterlab": {
     "extension": "babel/labplugin",
@@ -91,5 +105,10 @@
         "singleton": true
       }
     }
+  },
+  "packageManager": "yarn@1.22.17",
+  "volta": {
+    "node": "18.18.2",
+    "yarn": "1.22.17"
   }
 }

bindings/kepler.gl-jupyter/js/webpack/build-html.js

@@ -17,14 +17,15 @@ function clearCarats(version) {
 
 const VERSIONS = {
   react: clearCarats(dependencies.react),
-  reactDom:clearCarats(dependencies['react-dom']),
+  reactDom: clearCarats(dependencies['react-dom']),
   redux: clearCarats(dependencies.redux),
   reactRedux: clearCarats(dependencies['react-redux']),
-  reactIntl: clearCarats(dependencies['react-intl']),
+  // reactIntl UMD build is not available after 5.x
+  reactIntl: '4.7.6',
   reactCopyToClipboard: clearCarats(dependencies['react-copy-to-clipboard']),
   styledComponents: clearCarats(dependencies['styled-components']),
-  keplergl: clearCarats(dependencies['kepler.gl'])
-}
+  keplergl: clearCarats(dependencies['@kepler.gl/components'])
+};
 
 const externals = [
   {react: 'React'},