02-Major-TOM-Quick-Viewer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![image/png](https://cdn-uploads.huggingface.co/production/uploads/6304c06eeb6d777a838eab63/BJKsLwX0GG4W3-gdf40TJ.png)\r",
    "# 🗺️ Major-TOM Filtering\n",
    "r[![HF](https://img.shields.io/badge/%F0%9F%A4%97-Datasets-yellow)](https://www.huggingface.co/Major-TOM) [![paper](https://img.shields.io/badge/arXiv-2402.12095-D12424)](https://www.arxiv.org/abs/2402.12095) [![GitHub stars](https://img.shields.io/github/stars/ESA-PhiLab/Major-TOM?style=social&label=Star&maxAge=2592000)](https://github.com/ESA-PhiLab/Major-TOM/)\n",
    "\r\n",
    "This app provides a way of exploring samples present in the MajorTOM-Core dataset.\r\n",
    "It contains nearly every piece of Earth capture by ESA Sentinel-2 satellis>view."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "7hGBvLSp6Z4k"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import leafmap\n",
    "from src.grid import *\n",
    "\n",
    "if os.path.isfile('metadata.parquet'):\n",
    "    meta_path = 'metadata.parquet'\n",
    "else:\n",
    "    DATASET_NAME = 'Major-TOM/Core-S2L2A'\n",
    "    meta_path = 'https://huggingface.co/datasets/{}/resolve/main/metadata.parquet'.format(DATASET_NAME)\n",
    "\n",
    "grid = Grid(10, latitude_range=(-90,90), longitude_range=(-180,180))\n",
    "df = pd.read_parquet(meta_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "pONJLdYC7FVk",
    "outputId": "2cbcada3-6486-4cd6-933f-0c72387c387b"
   },
   "outputs": [],
   "source": [
    "from fsspec.parquet import open_parquet_file\n",
    "import fsspec\n",
    "import pyarrow.parquet as pq\n",
    "from io import BytesIO\n",
    "from PIL import Image\n",
    "\n",
    "def gridcell2ints(grid_string):\n",
    "    up = int(grid_string.split('_')[0][:-1]) * (2*int(grid_string.split('_')[0][-1]=='U') - 1) # +ve if up\n",
    "    right = int(grid_string.split('_')[1][:-1]) * (2*int(grid_string.split('_')[1][-1]=='R') - 1) # +ve if R\n",
    "\n",
    "    return up, right\n",
    "\n",
    "def row2image(parquet_url, parquet_row, fullrow_read=True):\n",
    "\n",
    "    if fullrow_read:\n",
    "        # option 1\n",
    "        f=fsspec.open(parquet_url)\n",
    "        temp_path = f.open()\n",
    "    else:\n",
    "        # option 2\n",
    "        temp_path = open_parquet_file(parquet_url,columns = [\"thumbnail\"])\n",
    "    \n",
    "    with pq.ParquetFile(temp_path) as pf:\n",
    "        first_row_group = pf.read_row_group(parquet_row, columns=['thumbnail'])\n",
    "\n",
    "    stream = BytesIO(first_row_group['thumbnail'][0].as_py())\n",
    "    return Image.open(stream)\n",
    "\n",
    "def row2s2(parquet_url, parquet_row, s2_bands = [\"B04\", \"B03\", \"B02\"]):\n",
    "    with open_parquet_file(parquet_url,columns = s2_bands) as f:\n",
    "        with pq.ParquetFile(f) as pf:\n",
    "            first_row_group = pf.read_row_group(parquet_row, columns=s2_bands)\n",
    "\n",
    "    return first_row_group\n",
    "\n",
    "def cell2row(grid_string, meta_df, return_row = False):\n",
    "    row_U, col_R = gridcell2ints(grid_string)\n",
    "    R = meta_df.query('grid_row_u == {} & grid_col_r == {}'.format(row_U, col_R))\n",
    "\n",
    "    if not R.empty:\n",
    "        if return_row:\n",
    "            return R.parquet_url.item(), R.parquet_row.item(), R\n",
    "        else:\n",
    "            return R.parquet_url.item(), R.parquet_row.item()\n",
    "    else:\n",
    "        return None\n",
    "\n",
    "def map_to_image(map):\n",
    "\n",
    "    # 1. get bounds\n",
    "    bbox = map.get_bbox()\n",
    "    center = [(bbox[3]+bbox[1])/2, (bbox[2]+bbox[0])/2]\n",
    "\n",
    "    # 2. translate coordinate to major-tom tile\n",
    "    rows, cols = grid.latlon2rowcol([center[0]], [center[1]])\n",
    "\n",
    "    # 3. translate major-tom cell to row in parquet\n",
    "    row = cell2row(\"{}_{}\".format(rows[0],cols[0]), df, return_row = True)\n",
    "\n",
    "    if row is not None:\n",
    "        parquet_url, parquet_row, meta_row = row\n",
    "        img = row2image(parquet_url, parquet_row)\n",
    "        # 4. acquire image # X. update map\n",
    "        lat, lon = meta_row.centre_lat.item(), meta_row.centre_lon.item()\n",
    "        \n",
    "        map.set_center(lon, lat, zoom=12)\n",
    "\n",
    "        return img\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Instructions\n",
    "To find a sample, navigate on the map to a place of interest. Click `Find Sample` to find a dataset sample that contains the central pixel of your current view."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 580,
     "referenced_widgets": [
      "327d9ea3c89948139f2c23b9c48e7ab9",
      "2552ff251677429b807bcb36e976a6c6",
      "76d8ce7251044a958002f66b7f29931d",
      "3e9db067244248959ceded14d6371ab1",
      "06fb09f144dd426ba9c195532c616631",
      "d66a36952d474a0882d8127c166cc771",
      "df2570005aae406aa0c0fe9eca0e1c05",
      "d157f2c06b7f4d089452873d11d92099",
      "e6e656d1ceb849bb89dea12c8000cc14",
      "5887a75218d34134a50fe09a7893fba9"
     ]
    },
    "id": "W8TO8Ae17I1K",
    "outputId": "a9cec03e-80c3-4734-b8fb-43a049c5db00"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec9b70dded514a0191d9a31cef45d413",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(VBox(children=(Map(center=[20, 0], controls=(AttributionControl(options=['position', 'prefix'],…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display\n",
    "from ipywidgets import Button, HBox, VBox, Layout, Output, Dropdown\n",
    "\n",
    "m = leafmap.Map(\n",
    "    height=559,\n",
    "    width=559,\n",
    "    zoom=4,\n",
    "    draw_control=False,\n",
    "    measure_control=False,\n",
    "    fullscreen_control=False,\n",
    "    toolbar_control=False,\n",
    "    attribution_control=True,\n",
    ")\n",
    "\n",
    "button = Button(description=\"Find Sample\", style={\"font_size\" : \"30px\"}, layout=Layout(width='50%', height='80px'))\n",
    "output = Output()\n",
    "\n",
    "left_box = VBox([m, button], layout=Layout(width='50%',display='flex',flex_flow='column',align_items='center'))\n",
    "right_box = VBox([output], layout=Layout(width='50%'))\n",
    "full_page = HBox([left_box, right_box])\n",
    "\n",
    "display(full_page)\n",
    "\n",
    "def on_button_clicked(b):\n",
    "    output = Output()\n",
    "    with output:\n",
    "        display(map_to_image(m))\n",
    "    right_box.children = [output]\n",
    "\n",
    "button.on_click(on_button_clicked)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<details><summary><strong>🏝 Couldn't find a sample? See this figure of global coverage:</strong></summary>\n",
    "    <img src='https://cdn-uploads.huggingface.co/production/uploads/6304c06eeb6d777a838eab63/2KTarfsM0a1dNYEbXriUH.png' />\n",
    "</details>"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.17"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "06fb09f144dd426ba9c195532c616631": {
     "model_module": "jupyter-leaflet",
     "model_module_version": "^0.18",
     "model_name": "LeafletScaleControlModel",
     "state": {
      "_model_module": "jupyter-leaflet",
      "_model_module_version": "^0.18",
      "_model_name": "LeafletScaleControlModel",
      "_view_count": null,
      "_view_module": "jupyter-leaflet",
      "_view_module_version": "^0.18",
      "_view_name": "LeafletScaleControlView",
      "imperial": true,
      "max_width": 100,
      "metric": true,
      "options": [
       "imperial",
       "max_width",
       "metric",
       "position",
       "update_when_idle"
      ],
      "position": "bottomleft",
      "update_when_idle": false
     }
    },
    "2552ff251677429b807bcb36e976a6c6": {
     "model_module": "jupyter-leaflet",
     "model_module_version": "^0.18",
     "model_name": "LeafletAttributionControlModel",
     "state": {
      "_model_module": "jupyter-leaflet",
      "_model_module_version": "^0.18",
      "_model_name": "LeafletAttributionControlModel",
      "_view_count": null,
      "_view_module": "jupyter-leaflet",
      "_view_module_version": "^0.18",
      "_view_name": "LeafletAttributionControlView",
      "options": [
       "position",
       "prefix"
      ],
      "position": "bottomright",
      "prefix": "ipyleaflet"
     }
    },
    "327d9ea3c89948139f2c23b9c48e7ab9": {
     "model_module": "jupyter-leaflet",
     "model_module_version": "^0.18",
     "model_name": "LeafletMapModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "jupyter-leaflet",
      "_model_module_version": "^0.18",
      "_model_name": "LeafletMapModel",
      "_view_count": null,
      "_view_module": "jupyter-leaflet",
      "_view_module_version": "^0.18",
      "_view_name": "LeafletMapView",
      "bottom": 458129,
      "bounce_at_zoom_limits": true,
      "box_zoom": true,
      "center": [
       22.230062043067587,
       -78.62499002169999
      ],
      "close_popup_on_click": true,
      "controls": [
       "IPY_MODEL_2552ff251677429b807bcb36e976a6c6",
       "IPY_MODEL_76d8ce7251044a958002f66b7f29931d",
       "IPY_MODEL_3e9db067244248959ceded14d6371ab1",
       "IPY_MODEL_06fb09f144dd426ba9c195532c616631"
      ],
      "crs": {
       "custom": false,
       "name": "EPSG3857"
      },
      "default_style": "IPY_MODEL_d66a36952d474a0882d8127c166cc771",
      "double_click_zoom": true,
      "dragging": true,
      "dragging_style": "IPY_MODEL_df2570005aae406aa0c0fe9eca0e1c05",
      "east": -78.52890014648438,
      "fullscreen": false,
      "inertia": true,
      "inertia_deceleration": 3000,
      "inertia_max_speed": 1500,
      "interpolation": "bilinear",
      "keyboard": true,
      "keyboard_pan_offset": 80,
      "keyboard_zoom_offset": 1,
      "layers": [
       "IPY_MODEL_d157f2c06b7f4d089452873d11d92099"
      ],
      "layout": "IPY_MODEL_e6e656d1ceb849bb89dea12c8000cc14",
      "left": 294997,
      "max_zoom": 24,
      "min_zoom": null,
      "modisdate": "2024-02-28",
      "north": 22.318954239411404,
      "options": [
       "bounce_at_zoom_limits",
       "box_zoom",
       "center",
       "close_popup_on_click",
       "double_click_zoom",
       "dragging",
       "fullscreen",
       "inertia",
       "inertia_deceleration",
       "inertia_max_speed",
       "interpolation",
       "keyboard",
       "keyboard_pan_offset",
       "keyboard_zoom_offset",
       "max_zoom",
       "min_zoom",
       "prefer_canvas",
       "scroll_wheel_zoom",
       "tap",
       "tap_tolerance",
       "touch_zoom",
       "world_copy_jump",
       "zoom",
       "zoom_animation_threshold",
       "zoom_delta",
       "zoom_snap"
      ],
      "panes": {},
      "prefer_canvas": false,
      "right": 295556,
      "scroll_wheel_zoom": true,
      "south": 22.141301795929245,
      "style": "IPY_MODEL_d66a36952d474a0882d8127c166cc771",
      "tap": true,
      "tap_tolerance": 15,
      "top": 457570,
      "touch_zoom": true,
      "west": -78.72081756591798,
      "window_url": "https://ttgsz7smh-496ff2e9c6d22116-0-colab.googleusercontent.com/outputframe.html?vrz=colab_20240227-060546_RC00_610708960",
      "world_copy_jump": false,
      "zoom": 12,
      "zoom_animation_threshold": 4,
      "zoom_delta": 1,
      "zoom_snap": 1
     }
    },
    "3e9db067244248959ceded14d6371ab1": {
     "model_module": "jupyter-leaflet",
     "model_module_version": "^0.18",
     "model_name": "LeafletAttributionControlModel",
     "state": {
      "_model_module": "jupyter-leaflet",
      "_model_module_version": "^0.18",
      "_model_name": "LeafletAttributionControlModel",
      "_view_count": null,
      "_view_module": "jupyter-leaflet",
      "_view_module_version": "^0.18",
      "_view_name": "LeafletAttributionControlView",
      "options": [
       "position",
       "prefix"
      ],
      "position": "bottomright",
      "prefix": "ipyleaflet"
     }
    },
    "5887a75218d34134a50fe09a7893fba9": {
     "model_module": "jupyter-leaflet",
     "model_module_version": "^0.18",
     "model_name": "LeafletMapStyleModel",
     "state": {
      "_model_module": "jupyter-leaflet",
      "_model_module_version": "^0.18",
      "_model_name": "LeafletMapStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "cursor": "grab"
     }
    },
    "76d8ce7251044a958002f66b7f29931d": {
     "model_module": "jupyter-leaflet",
     "model_module_version": "^0.18",
     "model_name": "LeafletZoomControlModel",
     "state": {
      "_model_module": "jupyter-leaflet",
      "_model_module_version": "^0.18",
      "_model_name": "LeafletZoomControlModel",
      "_view_count": null,
      "_view_module": "jupyter-leaflet",
      "_view_module_version": "^0.18",
      "_view_name": "LeafletZoomControlView",
      "options": [
       "position",
       "zoom_in_text",
       "zoom_in_title",
       "zoom_out_text",
       "zoom_out_title"
      ],
      "position": "topleft",
      "zoom_in_text": "+",
      "zoom_in_title": "Zoom in",
      "zoom_out_text": "-",
      "zoom_out_title": "Zoom out"
     }
    },
    "d157f2c06b7f4d089452873d11d92099": {
     "model_module": "jupyter-leaflet",
     "model_module_version": "^0.18",
     "model_name": "LeafletTileLayerModel",
     "state": {
      "_model_module": "jupyter-leaflet",
      "_model_module_version": "^0.18",
      "_model_name": "LeafletTileLayerModel",
      "_view_count": null,
      "_view_module": "jupyter-leaflet",
      "_view_module_version": "^0.18",
      "_view_name": "LeafletTileLayerView",
      "attribution": "&copy; <a href=\"https://www.openstreetmap.org/copyright\">OpenStreetMap</a> contributors",
      "base": true,
      "bottom": true,
      "bounds": null,
      "detect_retina": false,
      "loading": false,
      "max_native_zoom": null,
      "max_zoom": 19,
      "min_native_zoom": null,
      "min_zoom": 1,
      "name": "OpenStreetMap",
      "no_wrap": false,
      "opacity": 1,
      "options": [
       "attribution",
       "bounds",
       "detect_retina",
       "max_native_zoom",
       "max_zoom",
       "min_native_zoom",
       "min_zoom",
       "no_wrap",
       "tile_size",
       "tms",
       "zoom_offset"
      ],
      "pane": "",
      "popup": null,
      "popup_max_height": null,
      "popup_max_width": 300,
      "popup_min_width": 50,
      "show_loading": false,
      "subitems": [],
      "tile_size": 256,
      "tms": false,
      "url": "https://tile.openstreetmap.org/{z}/{x}/{y}.png",
      "visible": true,
      "zoom_offset": 0
     }
    },
    "d66a36952d474a0882d8127c166cc771": {
     "model_module": "jupyter-leaflet",
     "model_module_version": "^0.18",
     "model_name": "LeafletMapStyleModel",
     "state": {
      "_model_module": "jupyter-leaflet",
      "_model_module_version": "^0.18",
      "_model_name": "LeafletMapStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "cursor": "grab"
     }
    },
    "df2570005aae406aa0c0fe9eca0e1c05": {
     "model_module": "jupyter-leaflet",
     "model_module_version": "^0.18",
     "model_name": "LeafletMapStyleModel",
     "state": {
      "_model_module": "jupyter-leaflet",
      "_model_module_version": "^0.18",
      "_model_name": "LeafletMapStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "cursor": "move"
     }
    },
    "e6e656d1ceb849bb89dea12c8000cc14": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": "559px",
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": "559px"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}