notebooks/sliderule2geoparquet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "da6bab27-3d9c-46dd-b80c-39cdd6bd9635",
   "metadata": {
    "tags": [],
    "user_expressions": []
   },
   "source": [
    "## Generate Geoparquet Data Samples\n",
    "\n",
    "1. Convert ATL03 samples (whole granules) to geoparquet\n",
    "2. Use SlideRule with the geoparquet export option to download granules by ID\n",
    "3. Save to common bucket\n",
    "\n",
    "Following https://icesat-2-2023.hackweek.io/tutorials/sliderule/parquet-s3.html "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "0777eaa8-6dc6-4a93-93eb-25db76f7e95c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['USE_PYGEOS'] = '0'\n",
    "from sliderule import icesat2, earthdata, io\n",
    "import boto3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c36be52a-a09f-46d8-872b-ce0c93ec9841",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ATL03_20181019224323_03250112_005_01.h5'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#figure out the format sliderule expects granule IDs to be in\n",
    "# find granules for a spatial and temporal query\n",
    "box_lon = [-105, -105, -100, -100, -105]\n",
    "box_lat = [-75, -77.5, -77.5, -75, -75]\n",
    "poly = io.to_region(box_lon, box_lat)\n",
    "resources = earthdata.cmr(short_name='ATL03', polygon=poly, time_start='2018-10-19', time_end='2018-10-20') \n",
    "granule = resources[0]\n",
    "\n",
    "granule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b15d4206-dfe3-46d6-b4ec-b654ceaa5141",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-08-08 23:45:34 7754735138 ATL03_20181120182818_08110112_006_02.h5\n",
      "2023-08-08 23:47:04 6997123664 ATL03_20190219140808_08110212_006_02.h5\n",
      "2023-08-08 23:47:04 6925710500 ATL03_20200217204710_08110612_006_01.h5\n",
      "2023-08-08 23:47:04 8392279594 ATL03_20211114142614_08111312_006_01.h5\n",
      "2023-08-08 23:47:04 7954039827 ATL03_20230211164520_08111812_006_01.h5\n"
     ]
    }
   ],
   "source": [
    "!aws s3 ls s3://nasa-cryo-scratch/h5cloud/original/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bdec9620-4730-4f81-8d19-6afb11ce5fca",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# initialize\n",
    "icesat2.init(\"slideruleearth.io\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "48bf8260-c72e-41fa-87bd-fec3b2383d40",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "granules = ['ATL03_20181120182818_08110112_006_02.h5',\n",
    "            'ATL03_20190219140808_08110212_006_02.h5',\n",
    "            'ATL03_20200217204710_08110612_006_01.h5',\n",
    "            'ATL03_20211114142614_08111312_006_01.h5',\n",
    "            'ATL03_20230211164520_08111812_006_01.h5'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "2f0f6fad-df48-44bf-ad16-136282c6a975",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Cyrocloud specific permissions to bucket\n",
    "client = boto3.client('sts')\n",
    "\n",
    "with open(os.environ['AWS_WEB_IDENTITY_TOKEN_FILE']) as f:\n",
    "    TOKEN = f.read()\n",
    "\n",
    "response = client.assume_role_with_web_identity(\n",
    "    RoleArn=os.environ['AWS_ROLE_ARN'],\n",
    "    RoleSessionName=os.environ['JUPYTERHUB_CLIENT_ID'],\n",
    "    WebIdentityToken=TOKEN,\n",
    "    DurationSeconds=3600\n",
    ")\n",
    "\n",
    "ACCESS_KEY_ID = response['Credentials']['AccessKeyId']\n",
    "SECRET_ACCESS_KEY_ID = response['Credentials']['SecretAccessKey']\n",
    "SESSION_TOKEN = response['Credentials']['SessionToken']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0ad4483b-6152-405f-b986-24d62b3ce568",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_gpq(granule):\n",
    "    #granule = \"ATL03_20181120182818_08110112_006_02.h5\"\n",
    "    asset = \"icesat2\"\n",
    "    output = f\"s3://nasa-cryo-scratch/h5cloud/geoparquet/{granule}.gpq\"\n",
    "    #output = f\"/home/jovyan/{granule}.gpq\"\n",
    "    params = {\n",
    "        \"output\" : {\n",
    "            \"path\" : output,\n",
    "            \"format\" : \"parquet\",\n",
    "            \"open_on_complete\" : False,\n",
    "            \"region\": \"us-west-2\",\n",
    "            \"credentials\": {\n",
    "                 \"aws_access_key_id\": ACCESS_KEY_ID,\n",
    "                 \"aws_secret_access_key\": SECRET_ACCESS_KEY_ID,\n",
    "                 \"aws_session_token\": SESSION_TOKEN\n",
    "             }\n",
    "        }\n",
    "    }\n",
    "    status = icesat2.atl03s(parm=params, resource=granule, asset=asset)\n",
    "    \n",
    "    return status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d6ccc6ea-31c4-428d-a2dd-aa2fa9f678db",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 53.7 ms, sys: 4.1 ms, total: 57.8 ms\n",
      "Wall time: 3min 58s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "status = icesat2.atl03s(params, granule, asset=asset) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "5d972d1b-789a-428a-8691-1188b1d79b0a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3://nasa-cryo-scratch/h5cloud/geoparquet/ATL03_20190219140808_08110212_006_02.h5.gpq\n",
      "s3://nasa-cryo-scratch/h5cloud/geoparquet/ATL03_20200217204710_08110612_006_01.h5.gpq\n",
      "s3://nasa-cryo-scratch/h5cloud/geoparquet/ATL03_20211114142614_08111312_006_01.h5.gpq\n",
      "s3://nasa-cryo-scratch/h5cloud/geoparquet/ATL03_20230211164520_08111812_006_01.h5.gpq\n",
      "CPU times: user 149 ms, sys: 20.2 ms, total: 169 ms\n",
      "Wall time: 22min 35s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "for granule in granules[1:]:\n",
    "    result = get_gpq(granule)\n",
    "    print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "2b238230-0182-4c07-9c0d-64bb78d5267d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ATL03_20190219140808_08110212_006_02.h5',\n",
       " 'ATL03_20200217204710_08110612_006_01.h5',\n",
       " 'ATL03_20211114142614_08111312_006_01.h5',\n",
       " 'ATL03_20230211164520_08111812_006_01.h5']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "granules[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "67587c89-6631-4e2b-9e2e-5d78cbe78794",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'s3://nasa-cryo-scratch/h5cloud/geoparquet/ATL03_20181120182818_08110112_006_02.h5.gpq'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "status"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}