view_products.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "import boto3\n",
    "import h5py\n",
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# general structure for scenes in s3\n",
    "# s3_bucket/software/dem/scene/scene_files\n",
    "s3_bucket = 'deant-data-public-dev'\n",
    "aws_credentials = 'credentials/credentials_aws.yaml'\n",
    "\n",
    "with open(aws_credentials, \"r\", encoding='utf8') as f:\n",
    "    aws_cfg = yaml.safe_load(f.read())\n",
    "\n",
    "# setup s3\n",
    "s3 = boto3.client(\n",
    "    's3', \n",
    "    region_name=aws_cfg['AWS_DEFAULT_REGION'],\n",
    "    aws_access_key_id=aws_cfg['AWS_ACCESS_KEY_ID'],\n",
    "    aws_secret_access_key= aws_cfg['AWS_SECRET_ACCESS_KEY'])\n",
    "\n",
    "# make data directory to store local files\n",
    "data_dir = '/data/scratch'\n",
    "os.makedirs(data_dir, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set the scene we want to look at\n",
    "software = 'rtc-opera'\n",
    "dem = 'glo_30'\n",
    "crs = '3031'\n",
    "scene = 'S1B_IW_SLC__1SSH_20190315T195015_20190315T195045_015369_01CC73_DB8B'\n",
    "h5_file = f'OPERA_L2_RTC-{scene}.h5'\n",
    "s3_prefix = f'{software}/{dem}/{crs}/{scene}/{h5_file}'\n",
    "local_path = f'{data_dir}/{h5_file}'\n",
    "# download file\n",
    "download = True\n",
    "if download:\n",
    "    print(f'downloading {s3_prefix}')\n",
    "    s3.download_file(s3_bucket, s3_prefix, local_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the h5 file\n",
    "h5_data = h5py.File(local_path,'r')\n",
    "print('keys')\n",
    "print(list(h5_data))\n",
    "print('data keys')\n",
    "print(list(h5_data['data'].keys()))\n",
    "print('metadata keys')\n",
    "print(list(h5_data['metadata'].keys()))\n",
    "print(list(h5_data['metadata']['processingInformation']['parameters'].keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_array(\n",
    "    arr, \n",
    "    cmap='binary_r',\n",
    "    vmin=None,\n",
    "    vmax=None,\n",
    "    title='',\n",
    "    ylabel='',\n",
    "    save_path=''):\n",
    "\n",
    "    stats_arr = np.array(arr)[np.array((np.isfinite(arr)))]\n",
    "    print('Array Stats (excl nodata)')\n",
    "    print(f'min: {stats_arr.min()}', \n",
    "        f'max: {stats_arr.max()}',\n",
    "        f'mean: {stats_arr.mean()}',\n",
    "        f'median: {np.percentile(stats_arr, 50)}',\n",
    "        f'5th percentile: {np.percentile(stats_arr, 5)}',\n",
    "        f'90th percentile: {np.percentile(stats_arr, 95)}',\n",
    "        )\n",
    "\n",
    "    # calculate percentiles if vmin or vmax is set as 'PXX'\n",
    "    # e.g. vmin = 'P5'\n",
    "    if vmin:\n",
    "        if str(vmin[0]).upper() == 'P':\n",
    "            vmin = np.percentile(stats_arr,int(vmin[1:]))\n",
    "    if vmax:\n",
    "        if str(vmax[0]).upper() == 'P':\n",
    "            vmax = np.percentile(stats_arr,int(vmax[1:]))\n",
    "\n",
    "    f, ax = plt.subplots(figsize=(10,10))\n",
    "    im = ax.imshow(arr, \n",
    "            vmin = vmin, \n",
    "            vmax = vmax,\n",
    "            cmap = cmap)\n",
    "    ax.set_title(title)\n",
    "    f.colorbar(im, ax=ax, label=ylabel)\n",
    "    if save_path:\n",
    "        plt.savefig(save_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,k in enumerate(h5_data['data'].keys()):\n",
    "    print(k)\n",
    "    try:\n",
    "        plot_array(h5_data['data'][k],\n",
    "                save_path=os.path.join(data_dir,f'{scene}_{k}.jpg'),\n",
    "                vmin='P5',\n",
    "                vmax='P95')\n",
    "    except:\n",
    "        f'skipping, no image for: {k}'\n",
    "    # if i == 0:\n",
    "    #     break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rtc_otf_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}