Climate_bash.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Identification of interesting VIIRS/MODIS files\n",
    "\n",
    "This notebook process a general bash file that retrieve VIIRS and MODIS images to return a new bash file with the images that mathches ICESat-2 data in a given temporal window.\n",
    "\n",
    "Prerequisites:\n",
    "- bash script file with the name of all the VIIRS/MODIS files in a given specific region (see https://search.earthdata.nasa.gov/search?m=-41.467241310846475!75.05859375!4!1!0!0%2C2)\n",
    "\n",
    "Imput:\n",
    "- Maximum temporal tolerance between ICESat-2 ann VIIRS/MOIDS retrievals\n",
    "- Spatial extent: this should be a subset of the spatial extent used to retrieve the scipt with the names of all the VIIRS/MODIS files\n",
    "\n",
    "Output:\n",
    "- Modified bash file with the names of the VIIRS/MODIS files that temporaly match with retrievals from ICESat-2\n",
    "\n",
    "\n",
    "\n",
    "In order to download the files, make the script an executable by running the line 'chmod 777 download.sh' from the command line. After that is complete, the file can be executed by typing './download.sh'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from icepyx import query as ipd\n",
    "\n",
    "from utils_cloud import viirs_date, modis_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Spatial extend for the ICESat-2 search \n",
    "# Observation: in case that the VIIRS file retrieve images in a different spatial extent, then we are no going to have matches, \n",
    "# independently of the window. \n",
    "#spatial_extent = [-50, 70, -49, 71]\n",
    "spatial_extent = [-50, 68, -45, 73]\n",
    "\n",
    "# Temporal window in minutes\n",
    "minutes = 20\n",
    "hr = minutes / 60 \n",
    "# Maximim number of files we are interested in retrieve \n",
    "max_filtered = 2#1000000000000\n",
    "\n",
    "\n",
    "# lenght of line if bash file with VIIRS imagen name\n",
    "viirs_length_bash = 146\n",
    "# lenght of line if bash file with MODIS imagen name\n",
    "modis_aqua_length_bash = 121\n",
    "modis_terra_length_bash = 121\n",
    "\n",
    "\n",
    "# path with bash retrieval files\n",
    "#path_retrieve = \"data/VIIRS_bash/\"\n",
    "path_retrieve = \"data/Earthdata_scripts/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Select your satellite data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#satellite = \"VIIRS\"\n",
    "satellite = \"MODIS-Aqua\"\n",
    "#satellite = \"MODIS-Terra\"\n",
    "\n",
    "if satellite == \"VIIRS\":\n",
    "    \n",
    "    # name of the original bash file to retrieve the data\n",
    "    exe_old = \"VIIRS-Greenland-download.sh\"\n",
    "    # name of the output bash file \n",
    "    exe_new = \"VIIRS-Greenland-download-filtered.sh\"\n",
    "    lenght_bash = viirs_length_bash\n",
    "    \n",
    "if satellite == \"MODIS-Aqua\":\n",
    "    \n",
    "    exe_old = \"MODIS-Aqua-Greenland-download.sh\"\n",
    "    exe_new = \"MODIS-Aqua-Greenland-download-filtered.sh\"    \n",
    "    lenght_bash = modis_aqua_length_bash\n",
    "    \n",
    "if satellite == \"MODIS-Terra\":\n",
    "    \n",
    "    exe_old = \"MODIS-Terra-Greenland-download.sh\"\n",
    "    exe_new = \"MODIS-Terra-Greenland-download-filtered.sh\"    \n",
    "    lenght_bash = modis_terra_length_bash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = open(path_retrieve + exe_old, 'r')\n",
    "new_file = open(path_retrieve + exe_new, 'x')\n",
    "\n",
    "#viirs_names = []\n",
    "\n",
    "counter = 0\n",
    "counter_filtered = 0\n",
    "\n",
    "for line in file:\n",
    "    \n",
    "    #print(line)\n",
    "    \n",
    "    if (\"https://ladsweb.modaps.eosdis.nasa.gov\" in line) and (len(line) == lenght_bash):\n",
    "                \n",
    "        if counter_filtered < max_filtered:\n",
    "\n",
    "            if satellite == \"VIIRS\":\n",
    "                Vfile = line[:-1].split(\"/\")[-1]\n",
    "                f_t = viirs_date(Vfile)\n",
    "\n",
    "            if satellite == \"MODIS-Aqua\" or satellite == \"MODIS-Terra\":\n",
    "                Mafile = line[:-1].split(\"/\")[-1]\n",
    "                #print(Mafile)\n",
    "                f_t = modis_date(Mafile)\n",
    "                #print(f_t)\n",
    "                \n",
    "            # Temporal search window for \n",
    "            start = f_t - pd.DateOffset(hours=hr)\n",
    "            end   = f_t + pd.DateOffset(hours=hr)\n",
    "\n",
    "            start_date_str = start.strftime('%Y-%m-%d')\n",
    "            end_date_str   = end.strftime('%Y-%m-%d')\n",
    "            start_time_str = start.strftime('%H:%M:%S')\n",
    "            end_time_str   = end.strftime('%H:%M:%S')\n",
    "\n",
    "            try:\n",
    "\n",
    "                region_a = ipd.Query(\"ATL06\", spatial_extent, [start_date_str, end_date_str], start_time_str, end_time_str)  # This was different before\n",
    "                avail_granules = region_a.avail_granules(ids=True)\n",
    "\n",
    "                #viirs_names.append(Vfile)\n",
    "                #new_file.write(Vfile + \"\\n\") # add \\n\n",
    "                assert len(avail_granules) > 0\n",
    "                new_file.write(line)\n",
    "                counter_filtered += 1\n",
    "\n",
    "            except AssertionError:\n",
    "                None\n",
    "                \n",
    "            counter += 1\n",
    "        \n",
    "        if counter % 50 == 0:\n",
    "            print(\">>> There are a total of\", counter_filtered, \"files founded out of\", counter, \"file names.\")\n",
    "            \n",
    "        #break\n",
    "        \n",
    "    else:\n",
    "        \n",
    "        new_file.write(line)\n",
    "        \n",
    "    \n",
    "new_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:.conda-cloudmaskenv]",
   "language": "python",
   "name": "conda-env-.conda-cloudmaskenv-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}