Batch Processing-10Band.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 10-band Batch Processing Example\n",
    "\n",
    "In this example, we use the `micasense.imageset` class to load a set of directories of images into a list of `micasense.capture` objects, and we iterate over that list saving out each image as an aligned stack of images as separate bands in a single tiff file each. Next, we use the metadata from the original captures to write out a log file of the captures and their locations.  Finally, we use `exiftool` from the command line to inject that metadata into the processed images, allowing us to stitch those images using commercial software such as Pix4D or Agisoft.\n",
    "\n",
    "For an example dataset, download and unzip either sample dataset at https://www.micasense.com/dual-camera-sample-data into your Downloads folder, and ensure the paths below point to the correct location."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Images into ImageSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywidgets import FloatProgress, Layout\n",
    "from IPython.display import display\n",
    "import micasense.imageset as imageset\n",
    "import micasense.capture as capture\n",
    "import os, glob\n",
    "import multiprocessing\n",
    "from pathlib import Path\n",
    "\n",
    "panelNames = None\n",
    "useDLS = True\n",
    "\n",
    "imagePath = os.path.expanduser(os.path.join('.','data','REDEDGE-MX-DUAL'))\n",
    "panelNames = glob.glob(os.path.join(imagePath,'IMG_0001_*.tif'))\n",
    "\n",
    "outputPath = os.path.join(imagePath,'..','stacks')\n",
    "thumbnailPath = os.path.join(outputPath, '..', 'thumbnails')\n",
    "\n",
    "\n",
    "\n",
    "overwrite = False # Set to False to continue interrupted processing\n",
    "generateThumbnails = True\n",
    "\n",
    "# Allow this code to align both radiance and reflectance images; bu excluding\n",
    "# a definition for panelNames above, radiance images will be used\n",
    "# For panel images, efforts will be made to automatically extract the panel information\n",
    "# but if the panel/firmware is before Altum 1.3.5, RedEdge 5.1.7 the panel reflectance\n",
    "# will need to be set in the panel_reflectance_by_band variable.\n",
    "# Note: radiance images will not be used to properly create NDVI/NDRE images below.\n",
    "if panelNames is not None:\n",
    "    panelCap = capture.Capture.from_filelist(panelNames)\n",
    "else:\n",
    "    panelCap = None\n",
    "\n",
    "if panelCap is not None:\n",
    "    if panelCap.panel_albedo() is not None:\n",
    "        panel_reflectance_by_band = panelCap.panel_albedo()\n",
    "    else:\n",
    "        panel_reflectance_by_band = [0.65]*len(panelCap.images) #inexact, but quick\n",
    "    panel_irradiance = panelCap.panel_irradiance(panel_reflectance_by_band)    \n",
    "    img_type = \"reflectance\"\n",
    "else:\n",
    "    if useDLS:\n",
    "        img_type='reflectance'\n",
    "    else:\n",
    "        img_type = \"radiance\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "279fc40758f9463b8d8c7def1f4bb514",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, description='Loading', layout=Layout(width='100%'), max=1.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 60.7 ms, sys: 12.4 ms, total: 73.1 ms\n",
      "Wall time: 471 ms\n"
     ]
    }
   ],
   "source": [
    "## This progress widget is used for display of the long-running process\n",
    "f = FloatProgress(min=0, max=1, layout=Layout(width='100%'), description=\"Loading\")\n",
    "display(f)\n",
    "def update_f(val):\n",
    "    if (val - f.value) > 0.005 or val == 1: #reduces cpu usage from updating the progressbar by 10x\n",
    "        f.value=val\n",
    "\n",
    "%time imgset = imageset.ImageSet.from_directory(imagePath, progress_callback=update_f)\n",
    "update_f(1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/stephen/anaconda3/envs/micasense/lib/python3.7/site-packages/IPython/core/display.py:724: UserWarning: Consider using IPython.display.IFrame instead\n",
      "  warnings.warn(\"Consider using IPython.display.IFrame instead\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<iframe id=\"map\", srcdoc=\"<!DOCTYPE html>\n",
       "<html>\n",
       "<head>\n",
       "<title>mapboxgl-jupyter viz</title>\n",
       "<meta charset='UTF-8' />\n",
       "<meta name='viewport'\n",
       "      content='initial-scale=1,maximum-scale=1,user-scalable=no' />\n",
       "<script type='text/javascript'\n",
       "        src='https://api.tiles.mapbox.com/mapbox-gl-js/v1.0.0/mapbox-gl.js'></script>\n",
       "<link type='text/css'\n",
       "      href='https://api.tiles.mapbox.com/mapbox-gl-js/v1.0.0/mapbox-gl.css' \n",
       "      rel='stylesheet' />\n",
       "\n",
       "<style type='text/css'>\n",
       "    body { margin:0; padding:0; }\n",
       "    .map { position: absolute; top:0; bottom:0; width:100%; }\n",
       "    .legend {\n",
       "        background-color: white;\n",
       "        color: #6e6e6e;\n",
       "        border-radius: 3px;\n",
       "        bottom: 10px;\n",
       "        box-shadow: 0 1px 2px rgba(0, 0, 0, 0.10);\n",
       "        font: 12px/20px 'Helvetica Neue', Arial, Helvetica, sans-serif;\n",
       "        padding: 0;\n",
       "        position: absolute;\n",
       "        right: 10px;\n",
       "        z-index: 1;\n",
       "        min-width: 100px;\n",
       "    }\n",
       "   .legend.horizontal {bottom: 10px; text-align: left;}\n",
       "\n",
       "    /* legend header */\n",
       "    .legend .legend-header { border-radius: 3px 3px 0 0; background: white; }\n",
       "    .legend .legend-title {\n",
       "        padding: 6px 12px 6px 12px;\n",
       "        text-shadow: 0 0 2px white;\n",
       "        text-transform: capitalize;\n",
       "        text-align: center;\n",
       "        font-weight: bold !important;\n",
       "        font-size: 14px;\n",
       "        font: 12px/20px 'Helvetica Neue', Arial, Helvetica, sans-serif;\n",
       "        max-width: 160px;\n",
       "    }\n",
       "    .legend-title {padding: 6px 12px 6px 12px; text-shadow: 0 0 2px #FFF; text-transform: capitalize; text-align: center; max-width: 160px; font-size: 0.9em; font-weight: bold;}\n",
       "    .legend.horizontal .legend-title {text-align: left;}\n",
       "\n",
       "    /* legend items */\n",
       "    .legend-content {margin: 6px 12px 6px 12px; overflow: hidden; padding: 0; float: left; list-style: none; font-size: 0.8em;}\n",
       "    .legend.vertical .legend-item {white-space: nowrap;}\n",
       "    .legend-value {display: inline-block; line-height: 18px; vertical-align: top;}\n",
       "    .legend.horizontal ul.legend-content li.legend-item .legend-value,\n",
       "    .legend.horizontal ul.legend-content li.legend-item {display: inline-block; float: left; width: 30px; margin-bottom: 0; text-align: center; min-height: 30px;}\n",
       "\n",
       "    /* legend key styles */\n",
       "    .legend-key {display: inline-block; height: 10px;}\n",
       "    .legend-key.default, .legend-key.square {border-radius: 0;}\n",
       "    .legend-key.circle {border-radius: 50%;}\n",
       "    .legend-key.rounded-square {border-radius: 20%;}\n",
       "    .legend.vertical .legend-key {width: 10px; margin-right: 5px; margin-left: 1px;}\n",
       "    .legend.horizontal .legend-key {width: 30px; margin-right: 0; margin-top: 1px; float: left;}\n",
       "    .legend.horizontal .legend-key.square, .legend.horizontal .legend-key.rounded-square, .legend.horizontal .legend-key.circle {margin-left: 10px; width: 10px;}\n",
       "    .legend.horizontal .legend-key.line {margin-left: 5px;}\n",
       "    .legend.horizontal .legend-key.line, .legend.vertical .legend-key.line {border-radius: 10%; width: 20px; height: 3px; margin-bottom: 2px;}\n",
       "\n",
       "    /* gradient bar alignment */\n",
       "    .gradient-bar {margin: 6px 12px 6px 12px;}\n",
       "    .legend.horizontal .gradient-bar {width: 88%; height: 10px;}\n",
       "    .legend.vertical .gradient-bar {width: 10px; min-height: 50px; position: absolute; bottom: 4px;}\n",
       "\n",
       "    /* contiguous vertical bars (discrete) */\n",
       "    .legend.vertical.contig .legend-key {height: 15px; width: 10px;}\n",
       "    .legend.vertical.contig li.legend-item {height: 15px;}\n",
       "    .legend.vertical.contig {padding-bottom: 6px;}\n",
       "\n",
       "    /* vertical radius legend */\n",
       "    .legend.horizontal.legend-variable-radius ul.legend-content li.legend-item .legend-value,\n",
       "    .legend.horizontal.legend-variable-radius ul.legend-content li.legend-item {width: 30px; min-height: 20px;}\n",
       "\n",
       "    /* scale annotation */\n",
       "    .mapboxgl-ctrl.mapboxgl-ctrl-scale { border-color: #6e6e6e; \n",
       "                                         background-color: white; \n",
       "                                         color: #131516; }\n",
       "</style>\n",
       "\n",
       "\n",
       "<style>\n",
       "    .gradient-bar.bordered, .legend-key.bordered { border: solid grey 0.1px; }\n",
       "</style>\n",
       "\n",
       "\n",
       "</head>\n",
       "<body>\n",
       "\n",
       "<div id='map' class='map'></div>\n",
       "\n",
       "<script type='text/javascript'>\n",
       "\n",
       "var legendHeader;\n",
       "\n",
       "function calcColorLegend(myColorStops, title) {\n",
       "    // create legend\n",
       "    var legend = document.createElement('div'),\n",
       "        legendContainer = document.getElementsByClassName('mapboxgl-ctrl-bottom-right')[0];\n",
       "\n",
       "    if ('circle' === 'contiguous-bar') {\n",
       "        legend.className = 'legend vertical contig';\n",
       "    }\n",
       "    else {\n",
       "        legend.className = 'legend vertical';\n",
       "    }\n",
       "    legend.id = 'legend-0';\n",
       "    document.body.appendChild(legend);\n",
       "    // add legend header and content elements\n",
       "    var mytitle = document.createElement('div'),\n",
       "        legendContent = document.createElement('ul');\n",
       "    legendHeader = document.createElement('div');\n",
       "    mytitle.textContent = title;\n",
       "    mytitle.className = 'legend-title'\n",
       "    legendHeader.className = 'legend-header'\n",
       "    legendContent.className = 'legend-content'\n",
       "    legendHeader.appendChild(mytitle);\n",
       "    legend.appendChild(legendHeader);\n",
       "    legend.appendChild(legendContent);\n",
       "    if (false === true) {\n",
       "        var gradientText = 'linear-gradient(to right, ',\n",
       "            gradient = document.createElement('div');\n",
       "        gradient.className = 'gradient-bar';\n",
       "        legend.appendChild(gradient);\n",
       "    }\n",
       "    // calculate a legend entries on a Mapbox GL Style Spec property function stops array\n",
       "    for (p = 0; p < myColorStops.length; p++) {\n",
       "        if (!!document.getElementById('legend-color-points-value-' + p)) {\n",
       "            // update the legend if it already exists\n",
       "            document.getElementById('legend-color-points-value-' + p).textContent = myColorStops[p][0];\n",
       "            document.getElementById('legend-color-points-id-' + p).style.backgroundColor = myColorStops[p][1];\n",
       "        }\n",
       "        else {\n",
       "            // create the legend if it doesn't yet exist\n",
       "            var item = document.createElement('li');\n",
       "            item.className = 'legend-item';\n",
       "            var key = document.createElement('span');\n",
       "            key.className = 'legend-key circle';\n",
       "            key.id = 'legend-color-points-id-' + p;\n",
       "            key.style.backgroundColor = myColorStops[p][1];\n",
       "            var value = document.createElement('span');\n",
       "            value.className = 'legend-value';\n",
       "            value.id = 'legend-color-points-value-' + p;\n",
       "            item.appendChild(key);\n",
       "            item.appendChild(value);\n",
       "            legendContent.appendChild(item);\n",
       "            \n",
       "            data = document.getElementById('legend-color-points-value-' + p)\n",
       "            // round number values in legend if precision defined\n",
       "            if ((typeof(myColorStops[p][0]) == 'number') && (typeof(null) == 'number')) {\n",
       "                data.textContent = myColorStops[p][0].toFixed(null);\n",
       "            }\n",
       "            else {\n",
       "                data.textContent = myColorStops[p][0];\n",
       "            }\n",
       "            // add color stop to gradient list\n",
       "            if (false === true) {\n",
       "                if (p < myColorStops.length - 1) {\n",
       "                    gradientText = gradientText + myColorStops[p][1] + ', ';\n",
       "                }\n",
       "                else {\n",
       "                    gradientText = gradientText + myColorStops[p][1] + ')';\n",
       "                }\n",
       "                if ('vertical' === 'vertical') {\n",
       "                    gradientText = gradientText.replace('to right', 'to bottom');\n",
       "                }\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "    if (false === true) {\n",
       "        // convert to gradient scale appearance\n",
       "        gradient.style.background = gradientText;\n",
       "        // hide legend keys generated above\n",
       "        var keys = document.getElementsByClassName('legend-key');\n",
       "        for (var i=0; i < keys.length; i++) {\n",
       "            keys[i].style.visibility = 'hidden';\n",
       "        }\n",
       "        if ('vertical' === 'vertical') {\n",
       "            gradient.style.height = (legendContent.offsetHeight - 6) + 'px';\n",
       "        }\n",
       "    }\n",
       "    // add class for styling bordered legend keys\n",
       "    if (true) {\n",
       "        var keys = document.getElementsByClassName('legend-key');\n",
       "        for (var i=0; i < keys.length; i++) {\n",
       "            if (keys[i]) {\n",
       "                keys[i].classList.add('bordered');\n",
       "            }\n",
       "        }\n",
       "        var gradientBars = document.getElementsByClassName('gradient-bar');\n",
       "        for (var i=0; i < keys.length; i++) {\n",
       "            if (gradientBars[i]) {\n",
       "                gradientBars[i].classList.add('bordered');\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "    // update right-margin for compact Mapbox attribution based on calculated legend width\n",
       "    updateAttribMargin(legend);\n",
       "    updateLegendMargin(legend);\n",
       "}\n",
       "\n",
       "\n",
       "function calcRadiusLegend(myRadiusStops, title, color) {\n",
       "\n",
       "    // maximum legend item height\n",
       "    var maxLegendItemHeight = 2 * myRadiusStops[myRadiusStops.length - 1][1];\n",
       "\n",
       "    // create legend\n",
       "    var legend = document.createElement('div');\n",
       "    legend.className = 'legend vertical legend-variable-radius';\n",
       "\n",
       "    legend.id = 'legend-1';\n",
       "    document.body.appendChild(legend);\n",
       "\n",
       "    // add legend header and content elements\n",
       "    var mytitle = document.createElement('div'),\n",
       "        legendContent = document.createElement('ul');\n",
       "    legendHeader = document.createElement('div');\n",
       "    mytitle.textContent = title;\n",
       "    mytitle.className = 'legend-title'\n",
       "    legendHeader.className = 'legend-header'\n",
       "    legendContent.className = 'legend-content'\n",
       "    legendHeader.appendChild(mytitle);\n",
       "    legend.appendChild(legendHeader);\n",
       "    legend.appendChild(legendContent);\n",
       "\n",
       "    // calculate a legend entries on a Mapbox GL Style Spec property function stops array\n",
       "    for (p = 0; p < myRadiusStops.length; p++) {\n",
       "        if (!!document.getElementById('legend-radius-points-value-' + p)) {\n",
       "            //update the legend if it already exists\n",
       "            document.getElementById('legend-radius-points-value-' + p).textContent = myRadiusStops[p][0];\n",
       "            document.getElementById('legend-radius-points-id-' + p).style.backgroundColor = color;\n",
       "        }\n",
       "        else {\n",
       "            // create the legend if it doesn't yet exist\n",
       "            var item = document.createElement('li');\n",
       "            item.className = 'legend-item';\n",
       "            item.height = '' + maxLegendItemHeight + 'px';\n",
       "\n",
       "            var key = document.createElement('span');\n",
       "            key.className = 'legend-key circle';\n",
       "            key.id = 'legend-radius-points-id-' + p;\n",
       "            key.style.backgroundColor = color;   \n",
       "\n",
       "            key.style.width = '' + myRadiusStops[p][1] * 2 + 'px';\n",
       "            key.style.height = '' + myRadiusStops[p][1] * 2 + 'px';\n",
       "\n",
       "            keyVerticalMargin = (maxLegendItemHeight - myRadiusStops[p][1] * 2) * 0.5;\n",
       "            key.style.marginTop = '' + keyVerticalMargin + 'px';\n",
       "            key.style.marginBottom = '' + keyVerticalMargin + 'px';\n",
       "\n",
       "            var value = document.createElement('span');\n",
       "            value.className = 'legend-value';\n",
       "            value.id = 'legend-radius-points-value-' + p;\n",
       "\n",
       "            item.appendChild(key);\n",
       "            item.appendChild(value);\n",
       "            legendContent.appendChild(item);\n",
       "            \n",
       "            data = document.getElementById('legend-radius-points-value-' + p)\n",
       "\n",
       "            // round number values in legend if precision defined\n",
       "            if ((typeof(myRadiusStops[p][0]) == 'number') && (typeof(null) == 'number')) {\n",
       "                data.textContent = myRadiusStops[p][0].toFixed(null);\n",
       "            }\n",
       "            else {\n",
       "                data.textContent = myRadiusStops[p][0];\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "\n",
       "    // add class for styling bordered legend keys\n",
       "    if (true) {\n",
       "        var keys = document.getElementsByClassName('legend-key');\n",
       "        for (var i=0; i < keys.length; i++) {\n",
       "            if (keys[i]) {\n",
       "                keys[i].classList.add('bordered');\n",
       "            }\n",
       "        }\n",
       "    }\n",
       "\n",
       "    // update right-margin for compact Mapbox attribution based on calculated legend width\n",
       "    updateAttribMargin(legend);\n",
       "    updateLegendMargin(legend);\n",
       "\n",
       "}\n",
       "\n",
       "\n",
       "function updateAttribMargin(legend) {\n",
       "\n",
       "    // default margin is based on calculated legend width\n",
       "    var attribMargin = legend.offsetWidth + 15;\n",
       "    \n",
       "    // if horizontal legend layout (multiple legends are stacked vertically)\n",
       "    if ('vertical' === 'horizontal') {\n",
       "        document.getElementsByClassName('mapboxgl-ctrl-attrib')[0].style.marginRight = (attribMargin).toString() + 'px';\n",
       "    }\n",
       "    // vertical legend layout means multiple legends are side-by-side\n",
       "    else if ('vertical' === 'vertical') {\n",
       "        var currentMargin = Number(document.getElementsByClassName('mapboxgl-ctrl-attrib')[0].style.marginRight.replace('px', ''));\n",
       "        document.getElementsByClassName('mapboxgl-ctrl-attrib')[0].style.marginRight = (attribMargin + currentMargin).toString() + 'px';\n",
       "    }\n",
       "}\n",
       "\n",
       "\n",
       "function updateLegendMargin(legend) {\n",
       "\n",
       "    var verticalLegends = document.getElementsByClassName('legend vertical'),\n",
       "        horizontalLegends = document.getElementsByClassName('legend horizontal');\n",
       "\n",
       "    if (verticalLegends.length > 1) {\n",
       "        for (i = 1; i < verticalLegends.length; i++) {\n",
       "            verticalLegends[i].style.marginRight = (legend.offsetWidth - 5).toString() + 'px';\n",
       "            var legend = verticalLegends[i];\n",
       "        }\n",
       "    }\n",
       "    else if (horizontalLegends.length > 1) {\n",
       "        for (i = 1; i < horizontalLegends.length; i++) {\n",
       "            horizontalLegends[i].style.marginBottom = (legend.offsetHeight + 15).toString() + 'px';\n",
       "            var legend = horizontalLegends[i];\n",
       "        }\n",
       "    }\n",
       "}\n",
       "\n",
       "\n",
       "function generateInterpolateExpression(propertyValue, stops) {\n",
       "    var expression;\n",
       "    if (propertyValue == 'zoom') {\n",
       "        expression = ['interpolate', ['exponential', 1.2], ['zoom']]\n",
       "    }\n",
       "    else if (propertyValue == 'heatmap-density') {\n",
       "        expression = ['interpolate', ['linear'], ['heatmap-density']]\n",
       "    }\n",
       "    else {\n",
       "        expression = ['interpolate', ['linear'], ['get', propertyValue]]\n",
       "    }\n",
       "\n",
       "    for (var i=0; i<stops.length; i++) {\n",
       "        expression.push(stops[i][0], stops[i][1])\n",
       "    }\n",
       "    return expression\n",
       "}\n",
       "\n",
       "\n",
       "function generateMatchExpression(propertyValue, stops, defaultValue) {\n",
       "    var expression;\n",
       "    expression = ['match', ['get', propertyValue]]\n",
       "    for (var i=0; i<stops.length; i++) {\n",
       "        expression.push(stops[i][0], stops[i][1])\n",
       "    }\n",
       "    expression.push(defaultValue)\n",
       "    \n",
       "    return expression\n",
       "}\n",
       "\n",
       "\n",
       "function generatePropertyExpression(expressionType, propertyValue, stops, defaultValue) {\n",
       "    var expression;\n",
       "    if (expressionType == 'match') {\n",
       "        expression = generateMatchExpression(propertyValue, stops, defaultValue)\n",
       "    }\n",
       "    else if (propertyValue == 'identity') {\n",
       "        expression = ['get', propertyValue]\n",
       "    }\n",
       "    else {\n",
       "        expression = generateInterpolateExpression(propertyValue, stops)\n",
       "    }\n",
       "\n",
       "    return expression\n",
       "}\n",
       "\n",
       "</script>\n",
       "\n",
       "<!-- main map creation code, extended by mapboxgl/templates/circle.html -->\n",
       "<script type='text/javascript'>\n",
       "\n",
       "    mapboxgl.accessToken = 'pk.eyJ1IjoibWljYXNlbnNlIiwiYSI6ImNqYWx5dWNteTJ3cWYzMnBicmZid3g2YzcifQ.Zrq9t7GYocBtBzYyT3P4sw';\n",
       "\n",
       "    var transformRequest = function(url, resourceType) {\n",
       "        const isMapboxRequest = url.slice(8, 22) === 'api.mapbox.com' ||\n",
       "          url.slice(10, 26) === 'tiles.mapbox.com';\n",
       "      \n",
       "        return {\n",
       "          url: isMapboxRequest ? url.replace('?', '?pluginName=PythonMapboxgl&') : url\n",
       "        }\n",
       "    };\n",
       "\n",
       "    var map = new mapboxgl.Map({\n",
       "        container: 'map',\n",
       "        attributionControl: false,\n",
       "        style: 'mapbox://styles/mapbox/satellite-streets-v9',\n",
       "        center: [-122.14207895, 47.703668],\n",
       "        zoom: 16,\n",
       "        pitch: 0,\n",
       "        bearing: 0,\n",
       "        scrollZoom: true,\n",
       "        touchZoom: true,\n",
       "        doubleClickZoom: true,\n",
       "        boxZoom: true,\n",
       "        preserveDrawingBuffer: false,\n",
       "        transformRequest: transformRequest\n",
       "    });\n",
       "\n",
       "    \n",
       "    \n",
       "        map.addControl(new mapboxgl.AttributionControl({ compact: true }));\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "        map.addControl(new mapboxgl.NavigationControl());\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "        \n",
       "    \n",
       "        \n",
       "\n",
       "\n",
       "    \n",
       "        \n",
       "        \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    map.on('style.load', function() {\n",
       "        \n",
       "        \n",
       "\n",
       "        map.addSource('data', {\n",
       "            'type': 'geojson',\n",
       "            'data': {'type': 'FeatureCollection', 'features': [{'type': 'Feature', 'geometry': {'type': 'Point', 'coordinates': [-122.14144, 47.703622]}, 'properties': {'altitude': 11.135, 'capture_id': 'ohQlG2xArB2VuU71wWcD', 'dls-yaw': -1.7486912785, 'dls-pitch': -0.1560661742, 'dls-roll': 0.0288461905, 'irr-475': 0.8897353246, 'irr-560': 0.7990972151, 'irr-668': 0.7002288864, 'irr-842': 0.4217121605, 'irr-717': 0.5267560915, 'irr-444': 0.7556996918, 'irr-531': 0.8247714996, 'irr-650': 0.6744555664, 'irr-705': 0.6139940262, 'irr-740': 0.5729064178}}, {'type': 'Feature', 'geometry': {'type': 'Point', 'coordinates': [-122.142718, 47.703714]}, 'properties': {'altitude': 129.95, 'capture_id': 'hipPNxehlAuz3zq02gML', 'dls-yaw': -1.1411783046, 'dls-pitch': 0.4916283469, 'dls-roll': -0.0958754394, 'irr-475': 0.8891651301, 'irr-560': 0.8108716108, 'irr-668': 0.706543007, 'irr-842': 0.4295603539, 'irr-717': 0.5300914981, 'irr-444': 0.7601517487, 'irr-531': 0.8333349609, 'irr-650': 0.682999115, 'irr-705': 0.6153291321, 'irr-740': 0.57208992}}]},\n",
       "            'buffer': 1,\n",
       "            'maxzoom': 14,\n",
       "            'generateId': true\n",
       "        });\n",
       "\n",
       "        map.addLayer({\n",
       "            'id': 'label',\n",
       "            'source': 'data',\n",
       "            'type': 'symbol',\n",
       "            'maxzoom': 24,\n",
       "            'minzoom': 0,\n",
       "            'layout': {\n",
       "                \n",
       "                'text-size' : generateInterpolateExpression('zoom', [[0, 8],[22, 3* 8]] ),\n",
       "                'text-offset': [0,-1]\n",
       "            },\n",
       "            'paint': {\n",
       "                'text-halo-color': 'white',\n",
       "                'text-halo-width': generatePropertyExpression('interpolate', 'zoom', [[0,1], [18,5* 1]]),\n",
       "                'text-color': ['case',\n",
       "                    ['boolean', ['feature-state', 'hover'], false], \n",
       "                    'black', \n",
       "                    '#131516']\n",
       "            }\n",
       "        }, '' )\n",
       "        \n",
       "        map.addLayer({\n",
       "            'id': 'circle',\n",
       "            'source': 'data',\n",
       "            'type': 'circle',\n",
       "            'maxzoom': 24,\n",
       "            'minzoom': 0,\n",
       "            'paint': {\n",
       "                \n",
       "                    'circle-color': ['case',\n",
       "                        ['boolean', ['feature-state', 'hover'], false], \n",
       "                        'black',\n",
       "                        generatePropertyExpression('interpolate', 'altitude', None, 'grey' )],\n",
       "                    \n",
       "                'circle-radius' : generatePropertyExpression('interpolate', 'zoom', [[0,1], [22,10 * 1]]),\n",
       "                'circle-stroke-color': ['case',\n",
       "                    ['boolean', ['feature-state', 'hover'], false], \n",
       "                    'black', \n",
       "                    'grey'],\n",
       "                'circle-stroke-width': generatePropertyExpression('interpolate', 'zoom', [[0,0.1], [18,5* 0.1]]),\n",
       "                'circle-opacity' : 1,\n",
       "                'circle-stroke-opacity' : 1\n",
       "            }\n",
       "        }, 'label');\n",
       "        \n",
       "        \n",
       "\n",
       "        // Popups\n",
       "        \n",
       "            var popupAction = 'mousemove',\n",
       "                popupSettings =  {\n",
       "                    closeButton: false,\n",
       "                    closeOnClick: false\n",
       "                };\n",
       "        \n",
       "\n",
       "        // Create a popup\n",
       "        var popup = new mapboxgl.Popup(popupSettings);\n",
       "\n",
       "        \n",
       "\n",
       "        var hoveredStateId = 0;\n",
       "        \n",
       "        // Show the popup on mouseover\n",
       "        map.on(popupAction, function(e) {\n",
       "            \n",
       "            var features = map.queryRenderedFeatures(e.point, {layers: ['circle', 'label'] });\n",
       "\n",
       "            if (features.length > 0) {\n",
       "                map.getCanvas().style.cursor = 'pointer';\n",
       "                var f = features[0];\n",
       "                newHoveredStateId = f.id;\n",
       "                if (newHoveredStateId != hoveredStateId) {\n",
       "                    map.removeFeatureState({source: 'data', id: hoveredStateId});\n",
       "                    hoveredStateId = newHoveredStateId;\n",
       "                }\n",
       "                map.setFeatureState({source: 'data', id: hoveredStateId}, { hover: true});\n",
       "                let popup_html = '<div><li><b>Location</b>: ' + f.geometry.coordinates[0].toPrecision(6) + \n",
       "                    ', ' + f.geometry.coordinates[1].toPrecision(6) + '</li>';\n",
       "\n",
       "                for (key in f.properties) {\n",
       "                    popup_html += '<li><b> ' + key + '</b>: ' + f.properties[key] + ' </li>'\n",
       "                }\n",
       "\n",
       "                popup_html += '</div>'\n",
       "                popup.setLngLat(e.lngLat)\n",
       "                    .setHTML(popup_html)\n",
       "                    .addTo(map);\n",
       "            }\n",
       "            else {\n",
       "                map.getCanvas().style.cursor = '';\n",
       "                popup.remove();\n",
       "                map.removeFeatureState({source: 'data', id: hoveredStateId});\n",
       "            }\n",
       "        });\n",
       "\n",
       "        \n",
       "        \n",
       "        // Fly to on click\n",
       "        map.on('click', 'circle', function(e) {\n",
       "            map.easeTo({\n",
       "                center: e.features[0].geometry.coordinates\n",
       "            });\n",
       "        });\n",
       "    });\n",
       "\n",
       "\n",
       "\n",
       "</script>\n",
       "\n",
       "<!-- add capability to export map or legend to image file -->\n",
       "\n",
       "\n",
       "</body>\n",
       "</html>\" style=\"width: 100%; height: 600px;\"></iframe>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "from mapboxgl.viz import *\n",
    "from mapboxgl.utils import df_to_geojson, create_radius_stops, scale_between\n",
    "from mapboxgl.utils import create_color_stops\n",
    "import pandas as pd\n",
    "\n",
    "data, columns = imgset.as_nested_lists()\n",
    "df = pd.DataFrame.from_records(data, index='timestamp', columns=columns)\n",
    "\n",
    "#Insert your mapbox token here\n",
    "token = 'pk.eyJ1IjoibWljYXNlbnNlIiwiYSI6ImNqYWx5dWNteTJ3cWYzMnBicmZid3g2YzcifQ.Zrq9t7GYocBtBzYyT3P4sw'\n",
    "color_property = 'dls-yaw'\n",
    "color_property = 'altitude'\n",
    "num_color_classes = 8\n",
    "\n",
    "min_val = df[color_property].min()\n",
    "max_val = df[color_property].max()\n",
    "\n",
    "import jenkspy\n",
    "# breaks = jenkspy.jenks_breaks(df[color_property], nb_class=num_color_classes)\n",
    "\n",
    "# color_stops = create_color_stops(breaks,colors='YlOrRd')\n",
    "geojson_data = df_to_geojson(df,columns[3:],lat='latitude',lon='longitude')\n",
    "\n",
    "viz = CircleViz(geojson_data, access_token=token, color_property=color_property,\n",
    "#                 color_stops=color_stops,\n",
    "                center=[df['longitude'].median(),df['latitude'].median()], \n",
    "                zoom=16, height='600px',\n",
    "                style='mapbox://styles/mapbox/satellite-streets-v9')\n",
    "viz.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define which warp method to use\n",
    "For newer data sets with RigRelatives tags (images captured with RedEdge version 3.4.0 or greater with a valid calibration load, see https://support.micasense.com/hc/en-us/articles/360005428953-Updating-RedEdge-for-Pix4Dfields), we can use the RigRelatives for a simple alignment.\n",
    "\n",
    "For sets without those tags, or sets that require a RigRelatives optimization, we can go through the Alignment.ipynb notebook and get a set of `warp_matrices` that we can use here to align."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import array\n",
    "from numpy import float32\n",
    "\n",
    "# Use the warp_matrices derived from the Alignment Tutorial for this RedEdge set without RigRelatives\n",
    "warp_matrices = [array([[ 9.7928989e-01,  1.3615261e-04,  1.9574374e-02],\n",
    "       [-3.7023663e-03,  9.9245304e-01,  2.8355631e+01],\n",
    "       [-1.0651237e-05,  2.6095395e-06,  1.0000000e+00]], dtype=float32), array([[ 9.9525303e-01, -7.0169556e-04, -6.3409243e+00],\n",
    "       [-7.1914408e-05,  9.9811482e-01, -6.2239196e-02],\n",
    "       [-3.6460631e-06,  1.8568957e-06,  1.0000000e+00]], dtype=float32), array([[ 9.9596852e-01, -2.5820474e-03, -1.2950540e+01],\n",
    "       [ 2.8751660e-03,  9.9919146e-01,  2.6645420e+01],\n",
    "       [-1.8280600e-06,  2.3902323e-06,  1.0000000e+00]], dtype=float32), array([[ 9.9592507e-01,  4.5777867e-03,  1.4982515e+01],\n",
    "       [-3.4031910e-03,  9.9773425e-01,  2.2611160e+01],\n",
    "       [-9.0995320e-07,  2.3299665e-06,  1.0000000e+00]], dtype=float32), array([[ 1.0000000e+00,  2.8202797e-20,  8.9091027e-15],\n",
    "       [-5.6938062e-19,  1.0000000e+00,  6.4872178e-15],\n",
    "       [-1.2599002e-21,  1.3424955e-23,  1.0000000e+00]], dtype=float32), array([[ 9.9348122e-01, -1.8129036e-02,  1.1003100e+00],\n",
    "       [ 1.2132729e-02,  9.9091125e-01,  2.8742294e+01],\n",
    "       [-2.9285566e-06, -6.2816152e-06,  1.0000000e+00]], dtype=float32), array([[ 9.8954433e-01, -1.8810771e-02,  1.1892141e+01],\n",
    "       [ 1.3862181e-02,  9.8975760e-01,  2.0044521e+01],\n",
    "       [-4.0683108e-06, -4.2664565e-06,  1.0000000e+00]], dtype=float32), array([[ 9.8794365e-01, -1.2153405e-02,  1.8298512e+01],\n",
    "       [ 8.4343022e-03,  9.8829275e-01,  2.8633160e+01],\n",
    "       [-2.5430195e-06, -3.1491891e-06,  1.0000000e+00]], dtype=float32), array([[ 9.9505156e-01, -1.7393423e-02,  3.6898847e+00],\n",
    "       [ 1.3367594e-02,  9.9287426e-01,  1.8703876e+01],\n",
    "       [-1.3000300e-06, -4.7085673e-06,  1.0000000e+00]], dtype=float32), array([[ 9.9182397e-01, -1.2042225e-02,  1.2686370e+01],\n",
    "       [ 7.6934961e-03,  9.8897797e-01,  2.7970253e+01],\n",
    "       [-9.2655773e-07, -6.0620891e-06,  1.0000000e+00]], dtype=float32)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Align images and save each capture to a layered tiff file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7142fb6b14c4be980bee622e12b7cc3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "FloatProgress(value=0.0, description='Saving', layout=Layout(width='100%'), max=1.0)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import exiftool\n",
    "import datetime\n",
    "\n",
    "use_multi_process = True # set to False for single-process saving\n",
    "overwrite_existing = False # skip existing files, set to True to overwrite\n",
    "\n",
    "## This progress widget is used for display of the long-running process\n",
    "f2 = FloatProgress(min=0, max=1, layout=Layout(width='100%'), description=\"Saving\")\n",
    "display(f2)\n",
    "def update_f2(val):\n",
    "    f2.value=val\n",
    "\n",
    "if not os.path.exists(outputPath):\n",
    "    os.makedirs(outputPath)\n",
    "if generateThumbnails and not os.path.exists(thumbnailPath):\n",
    "    os.makedirs(thumbnailPath)\n",
    "    \n",
    "# Save out geojson data so we can open the image capture locations in our GIS\n",
    "with open(os.path.join(outputPath,'imageSet.json'),'w') as f:\n",
    "    f.write(str(geojson_data))\n",
    "    \n",
    "# If we didn't provide a panel above, irradiance set to None will cause DLS data to be used\n",
    "try:\n",
    "    irradiance = panel_irradiance+[0]\n",
    "except NameError:\n",
    "    irradiance = None\n",
    "\n",
    "start_time = datetime.datetime.now()\n",
    "\n",
    "# Save all captures in the imageset as aligned stacks\n",
    "imgset.save_stacks(warp_matrices,\n",
    "                     outputPath,\n",
    "                     thumbnailPath,\n",
    "                     irradiance = irradiance,\n",
    "                     multiprocess=use_multi_process, \n",
    "                     overwrite=overwrite_existing, \n",
    "                     progress_callback=update_f2)\n",
    "\n",
    "end_time = datetime.datetime.now()\n",
    "update_f2(1.0)\n",
    "\n",
    "print(\"Saving time: {}\".format(end_time-start_time))\n",
    "print(\"Alignment+Saving rate: {:.2f} captures per second\".format(float(len(imgset.captures))/float((end_time-start_time).total_seconds())))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract Metadata from Captures list and save to log.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def decdeg2dms(dd):\n",
    "   is_positive = dd >= 0\n",
    "   dd = abs(dd)\n",
    "   minutes,seconds = divmod(dd*3600,60)\n",
    "   degrees,minutes = divmod(minutes,60)\n",
    "   degrees = degrees if is_positive else -degrees\n",
    "   return (degrees,minutes,seconds)\n",
    "\n",
    "header = \"SourceFile,\\\n",
    "GPSDateStamp,GPSTimeStamp,\\\n",
    "GPSLatitude,GpsLatitudeRef,\\\n",
    "GPSLongitude,GPSLongitudeRef,\\\n",
    "GPSAltitude,GPSAltitudeRef,\\\n",
    "FocalLength,\\\n",
    "XResolution,YResolution,ResolutionUnits\\n\"\n",
    "\n",
    "lines = [header]\n",
    "for capture in imgset.captures:\n",
    "    #get lat,lon,alt,time\n",
    "    outputFilename = capture.uuid+'.tif'\n",
    "    fullOutputPath = os.path.join(outputPath, outputFilename)\n",
    "    lat,lon,alt = capture.location()\n",
    "    #write to csv in format:\n",
    "    # IMG_0199_1.tif,\"33 deg 32' 9.73\"\" N\",\"111 deg 51' 1.41\"\" W\",526 m Above Sea Level\n",
    "    latdeg, latmin, latsec = decdeg2dms(lat)\n",
    "    londeg, lonmin, lonsec = decdeg2dms(lon)\n",
    "    latdir = 'North'\n",
    "    if latdeg < 0:\n",
    "        latdeg = -latdeg\n",
    "        latdir = 'South'\n",
    "    londir = 'East'\n",
    "    if londeg < 0:\n",
    "        londeg = -londeg\n",
    "        londir = 'West'\n",
    "    resolution = capture.images[0].focal_plane_resolution_px_per_mm\n",
    "\n",
    "    linestr = '\"{}\",'.format(fullOutputPath)\n",
    "    linestr += capture.utc_time().strftime(\"%Y:%m:%d,%H:%M:%S,\")\n",
    "    linestr += '\"{:d} deg {:d}\\' {:.2f}\"\" {}\",{},'.format(int(latdeg),int(latmin),latsec,latdir[0],latdir)\n",
    "    linestr += '\"{:d} deg {:d}\\' {:.2f}\"\" {}\",{},{:.1f} m Above Sea Level,Above Sea Level,'.format(int(londeg),int(lonmin),lonsec,londir[0],londir,alt)\n",
    "    linestr += '{}'.format(capture.images[0].focal_length)\n",
    "    linestr += '{},{},mm'.format(resolution,resolution)\n",
    "    linestr += '\\n' # when writing in text mode, the write command will convert to os.linesep\n",
    "    lines.append(linestr)\n",
    "\n",
    "fullCsvPath = os.path.join(outputPath,'log.csv')\n",
    "with open(fullCsvPath, 'w') as csvfile: #create CSV\n",
    "    csvfile.writelines(lines)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use Exiftool from the command line to write metadata to images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "\n",
    "if os.environ.get('exiftoolpath') is not None:\n",
    "    exiftool_cmd = os.path.normpath(os.environ.get('exiftoolpath'))\n",
    "else:\n",
    "    exiftool_cmd = 'exiftool'\n",
    "        \n",
    "cmd = '{} -csv=\"{}\" -overwrite_original {}'.format(exiftool_cmd, fullCsvPath, outputPath)\n",
    "print(cmd)\n",
    "if(subprocess.check_call(cmd) == 0):\n",
    "    print(\"Successfully updated stack metadata\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}