Pipelines/RNAseq/checkProgress.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "#CWD='/cellar/users/btsui/Project/METAMAP/notebook/RapMapTest/Pipelines/RNAseq/'\n",
    "import param as param\n",
    "import pandas as pdh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get time stamp for each dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 12 ms, sys: 24 ms, total: 36 ms\n",
      "Wall time: 6min 23s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time os.system('ls -laht --full-time {}>.progress.tsv'.format(param.count_out_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "progressDf=pd.read_csv('.progress.tsv',sep='\\s+',skiprows=3,header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "progressDf.columns=['permission','','','','size','date_str','time','','fname']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "m=progressDf['fname'].str.contains('abundance.tsv.gz')\n",
    "progressDf_sub=progressDf[m]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!ls -laht /nrnb/users/btsui/Data/all_seq/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(426702, 9)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#425232, 426702\n",
    "progressDf_sub.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/cellar/users/btsui/anaconda3/envs/deep_nlp_cpu/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "progressDf_sub['date_time']=pd.to_datetime(progressDf_sub['date_str'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### data cumulative distribution over time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZsAAAEsCAYAAAAVVx41AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3X+UXWV97/H3hwQwgpDwo9M0SQ0t0TaaGmUK6aJdd4QKgVqDLVoomqCpsRWuts2tBNslVMTiahFLr9KVlpRg1RC1lgjRNBc41t5efiSKhJ8yxVCS8qMmATpY0KHf+8d+xm6GM2fOzOSZvWfzea111uz93c/e+/lmTs539t7P2VsRgZmZWU4HVN0BMzNrPhcbMzPLzsXGzMyyc7ExM7PsXGzMzCw7FxszM8vOxcbMzLJzsTEzs+xcbMzMLLvpVXegLo466qiYP3/+qO2eeeYZDjnkkPwdyqwJeTiHenAO9VBVDtu3b/9eRBw9WjsXm2T+/Pls27Zt1HatVou+vr78HcqsCXk4h3pwDvVQVQ6SHu6mnU+jmZlZdi42ZmaWnYuNmZll52JjZmbZudiYmVl2LjZmZpadi42ZmWXnYmNmZtm52JiZWXa+g4CZWQ3MX3PjhNZfvWiQc8e5jZ2X/cqE9t0NH9mYmVl2LjZmZpadi42ZmWXnYmNmZtm52JiZWXYuNmZmlp2LjZmZZediY2Zm2WUvNpKmSfqWpBvS/DGSbpPUL+k6SQel+MFpvj8tn1/axoUp/oCkU0vxpSnWL2lNKd52H2ZmVo3JOLL5AHBfaf7jwBURcSywD1iZ4iuBfSl+RWqHpIXAWcBrgKXAp1MBmwZ8CjgNWAicndp22oeZmVUga7GRNBf4FeCv07yAk4AvpibrgTPS9LI0T1p+cmq/DNgQEc9FxHeBfuD49OqPiIci4gfABmDZKPswM7MK5D6y+STwQeC/0vyRwJMRMZjmdwFz0vQc4BGAtPyp1P5H8WHrjBTvtA8zM6tAthtxSnoz8EREbJfUl2s/EyFpFbAKoKenh1arNeo6AwMDXbWruybk4RzqwTnsH6sXDY7eqIOeGePfxmTknvOuzycCb5F0OvAy4DDgz4GZkqanI4+5wO7UfjcwD9glaTpwOLCnFB9SXqddfE+HfbxARKwF1gL09vZGX1/fqEm1Wi26aVd3TcjDOdSDc9g/xnvH5iGrFw1y+Y7xfaTvPKdvQvvuRrbTaBFxYUTMjYj5FBf4b46Ic4BbgDNTsxXA9Wl6U5onLb85IiLFz0qj1Y4BFgC3A3cAC9LIs4PSPjaldUbah5mZVaCK79lcAPy+pH6K6ytXp/jVwJEp/vvAGoCIuAfYCNwLfA04LyKeT0ct5wNbKEa7bUxtO+3DzMwqMCkPT4uIFtBK0w9RjCQb3uZZ4G0jrH8pcGmb+GZgc5t4232YmVk1fAcBMzPLzsXGzMyyc7ExM7PsXGzMzCw7FxszM8vOxcbMzLJzsTEzs+xcbMzMLDsXGzMzy87FxszMsnOxMTOz7FxszMwsOxcbMzPLzsXGzMyyc7ExM7PsshUbSS+TdLukb0u6R9Ifp/g1kr4r6c70WpziknSlpH5Jd0l6Q2lbKyQ9mF4rSvHjJO1I61wpSSl+hKStqf1WSbNy5WlmZqPLeWTzHHBSRLwOWAwslbQkLfuDiFicXnem2GkUj3xeAKwCroKicAAXASdQPBDtolLxuAp4T2m9pSm+BrgpIhYAN6V5MzOrSLZiE4WBNHtgekWHVZYB16b1bgVmSpoNnApsjYi9EbEP2EpRuGYDh0XErRERwLXAGaVtrU/T60txMzOrQNZrNpKmSboTeIKiYNyWFl2aTpVdIengFJsDPFJafVeKdYrvahMH6ImIR9P0Y0DP/srJzMzGbnrOjUfE88BiSTOBL0t6LXAhRQE4CFgLXAB8JGMfQlLbIypJqyhO2dHT00Or1Rp1ewMDA121q7sm5OEc6sE57B+rFw1OaP2eGePfxmTknrXYDImIJyXdAiyNiD9L4eck/Q3wv9L8bmBeabW5KbYb6BsWb6X43DbtAR6XNDsiHk2n254YoV9rKQoevb290dfX167ZC7RaLbppV3dNyMM51INz2D/OXXPjhNZfvWiQy3eM7yN95zl9E9p3N3KORjs6HdEgaQbwJuD+9OFPGjl2BnB3WmUTsDyNSlsCPJVOhW0BTpE0Kw0MOAXYkpY9LWlJ2tZy4PrStoZGra0oxc3MrAI5j2xmA+slTaMoahsj4gZJN0s6GhBwJ/Dbqf1m4HSgH/g+8C6AiNgr6RLgjtTuIxGxN02/D7gGmAF8Nb0ALgM2SloJPAy8PVuWZmY2qmzFJiLuAl7fJn7SCO0DOG+EZeuAdW3i24DXtonvAU4eY5fNzCwT30HAzMyyc7ExM7PsXGzMzCw7FxszM8vOxcbMzLJzsTEzs+xcbMzMLDsXGzMzy87FxszMsnOxMTOz7FxszMwsOxcbMzPLzsXGzMyyc7ExM7PsXGzMzCw7FxszM8su52OhXybpdknflnSPpD9O8WMk3SapX9J1kg5K8YPTfH9aPr+0rQtT/AFJp5biS1OsX9KaUrztPszMrBo5j2yeA06KiNcBi4GlkpYAHweuiIhjgX3AytR+JbAvxa9I7ZC0EDgLeA2wFPi0pGnpcdOfAk4DFgJnp7Z02IeZmVUgW7GJwkCaPTC9AjgJ+GKKrwfOSNPL0jxp+cmSlOIbIuK5iPgu0A8cn179EfFQRPwA2AAsS+uMtA8zM6vA9JwbT0cf24FjKY5C/gV4MiIGU5NdwJw0PQd4BCAiBiU9BRyZ4reWNlte55Fh8RPSOiPtY3j/VgGrAHp6emi1WqPmNDAw0FW7umtCHs6hHpzD/rF60eDojTromTH+bUxG7lmLTUQ8DyyWNBP4MvAzOfc3VhGxFlgL0NvbG319faOu02q16KZd3TUhD+dQD85h/zh3zY0TWn/1okEu3zG+j/Sd5/RNaN/dmJTRaBHxJHAL8AvATElD/yJzgd1pejcwDyAtPxzYU44PW2ek+J4O+zAzswrkHI12dDqiQdIM4E3AfRRF58zUbAVwfZrelOZJy2+OiEjxs9JotWOABcDtwB3AgjTy7CCKQQSb0joj7cPMzCqQ8zTabGB9um5zALAxIm6QdC+wQdJHgW8BV6f2VwOfkdQP7KUoHkTEPZI2AvcCg8B56fQcks4HtgDTgHURcU/a1gUj7MPMzCqg4kDAent7Y9u2baO2a3dud/4Ez7VWYSLnd+vCOdSDc6iHCV2zuexXxr1fSdsjone0dr6DgJmZZediY2Zm2bnYmJlZdi42ZmaWnYuNmZll52JjZmbZudiYmVl2LjZmZpadi42ZmWXnYmNmZtmNem8DSQuAP6F4GubLhuIR8VMZ+2VmZg3SzZHN3wBXUdwE843AtcDf5uyUmZk1SzfFZkZE3ERx086HI+JiYPx3bTMzs5ecbm4R+pykA4AH0y39dwOH5u2WmZk1STdHNh8AXg68HzgOeAewPGenzMysWbopNvMjYiAidkXEuyLi14GfHG0lSfMk3SLpXkn3SPpAil8sabekO9Pr9NI6F0rql/SApFNL8aUp1i9pTSl+jKTbUvy69MRO0lM9r0vx2yTN7/6fxMzM9rduis2FXcaGGwRWR8RCYAlwnqSFadkVEbE4vTYDpGVnAa8BlgKfljQtPenzU8BpFCPizi5t5+NpW8cC+4CVKb4S2JfiV6R2ZmZWkRGv2Ug6DTgdmCPpytKiwygKSUcR8SjwaJr+D0n3AXM6rLIM2BARzwHfTY+HPj4t64+Ih1K/NgDL0vZOAn4ztVkPXEwxcm5Zmgb4IvC/JSn8WFIzs0p0OrL5N2Ab8CywvfTaBJzaYb0XSaexXg/clkLnS7pL0jpJs1JsDvBIabVdKTZS/EjgyYgYHBZ/wbbS8qdSezMzq8CIRzYR8W3g25I+FxE/HO8OJB0KfAn43Yh4WtJVwCVApJ+XA+8e7/YnQtIqYBVAT08PrVZr1HUGBgZe1G71olEP9GqnZ8bU7HeZc6gH51APE8mhm8++iepm6PN8SeO6g4CkAykKzWcj4u/Seo+Xlv8VcEOa3Q3MK60+N8UYIb4HmClpejp6Kbcf2tYuSdOBw1P7F4iItcBagN7e3ujr6xstJVqtFsPbnbvmxlHXq5vViwa5fEc3v/76cg714BzqYSI57Dynb/92po1sdxCQJOBq4L6I+EQpPrvU7K3A3Wl6E3BWGkl2DLAAuB24A1iQRp4dRDGIYFO6/nILcGZafwVwfWlbK9L0mcDNvl5jZladbsrgjIi4KV1gfxi4WNJ24MOjrHci8E5gh6Q7U+xDFKPJFlOcRtsJvBcgIu6RtBG4l6KwnRcRzwOkL5NuAaYB6yLinrS9C4ANkj4KfIuiuJF+fiYNMthLUaDMzKwi2e4gEBH/BKjNos0d1rkUuLRNfHO79dIItePbxJ8F3jZaH83MbHKM5w4C7+S/T1GZmZmNatQjm4i4I00OAO/K2x0zM2uiTl/q/ArFdZW2IuItWXpkZmaN0+nI5s/Sz18Dfpz/HoF2NvB42zXMzMza6PSlzq8DSLo8InpLi74iaVv2npmZWWN0M0DgEEk/+gJn+g7MIfm6ZGZmTdPN0OffA1qSHqIYyvxK0i1ezMzMutHNaLSvSVoA/EwK3Z/uzGxmZtaVrm6kk4rLtzP3xczMGqqbazZmZmYTMmKxkXRi+nnw5HXHzMyaqNORzdDTOf/fZHTEzMyaq9M1mx9KWsuLHwsNQES8P1+3zMysSToVmzcDv0zxCOjtk9MdMzNrok53EPgexbNi7kuPiDYzMxuXbkaj7ZH0ZUlPpNeXJM0dbSVJ8yTdIuleSfdI+kCKHyFpq6QH089ZKS5JV0rql3SXpDeUtrUitX9Q0opS/DhJO9I6V6ang464DzMzq0a3j4XeBPxEen0lxUYzCKyOiIXAEuA8SQuBNcBNEbEAuCnNA5xG8SjoBRR3KLgKisIBXAScQPGgtItKxeMq4D2l9Zam+Ej7MDOzCnRTbH4sIv4mIgbT6xrg6NFWiohHI+Kbafo/gPuAOcAyYH1qth44I00vA66Nwq3ATEmzKa4ZbY2IvRGxD9gKLE3LDouIWyMigGuHbavdPszMrALdFJvvSXqHpGnp9Q5gz1h2Imk+8HrgNqAnIh5Nix4DetL0HOCR0mq7UqxTfFebOB32YWZmFejmdjXvBv4CuILiYWr/zBie2CnpUOBLwO9GxNPpsgoAERGSRnxA2/7QaR+SVpFuKtrT00Or1Rp1ewMDAy9qt3rR4IT7Odl6ZkzNfpc5h3pwDvUwkRy6+eybqG5uxPkwMK6ncko6kKLQfDYi/i6FH5c0OyIeTafCnkjx3cC80upzU2w30Dcs3krxuW3ad9rH8NzWAmsBent7o6+vr12zF2i1Wgxvd+6aG0ddr25WLxrk8h1d3RqvtpxDPTiHephIDjvP6du/nWkj273R0siwq4H7IuITpUWbgKERZSuA60vx5WlU2hLgqXQqbAtwiqRZaWDAKcCWtOxpSUvSvpYP21a7fZiZWQVylvITgXcCOyTdmWIfAi4DNkpaCTwMvD0t2wycDvQD3yedqouIvZIuAe5I7T4SEXvT9PuAa4AZwFfTiw77MDOzCmQrNhHxTxQPW2vn5DbtAzhvhG2tA9a1iW8DXtsmvqfdPszMrBqjnkaT9Eelad8B2szMxqzTIwYukPQLwJmlsO8AbWZmY9bpNNr9wNuAn5L0jTR/pKRXR8QDk9I7MzNrhE6n0Z6kuKDfTzH0+M9TfI2kf87cLzMza5BORzanAh8Gfhr4BHAX8ExEdP2FTjMzM+hwZBMRH4qIk4GdwGeAacDRkv5J0lcmqX9mZtYA3Qx93pKGGG+T9DsR8YuSjsrdMTMza45Rhz5HxAdLs+em2PdydcjMzJpnTLer8RM7zcxsPLLdG83MzGyIi42ZmWXnYmNmZtm52JiZWXYuNmZmlp2LjZmZZZfzSZ3rJD0h6e5S7GJJuyXdmV6nl5ZdKKlf0gOSTi3Fl6ZYv6Q1pfgxkm5L8eskHZTiB6f5/rR8fq4czcysOzmPbK4BlraJXxERi9NrM4CkhcBZwGvSOp+WNE3SNOBTwGnAQuDs1Bbg42lbxwL7gJUpvhLYl+JXpHZmZlahbMUmIv4R2Dtqw8IyYENEPBcR36W40/Tx6dUfEQ9FxA+ADcAySQJOAr6Y1l8PnFHa1vo0/UXg5NTezMwqUsU1m/Ml3ZVOs81KsTnAI6U2u1JspPiRwJMRMTgs/oJtpeVPpfZmZlaRbm7EuT9dBVwCRPp5OfDuSe7Dj0haBawC6OnpodVqjbrOwMDAi9qtXjTYvnGN9cyYmv0ucw714BzqYSI5dPPZN1GTWmwi4vGhaUl/BdyQZncD80pN56YYI8T3ADMlTU9HL+X2Q9vaJWk6cHhq364/a4G1AL29vdHX1zdqDq1Wi+Htzl1z46jr1c3qRYNcvmOy/9bYv5xDPTiHephIDjvP6du/nWljUk+jSZpdmn0rMDRSbRNwVhpJdgywALgduANYkEaeHUQxiGBTRARwC3BmWn8FcH1pWyvS9JnAzam9mZlVJFspl/R5isdJHyVpF3AR0CdpMcVptJ3AewEi4h5JG4F7gUHgvIh4Pm3nfGALxcPb1kXEPWkXFwAbJH0U+BZwdYpfDXxGUj/FAIWzcuVoZmbdyVZsIuLsNuGr28SG2l8KXNomvhnY3Cb+EMVoteHxZ4G3jamzZmaWle8gYGZm2bnYmJlZdi42ZmaWnYuNmZll52JjZmbZudiYmVl2LjZmZpadi42ZmWXnYmNmZtm52JiZWXYuNmZmlp2LjZmZZediY2Zm2bnYmJlZdi42ZmaWnYuNmZlll63YSFon6QlJd5diR0jaKunB9HNWikvSlZL6Jd0l6Q2ldVak9g9KWlGKHydpR1rnSknqtA8zM6tOziOba4Clw2JrgJsiYgFwU5oHOA1YkF6rgKugKBwUj5M+geKpnBeVisdVwHtK6y0dZR9mZlaRbMUmIv4R2DssvAxYn6bXA2eU4tdG4VZgpqTZwKnA1ojYGxH7gK3A0rTssIi4NSICuHbYttrtw8zMKjJ9kvfXExGPpunHgJ40PQd4pNRuV4p1iu9qE++0jxeRtIriSIqenh5ardaoCQwMDLyo3epFg6OuVzc9M6Zmv8ucQz04h3qYSA7dfPZN1GQXmx+JiJAUVe4jItYCawF6e3ujr69v1G22Wi2Gtzt3zY0T6mcVVi8a5PIdlf369wvnUA/OoR4mksPOc/r2b2famOzRaI+nU2Ckn0+k+G5gXqnd3BTrFJ/bJt5pH2ZmVpHJLjabgKERZSuA60vx5WlU2hLgqXQqbAtwiqRZaWDAKcCWtOxpSUvSKLTlw7bVbh9mZlaRbMeNkj4P9AFHSdpFMarsMmCjpJXAw8DbU/PNwOlAP/B94F0AEbFX0iXAHandRyJiaNDB+yhGvM0AvppedNiHmZlVJFuxiYizR1h0cpu2AZw3wnbWAevaxLcBr20T39NuH2ZmVh3fQcDMzLJzsTEzs+xcbMzMLDsXGzMzy87FxszMsnOxMTOz7FxszMwsOxcbMzPLzsXGzMyyc7ExM7PsXGzMzCw7FxszM8vOxcbMzLJzsTEzs+xcbMzMLLtKio2knZJ2SLpT0rYUO0LSVkkPpp+zUlySrpTUL+kuSW8obWdFav+gpBWl+HFp+/1pXU1+lmZmNqTKI5s3RsTiiOhN82uAmyJiAXBTmgc4DViQXquAq6AoThRP/zwBOB64aKhApTbvKa23NH86ZmY2kjqdRlsGrE/T64EzSvFro3ArMFPSbOBUYGtE7I2IfcBWYGladlhE3JqeAHptaVtmZlaBqopNAP8gabukVSnWExGPpunHgJ40PQd4pLTurhTrFN/VJm5mZhWZXtF+fzEidkv6MWCrpPvLCyMiJEXuTqRCtwqgp6eHVqs16joDAwMvard60WCG3uXVM2Nq9rvMOdSDc6iHieTQzWffRFVSbCJid/r5hKQvU1xzeVzS7Ih4NJ0KeyI13w3MK60+N8V2A33D4q0Un9umfbt+rAXWAvT29kZfX1+7Zi/QarUY3u7cNTeOul7drF40yOU7qvpbY/9wDvXgHOphIjnsPKdv/3amjUk/jSbpEEmvGJoGTgHuBjYBQyPKVgDXp+lNwPI0Km0J8FQ63bYFOEXSrDQw4BRgS1r2tKQlaRTa8tK2zMysAlWU8h7gy2k08nTgcxHxNUl3ABslrQQeBt6e2m8GTgf6ge8D7wKIiL2SLgHuSO0+EhF70/T7gGuAGcBX08vMzCoy6cUmIh4CXtcmvgc4uU08gPNG2NY6YF2b+DbgtRPurJmZ7Rd1GvpsZmYN5WJjZmbZudiYmVl2LjZmZpadi42ZmWXnYmNmZtm52JiZWXYuNmZmlp2LjZmZZediY2Zm2bnYmJlZdi42ZmaWnYuNmZll52JjZmbZudiYmVl2jS02kpZKekBSv6Q1VffHzOylrJHFRtI04FPAacBC4GxJC6vtlZnZS1cjiw1wPNAfEQ9FxA+ADcCyivtkZvaS1dRiMwd4pDS/K8XMzKwCioiq+7DfSToTWBoRv5Xm3wmcEBHnD2u3CliVZl8NPNDF5o8Cvrcfu1uVJuThHOrBOdRDVTm8MiKOHq3R9MnoSQV2A/NK83NT7AUiYi2wdiwblrQtInon1r3qNSEP51APzqEe6p5DU0+j3QEskHSMpIOAs4BNFffJzOwlq5FHNhExKOl8YAswDVgXEfdU3C0zs5esRhYbgIjYDGzOsOkxnXarsSbk4RzqwTnUQ61zaOQAATMzq5emXrMxM7MacbExM7PsXGzMzCw7Fxsze8mT9Iaq+zBRkn6s6j504mLTgaTDJV0m6X5JeyXtkXRfis2sun8TJemrVfehW5LmSdog6RuSPiTpwNKyv6+yb91qSA4/LukqSZ+SdKSkiyXtkLRR0uyq+9cNSW8Y9joO2CTp9VOl6Eg6YtjrSOB2SbMkHVF1/9pp7NDn/WQjcDPQFxGPQfGfDViRlp1SYd+60uE/j4DFk9mXCVoHfAm4FVgJfF3Sr0bEHuCVlfase03I4RrgRuAQ4Bbgs8DpwBnAXzI1bni7jeJ38FwpdiTwCSCAk6ro1Bh9D3h4WGwO8E2KHH5q0ns0Cg997kDSAxHx6rEuqxNJzwNfpyguwy2JiBmT3KVxkXRnRCwuzb8DuBB4C/CFiKj9X6QNyeFbEfH6NP2vEfGTpWUvyK+uJP068H7gsoj4aop9NyKOqbZn3ZO0GngT8AcRsSPFap2Dj2w6e1jSB4H1EfE4gKQe4FxeeFfpOrsPeG9EPDh8gaSpkgPAgZJeFhHPAkTE30p6jOIuEYdU27WuNSGH8qn3azssq62I+JKkLcAlkt4NrKY4GpgyIuJySdcBV6T/xxdR8xymxJujQr9BcXj99XTNZi/QAo4A3l5lx8bgYkb+Pf/PSezHRP01cEI5EBH/B3gbcHclPRq7JuRwvaRDASLij4aCko4FvlNZr8YoIgYi4veAjwHrgUMr7tKYRcSuiHgbxWfSVuDl1faoM59GM7OXNEkCXhERT1fdl/GSNAP46Yio7R8tPrIZJ0nvqroP3ZJ0ahpBtCm9rpK0tOp+jVXKY6Wk+cPi766mR/uPpA9X3YduNeH9VH4vReHpFJ+S76WI+M+hQlPXzyYf2YzT8IujdSXpk8CrKM6v70rhucBy4MGI+EBVfRsLSX8CnEgx2uZXgU9GxF+kZd+cChfXO/H7afJI+hjwi/i9NKlcbDqQdNdIi4BXRcTBk9mf8ZD0nYh4VZu4gO9ExIIKujVmknYAr0+Pj5gJfA54ICJ+rzxCqs4kjXSaRsCMiKj9gJ0mvJ8a8l6acp9NtX9zV6wHOBXYNywu4J8nvzvj8qykn4+IO4bFfx54tooOjdP0iBgEiIgnJf0qsFbSF4CDqu1a154Efn5oZGPZFBoZ2IT3UxPeS1Pus8nFprMbgEMj4s7hCyS1Jr8743IucJWkV/Dfpz3mAU+lZVPFv0j6HxHxdYCIeB5YKemjwK9X27WuXUvx5c0XFRuKv66ngnOZ+u+nJryXptxnk0+jvUSkOx/MSbO7h+6IMFWk0TZExH+2WTYnInZPfq9euqby+8nvpWp4NNoYSVpVdR/GIyIei4jtEbEd+O2q+zNWabTNCz4cJF2clk3ZD4ehHKaaqfx+avB7qdafTS42Yzel/mON4C1Vd2A/aUIezqEempBDrT+bXGzGrt09xqaaJuQAzcjDOdSDc8jM12zGSNLciNg1esv6knRARPxX1f2YqCbkIUkxxf8TNuT30IQcav3Z5CObsRt+88Fak3TUsPl3AJ+UtCp9N2JKaEIekt6q9KwRSUdLuha4S9J1kuZW3L2uSPqEpBPLsan2Id2QHI6Q9GFJv6XCHwJ/KelPJc2qun/t+MimgzZfnBLFt6cfAIiIn5v0To1R+RvRkv4I+CWKYbZvBnalmxHWXhPykHRvRCxM09dRPFPlC8AvA+dExJuq7F83JP07xXNUjgauAz4fEd+qtldj05AcNgM7gMOAn03TGykeO/C6iKjdc4X8PZvOdgJPAx8F/pOi2HyD4hYXU0X5r/5fA34pIp6R9DmK23VMFU3IY1pp+tiI+I00fY2k362iQ+OwKyJ6Jb2K4q7ofytpGvB5ig/tqXDn5ybk8BMRcXo6qt8VEX0p/g1JL/ruTR34NFoHEfEWiicrrqX4a2En8MOIeDgihj8lr65mqHjc7XHAtIh4BiAifgg8X23XxqQJebQkfSR9z6Ml6a0Akt5I8aXIqSAAIuI7EXFJRLyG4nEbLwM2V9qz7jUhhwPS6bJ5wKFKN6dV8XjoWt4FwafRuiDpEOAS4KeB4yJiSpxfB5B0y7DQb0bEo+lNuSUieqvo11g1IQ9JBwJ/CAzdWXgu8AzwFWBNRPxrVX3r1lS5d1gnDcnhbOCTafZ9wO+k6Z8F/jgi1lbSsQ5cbMZA0uuAX4iIv6y6LxOVThscHBHfr7ovEzFV85B0OMU9uvZU3ZexkHRoRAxU3Y+JaEIO8KP3vtINRacDiynu5vBoxV1ry9dsRpE+FJZSujVrIWJCAAAEPklEQVSHpJkR8WSF3RqTdjlQHA1MmRygGXkMz0HSlMohIgam+u+hCTkkhwJLJZVz6K+wPx35mk0HkpZTXHzuo3jk6suBNwLb07Laa0IO0Iw8nEM9OIdq+DRaB5IeAE4Y/tdOujB3W7vnetRNE3KAZuThHOrBOVTDRzadiTRyZZj/oua3hihpQg7QjDycQz04hwr4mk1nlwLflPQPwNDDrX6S4otTl1TWq7FpQg7QjDycQz04hwr4NNoo0mHpqbz4QuLwJ+TVVhNygGbk4RzqwTlMPhebMZL05oi4oep+TEQTcoBm5OEc6sE55OdiM0ble3RNVU3IAZqRh3OoB+eQnwcIjF0tL76NURNygGbk4RzqwTlk5mIzdu+tugP7QRNygGbk4RzqwTlk5mLTgaSDJC2X9Mtp/jeB5ZLOS/e5qr0m5ADNyMM51INzqIav2XQg6bMUw8NfDjxJcXuIvwNOpvi3W1Fh97rShBygGXk4h3pwDtVwselA0l0R8XPpJne7KZ4h8bwkAd+eIg9Pm/I5QDPycA714Byq4dNonR0g6SDgFRR/QRye4gcDtTxUbaMJOUAz8nAO9eAcKuA7CHR2NXA/xRMW/xD4gqSHgCXAhio7NgZNyAGakYdzqAfnUAGfRhuFpJ8AiIh/kzST4nnx/xoRt1fbs+41IQdoRh7OoR6cw+RzsRknNeABTE3IAZqRh3OoB+eQj6/ZjN+9VXdgP2hCDtCMPJxDPTiHTHzNpgNJvz/SIoqhhrXXhBygGXk4h3pwDtXwkU1nHwNmUYz4KL8OZer82zUhB2hGHs6hHpxDBXxk09k3gb+PiO3DF0j6rQr6Mx5NyAGakYdzqAfnUAEPEOhA0quBvRHx722W9UTE4xV0a0yakAM0Iw/nUA/OoRouNmZmll0tz+3VhaTDJV0m6X5JeyXtkXRfis2sun/daEIO0Iw8nEM9OIdquNh0thHYB/RFxBERcSTwxhTbWGnPuteEHKAZeTiHenAOFfBptA4kPRARrx7rsjppQg7QjDycQz04h2r4yKazhyV9UFLPUEBSj6QLgEcq7NdYNCEHaEYezqEenEMFXGw6+w3gSODrkvZJ2gu0gCOAt1fZsTFoQg7QjDycQz04hwr4NNooJP0MMBe4tXy/IUlLI+Jr1fWse03IAZqRh3OoB+cw+Xxk04Gk9wPXA+cDd0taVlr8sWp6NTZNyAGakYdzqAfnUA3fQaCz9wDHRcSApPnAFyXNj4g/p7gH0VTQhBygGXk4h3pwDhVwsensgKHD04jYKamP4pf6Smr6C22jCTlAM/JwDvXgHCrg02idPS5p8dBM+uW+GTgKWFRZr8amCTlAM/JwDvXgHCrgAQIdSJoLDEbEY22WnRgR/7eCbo1JE3KAZuThHOrBOVTDxcbMzLLzaTQzM8vOxcbMzLJzsTEzs+xcbMzMLDsXGzMzy+7/A8yXP01mP9goAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax=progressDf_sub['date_time'].hist(cumulative=True)\n",
    "ax.set_ylabel('# of data')\n",
    "#plt.xticks(x, labels, rotation='vertical')\n",
    "plt.setp( ax.xaxis.get_majorticklabels(), rotation=90 )\n",
    "\n",
    "ax.figure.savefig('../../Analysis/Figures/DataAvail/{}.pdf'.format(param.pipelineName))\n",
    "ax.figure.savefig('../../Analysis/Figures/DataAvail/{}.pdf'.format(param.pipelineName))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.datetime64('2005-02-25')"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.datetime64('2005-02-25')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "numpy.datetime64('2018-04-01')"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#np.datetime64('2018-04-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAERCAYAAAB4jRxOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAGNFJREFUeJzt3X+YJHVh5/H3hx+aldE1Co5mUVcvBiXZiO6IetwlMxqVH4a7y0mEExHO3OZMFKJ7F9e7nD+eO+OaJxtDnqgnUcEfkYmH4YCQoD7K4HkJyCxZWfkVFJcLm7CICDhI1JXP/VE1MDPb3dM72zVVZX1ez9PPznR3dX22uuczPd/+VpVsExERP/4OqjtARESsjhR+RERHpPAjIjoihR8R0REp/IiIjkjhR0R0RAo/IqIjUvgRER2Rwo+I6IhD6g6w0OGHH+7169f3vO2BBx7gsMMOW91AK5Cco9WWnNCerMk5WnXn3L59+922jxjqzrYbc9m4caP7ufLKK/ve1iTJOVptyWm3J2tyjlbdOYFZD9mxGdKJiOiIFH5EREek8CMiOiKFHxHRESn8iIiOqKzwJR0laceCy/2Sfquq9UVExGCVzcO3fQtwDICkg4HdwMVVrS8iIgZbrSGdlwLfsH37Kq0vIiKWkFfhnLaSPgpcZ/uPe9y2CdgEMD4+vnF6errnY8zNzTE2NlZpzlFIztFqS05oT9bk7G/n7vv2e5nxNbDnwQNb74Z1a1e87NTU1HbbE8Pct/LCl/Qo4B+An7W9Z9B9JyYmPDs72/O2mZkZJicnRx9wxJJztNqSE9qTNTn7W7/l8v1eZvOGvWzbeWCj47u2nrTiZSUNXfirMaRzAsW7+4FlHxER1VqNwj8NuHAV1hMREQNUWviSDgNeBvx5leuJiIjlVXp4ZNsPAE+sch0RETGc7GkbEdERKfyIiI5I4UdEdEQKPyKiI1L4EREdkcKPiOiIFH5EREek8CMiOiKFHxHRESn8iIiOSOFHRHRECj8ioiNS+BERHZHCj4joiBR+RERHpPAjIjoihR8R0REp/IiIjkjhR0R0RAo/IqIjUvgRER1RaeFLerykiyTdLOkmSS+ucn0REdHfIRU//rnAFbZfJelRwGMqXl9ERPRRWeFLWgv8AnAmgO0fAD+oan0RETGYbFfzwNIxwHnAjcBzge3AObYfWHK/TcAmgPHx8Y3T09M9H29ubo6xsbFKso5Sco5WW3JCe7IOm3Pn7vtWIU1/42tgz4O1RhjKKHJuWLd2xctOTU1ttz0xzH2rLPwJ4GrgONvXSDoXuN/2f+u3zMTEhGdnZ3veNjMzw+TkZCVZRyk5R6stOaE9WYfNuX7L5dWHGWDzhr1s21n1qPOBG0XOXVtPWvGykoYu/Co/tL0DuMP2NeX3FwHPr3B9ERExQGWFb/tO4O8lHVVe9VKK4Z2IiKhB1X8vvQn403KGzm3AWRWvLyIi+qi08G3vAIYaW4qIiGplT9uIiI5I4UdEdEQKPyKiI1L4EREdkcKPiOiIFH5EREek8CMiOiKFHxHRESn8iIiOSOFHRHRECj8ioiNS+BERHZHCj4joiBR+RERHpPAjIjoihR8R0REp/IiIjkjhR0R0RAo/IqIjUvgRER1R6UnMJe0Cvgv8CNhrOyc0j4ioSaWFX5qyffcqrCciIgbIkE5EREfIdnUPLn0T+A5g4EO2z+txn03AJoDx8fGN09PTPR9rbm6OsbGxyrKOymrm3Ln7vhUvO74G9jw4wjAVaUtOaE/W5BytUeTcsG7tipedmpraPuxwedWFv872bklPAj4PvMn2l/rdf2JiwrOzsz1vm5mZYXJyspqgI7SaOddvuXzFy27esJdtO1djRO/AtCUntCdrco7WKHLu2nrSipeVNHThVzqkY3t3+e9dwMXAsVWuLyIi+qus8CUdJumx818DLwe+VtX6IiJisCr/XhoHLpY0v55P2b6iwvVFRMQAlRW+7duA51b1+BERsX8yLTMioiNS+BERHZHCj4joiBR+RERHpPAjIjoihR8R0REp/IiIjkjhR0R0RAo/IqIjUvgRER2Rwo+I6IgUfkRER6TwIyI6YtmjZUp6FvAe4GjgJ+avt/3MCnNFRMSIDfMO/3zgg8BeYAr4OPDJKkNFRMToDVP4a2x/geL8t7fbfiew8hMwRkRELYY5Acr3JR0E3CrpjcBuYKzaWBERMWrDvMM/B3gMcDawETgdOKPKUBERMXrDFP5623O277B9lu1/Czyt6mARETFawxT+24a8LiIiGqzvGL6kE4ATgXWS/mjBTY+jmLETEREtMuhD238AZoGTge0Lrv8u8OZhVyDp4PJxdtt+5UpCRkTEgetb+La/CnxV0qds//AA1nEOcBPFXwYREVGToT60lXSRpBsl3TZ/GebBJR1JMWf/wweUMiIiDphsD76D9GXgHcD7gF8GzgIOsv32ZR9cuojisAyPBf5TryEdSZuATQDj4+Mbp6enez7W3NwcY2PNn/6/mjl37r5vxcuOr4E9D44wTEXakhPakzU5R2sUOTesW7viZaemprbbnhjmvsPseLXG9hckyfbtwDslbQcGFr6kVwJ32d4uabLf/WyfB5wHMDEx4cnJ3nedmZmh321Nspo5z9xy+YqX3bxhL9t2DvP016stOaE9WZNztEaRc9drJkcTZhlV7ml7HHCypBMpDrr2OEmftH36yuNGRMRKrWRP29cCr1tuIdtvs32k7fXAqcAXU/YREfVZ9h2+7WvLL+coxu8jIqKFBu14dRnQ9xNd2ycPuxLbM8DM/gSLiIjRGvQO//fLf38FeDKPHAP/NGBPlaEiImL0Bu14dRWApG1LpvxcJmm28mQRETFSw3xoe5ikh09nKOkZwGHVRYqIiCoMMy3zzcBMuXetgKdT7igVERHtMcwsnSvKE5k/u7zqZtvfrzZWRESM2lC7h5UF/9WKs0RERIWGGcOPiIgfA30LX9Jx5b+PXr04ERFRlUHv8OfPcvU3qxEkIiKqNWgM/4eSzmPfUxwCYPvs6mJFRMSoDSr8VwK/BLyCxac4jIiIFhq0p+3dwLSkm8rTHUZERIsNM0vn25IulnRXeflMeerCiIhokWEK/3zgUuCnystl5XUREdEiwxT+k2yfb3tvebkAOKLiXBERMWLDFP7dkk6XdHB5OR34dtXBIiJitIYp/H8P/CpwJ/CPwKvIma8iIlpnmIOn3Q4MfXariIhophxLJyKiI1L4EREdUVnhS/oJSV+R9FVJN0h6V1XrioiI5S1b+JJ+Z8HX+3PkzO8DL7H9XOAY4HhJL9r/iBERMQqDDo/8VkkvppiVM2/oI2e6MFd+e2h58YpSRkTEARv0Dv9m4BTgmZL+j6Q/AZ4o6ahhH7yct78DuAv4vO1rDixuRESslOzeb7ol/SJwDfDXwAuA5wCXA18EjrL9z4deifR44GLgTba/tuS2TZQnRR8fH984PT3d8zHm5uYYGxsbdpW1Wc2cO3fft+Jlx9fAngdHGKYibckJ7cmanKM1ipwb1q1d8bJTU1PbbU8Mc99Bhf+7wAuBCeAC4Hpgs+2jVxJK0tuB79n+/X73mZiY8OzsbM/bZmZmmJycXMmqV9Vq5ly/5fIVL7t5w1627RzqlMa1aktOaE/W5BytUeTctfWkFS8raejC7zukY/u/2H4psAv4BHAwcISkL0u6bIgQR5Tv7JG0BngZxTBRRETUYJhfS5+1PQvMSnqD7X8h6fAhlnsK8DFJB1P8Yvm07b84kLAREbFywxxa4bcXfHtmed3dQyx3PfC8FSeLiIiR2q8dr3Lmq4iI9sqhFSIiOiKFHxHRESn8iIiOSOFHRHRECj8ioiNS+BERHZHCj4joiBR+RERHpPAjIjoihR8R0REp/IiIjkjhR0R0RAo/IqIjUvgRER2Rwo+I6IgUfkRER6TwIyI6IoUfEdERKfyIiI5I4UdEdERlhS/pqZKulHSjpBsknVPVuiIiYnmHVPjYe4HNtq+T9Fhgu6TP276xwnVGREQflb3Dt/2Ptq8rv/4ucBOwrqr1RUTEYLJd/Uqk9cCXgJ+zff+S2zYBmwDGx8c3Tk9P93yMubk5xsbGqg26Ajt337fo+/E1sOfBmsLsh+QcvbZkTc7RGkXODevWrnjZqamp7bYnhrlv5YUvaQy4Cni37T8fdN+JiQnPzs72vG1mZobJycnRBzxA67dcvuj7zRv2sm1nlSNlo5Gco9eWrMk5WqPIuWvrSSteVtLQhV/pLB1JhwKfAf50ubKPiIhqVTlLR8BHgJts/0FV64mIiOFU+Q7/OOC1wEsk7SgvJ1a4voiIGKCyATLbXwZU1eNHRMT+yZ62EREdkcKPiOiIFH5EREek8CMiOiKFHxHRESn8iIiOSOFHRHRECj8ioiNS+BERHZHCj4joiBR+RERHpPAjIjoihR8R0REp/IiIjkjhR0R0RAo/IqIjUvgRER2Rwo+I6IgUfkRER6TwIyI6IoUfEdERlRW+pI9KukvS16paR0REDK/Kd/gXAMdX+PgREbEfKit8218C7qnq8SMiYv/IdnUPLq0H/sL2zw24zyZgE8D4+PjG6enpnvebm5tjbGys77p27r7vQKKOzPga2PNg3SmWl5yj15asyTlao8i5Yd3aFS87NTW13fbEMPc9ZMVrGRHb5wHnAUxMTHhycrLn/WZmZuh3G8CZWy6vIN3+27xhL9t21r5Zl5Wco9eWrMk5WqPIues1k6MJs4zM0omI6IgUfkRER1Q5LfNC4G+AoyTdIen1Va0rIiKWV9kAme3TqnrsiIjYfxnSiYjoiBR+RERHpPAjIjoihR8R0REp/IiIjkjhR0R0RAo/IqIjUvgRER2Rwo+I6IgUfkRER6TwIyI6IoUfEdERKfyIiI5I4UdEdEQKPyKiI1L4EREdkcKPiOiIFH5EREek8CMiOiKFHxHREZUWvqTjJd0i6euStlS5roiIGKyywpd0MPB+4ATgaOA0SUdXtb6IiBisynf4xwJft32b7R8A08C/qnB9ERExgGxX88DSq4Djbf9a+f1rgRfafuOS+20CNpXfHgXc0uchDwfuriTsaCXnaLUlJ7Qna3KOVt05n277iGHueEjVSZZj+zzgvOXuJ2nW9sQqRDogyTlabckJ7cmanKPVlpxQ7ZDObuCpC74/srwuIiJqUGXhXws8S9IzJD0KOBW4tML1RUTEAJUN6djeK+mNwGeBg4GP2r7hAB5y2WGfhkjO0WpLTmhP1uQcrbbkrO5D24iIaJbsaRsR0REp/IiIjkjhR0R0RAo/IqIjat/xqhdJa4HjgXXlVbuBz9q+t75UEe0k6dkUhzVZ+PN0qe2b6kvVXm3eno17hy/pDOA6YBJ4THmZAraXt8WPMUmvkPRBSZeWlw9KOr7uXAtJOkTSr0u6QtL15eWvJP1HSYfWnW8hSW+lOI6VgK+UFwEXNu0Iti157luzPXtp3LRMSbdQHHPn3iXX/yRwje2fqSdZb5JeAfxrFv+2v8T2FfWlWkzSIcDrgX8D/FR59W7gEuAjtn9YV7aFJP0h8DPAx4E7yquPBM4AbrV9Tl3ZFpJ0IXAv8DEW53wd8ATbr64r21KS/g742aXPcbkz5A22n1VPssVa9Ny3Ynv208QhHQG9fgs9VN7WGANepGdLOqEpL1LgExQF9U72LahPAk0pqBN7/UKX9GfA3wFN2Z4be+S8A7i6LIQmeYjil/ztS65/SnlbU7TluW/L9uypiYX/buA6SZ8D/r687mnAy4D/Xluq3tryIm1LQf2TpBfYvnbJ9S8A/qmOQH3cI+kU4DO2HwKQdBBwCvCdWpPt67eAL0i6lcU/Tz8NvLHvUquvLc99W7ZnT40b0oGHh29ewb4f2jbqh0nS9cDrl75IJR1LMVSyoZ5ki0m6GthG74J6i+0X1plvnqTnAx8EHssjf4k8FbgP+E3b2+vKtpCk9cB7gZdQFLyAxwNfBLbY/mZt4Xoon+tjWfzzdK3tH9WXarG2PPfQju3ZTyMLH0DSOAs2qO09debppS0v0hYW1JNZ/NzfWWeeQSQ9EcD2t+vO0o8ksW9BfcUN/OFvw3Pfpu25VOMKX9IxwP8E1lKUqCjGm+8FfsP2dTXG66kNL9J5TS+otkzJ7TM17xLbN9eXal+SXg58ALiVRw5PfiTFEMRv2P5cXdmWasNz36bt2UsTC38H8Ou2r1ly/YuAD9l+bj3JemvDixTaUVDltNt3AJ9j8Q/Ty4B32f54XdkWKqfmnUYxPW/hh+CnAtO2t9aVbSlJNwEn2N615PpnAH9p+zm1BFuiRc99K7ZnP00s/Fv7TW2S9HXbP73amfpp0Yu0FQXVlim5bZqaV364+Bzbe5dc/yjgxqb8PLXouW/F9uynibN0/krS5RRTHec/BX8qxXzcxsxtL/1XihkwPV+kFP+HJng9vQvqD4AbgEYUPu2ZktumqXkfBa6VNM3in6dTgY/UlmpfbXnu27I9e2pc4ds+W9IJ7Dv88H7bf1lfsp7a8iJtS0G1ZUpua6bm2X6PpEuAk4EXl1fvBl5j+8b6ku2jFc99i7ZnT40b0mkTSa8D3k4xpLPPi9T2BTVFW6TcPf2PKT5o2qegGrZXcFum5LZuap6kJwDYvqfuLL205bmf1/Tt2UvjCr/8EPRtFO/wxyneQd9FcRiArQ38MLQVL9I2FVRLpuS2YmqepKcBv0cxJfc+ir88H8cjU3J31ZduX01/7tu2PZdq3JAO8GmKjTc1P72xnPZ4Znnby+uLti/b35F0JYtfpI0q+5IXXOa/b9JwTt8puZIaNSV30NQ8SU2bmvdnwB9SDDn8CEDSwRQ73U0DL6ox28Pa8tzTku3ZTxPf4d9i+6j9va0ObdlnoC1zh9syJbdNU/OWmfXW97bV1qLnvhXbs58mvsO/XdJvAx+b/3Ou/DPvTB4Zf26KC+j/Ij0faMSLFDgX+KV+BQU0paAOW7otAWxfLemwOgL1cQiPTG9daDfQqMMjUxxW/AMUR/ZcOKvkdcDf1pZqX2157tuyPXtqYuG/GtgCXFUWvYE9wKXAr9YZrIe2vEjbUlBtmZLbpql5Z1BMy30XS07YQbOytuW5b8v27KlxQzpLSfqXFB+O7WzK0MM8SX8E/DN6v0i/absRU/QkvY3il2Wvgvq07ffUlW2pPlNyL23alFxJz6F3zsZPzWuqtjz3bda4wpf0FdvHll//GvCbwP+m+LD2sqbsFTqvLS/SFFQ36ZGT3+xzkh4adPKbtmj79mxi4f+t7eeVX19Lccz5b5VDJFe7IYccjtFry5RcScfP77tQZt5G8Vfo14A3N2kqoVpydi5JP2/7+vLrQ4G38sg2/R+2v1dnvnlt2Z79NO6ctsBBkn6yPKqjbH8LwPYDwN7Bi64uSWslbZV0k6R7JH27/HqrpMfXnW+eFpwXtMz8YRXnYf1U+TlJU3ya4vDNU7afYPuJFOczvre8rSl+d8HX24A7gV8GrgU+VEui/jbafoPtq23fUV6utv0G4Hl1h1vgggVfb6WYQbYNWEMxE64p2rI9e2pi4a8FtgOzwBMkPQVA0hjNOlwBpKBGbb3t93rB4aVt31kO4z29xlyDTNj+Hdu3234fsL7uQEvcI+mUcsc7oNgJT9KradbZuRb+bL8U+A+2rwLeAhxTT6Se2rI9e2rcLB3b6/vc9BDFSbibZL3t9y68oiyrrZLOqinTciZsz/8Ava88PERTtGVK7pMkvYVyL0tJWrCHbdPeRJ1KcfKb95c7MUFx8psry9uaYq2kX6HYpo+eHwu3bUlNGnee354fkDR/MqG1NG979tS4wu+nHMNr1JmZSEGNWlum5P4JxVnOoBjLPRz4VrlH+I7aUvVge5eKo6JuA74BPJvioF83ullnOruK4q9OKM61PG57T7lN764x1yLlviyvhkdOJgSca/v02kLth8Z9aNsm5XF0tlB8yPik8ur5gtralEMsSHrHkqs+UH4Q/mTg92yfUUeuXlScqOVIig/o5xZc//AHpU1Q5lxHcaz2Jud8B3ACxZu7z1N8EDpDcYC/z9p+d33pFpP0QuAh29dKOprixEI3N2nGm6RLe1z9EorDwWD75NVNtH9S+BWRdJbt8+vOsZwm5ZR0NsU03Jsoxm3PsX1Jedt1tp9fZ755kt5EcRjkRucEkLSTIuOjKT67OdL2/ZLWUPyy+vlaA5ba8otJ0nXAjcCHKf4CFXAh5XBO+blDY6XwKyLp/9l+Wt05ltOknGU5vdj2nIoTr18EfML2uQun69atLTlhn2nOi7JJ2rHg85xategX00HAOcCJwH+2vUPSbbafWXO0obRmDL+JJF3f7yaKeeSN0JacwEHzwyPl2PMkcJGkp9OsGVptyQnwA0mPKT8D2zh/Zbn/QJOOlrrXxdEnvyfpG7bvB7D9oKTG5LT9EMVkh/9V/ruHFvVoa4I21DjFsfCXjtUL+OvVj9NXW3LukXSM7R0A5TvoV1Icu6ZJO9y1JSfAL9j+PjxcVvMOpdhZqCna8osJANt3AKdIOgm4v+48w8qQzgGQ9BHgfNtf7nHbp2z/uxpi7aNFOY+keKd3Z4/bjrP9f2uItY+25GwTSY+e/8W05PrDgafY3llDrB87KfyIiI5o0hzsiIioUAo/IqIjUvgRER2Rwo+I6Ij/D9QPGysSLisKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "ax=progressDf_sub['date_time'][progressDf_sub['date_time']>np.datetime64('2018-08-01')].value_counts().sort_index().hist(cumulative=True)\n",
    "ax.set_ylabel('# of data')\n",
    "#plt.xticks(x, labels, rotation='vertical')\n",
    "plt.setp( ax.xaxis.get_majorticklabels(), rotation=90 )\n",
    "\n",
    "#ax.figure.savefig('../../Analysis/Figures/DataAvail/{}.pdf'.format(param.pipelineName))\n",
    "#ax.figure.savefig('../../Analysis/Figures/DataAvail/{}.pdf'.format(param.pipelineName))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../../Analysis/Figures/DataAvail/TranscriptCounting.pdf'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'../../Analysis/Figures/DataAvail/{}.pdf'.format(param.pipelineName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>permission</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>size</th>\n",
       "      <th>date_str</th>\n",
       "      <th>time</th>\n",
       "      <th></th>\n",
       "      <th>fname</th>\n",
       "      <th>date_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>853403</th>\n",
       "      <td>-rw-r--r--</td>\n",
       "      <td>1</td>\n",
       "      <td>btsui</td>\n",
       "      <td>users</td>\n",
       "      <td>262K</td>\n",
       "      <td>2018-04-12</td>\n",
       "      <td>00:22:45.015240000</td>\n",
       "      <td>-700</td>\n",
       "      <td>SRR1992284.abundance.tsv.gz</td>\n",
       "      <td>2018-04-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>635645</th>\n",
       "      <td>-rw-r--r--</td>\n",
       "      <td>1</td>\n",
       "      <td>btsui</td>\n",
       "      <td>users</td>\n",
       "      <td>1.5M</td>\n",
       "      <td>2018-04-12</td>\n",
       "      <td>10:15:02.168544000</td>\n",
       "      <td>-700</td>\n",
       "      <td>SRR3127552.abundance.tsv.gz</td>\n",
       "      <td>2018-04-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>635643</th>\n",
       "      <td>-rw-r--r--</td>\n",
       "      <td>1</td>\n",
       "      <td>btsui</td>\n",
       "      <td>users</td>\n",
       "      <td>1.3M</td>\n",
       "      <td>2018-04-12</td>\n",
       "      <td>10:15:02.340884000</td>\n",
       "      <td>-700</td>\n",
       "      <td>SRR402462.abundance.tsv.gz</td>\n",
       "      <td>2018-04-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>635641</th>\n",
       "      <td>-rw-r--r--</td>\n",
       "      <td>1</td>\n",
       "      <td>btsui</td>\n",
       "      <td>users</td>\n",
       "      <td>946K</td>\n",
       "      <td>2018-04-12</td>\n",
       "      <td>10:15:02.455128000</td>\n",
       "      <td>-700</td>\n",
       "      <td>SRR548299.abundance.tsv.gz</td>\n",
       "      <td>2018-04-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>635639</th>\n",
       "      <td>-rw-r--r--</td>\n",
       "      <td>1</td>\n",
       "      <td>btsui</td>\n",
       "      <td>users</td>\n",
       "      <td>1.9M</td>\n",
       "      <td>2018-04-12</td>\n",
       "      <td>10:15:02.822453000</td>\n",
       "      <td>-700</td>\n",
       "      <td>SRR5185531.abundance.tsv.gz</td>\n",
       "      <td>2018-04-12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        permission                   size    date_str                time  \\\n",
       "853403  -rw-r--r--  1  btsui  users  262K  2018-04-12  00:22:45.015240000   \n",
       "635645  -rw-r--r--  1  btsui  users  1.5M  2018-04-12  10:15:02.168544000   \n",
       "635643  -rw-r--r--  1  btsui  users  1.3M  2018-04-12  10:15:02.340884000   \n",
       "635641  -rw-r--r--  1  btsui  users  946K  2018-04-12  10:15:02.455128000   \n",
       "635639  -rw-r--r--  1  btsui  users  1.9M  2018-04-12  10:15:02.822453000   \n",
       "\n",
       "                                   fname  date_time  \n",
       "853403 -700  SRR1992284.abundance.tsv.gz 2018-04-12  \n",
       "635645 -700  SRR3127552.abundance.tsv.gz 2018-04-12  \n",
       "635643 -700   SRR402462.abundance.tsv.gz 2018-04-12  \n",
       "635641 -700   SRR548299.abundance.tsv.gz 2018-04-12  \n",
       "635639 -700  SRR5185531.abundance.tsv.gz 2018-04-12  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "progressDf_sub.sort_values('date_time').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}