2016_WGS_DataPrep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#HIDDEN\n",
    "%matplotlib inline\n",
    "\n",
    "from glob import glob\n",
    "from IPython.display import Audio\n",
    "from ipywidgets import Text, Dropdown, Output, interact\n",
    "from matplotlib.collections import LineCollection\n",
    "from matplotlib import pyplot as plt\n",
    "from os import chdir,system\n",
    "from pandas import DataFrame, merge, read_csv, Series, to_datetime\n",
    "from pickle import dump, load\n",
    "from random import randint \n",
    "from sklearn import manifold\n",
    "from sklearn.metrics import euclidean_distances\n",
    "# import cufflinks as cf\n",
    "import numpy as np\n",
    "import warnings\n",
    "# cf.go_offline()\n",
    "# cf.set_config_file(offline=False, world_readable=True)\n",
    "chdir(\"data\")\n",
    "# done = Audio('done.wav',autoplay=True)\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "def comma(number): return \"{:,}\".format(number)\n",
    "def scolor(i,n,pickColor): \n",
    "    scalar = 255 - int((i/float(n))*120) \n",
    "    if pickColor: return '#%02X%02X%02X' % (128,scalar,128)\n",
    "    return '#%02X%02X%02X' % (75,scalar,200)\n",
    "\n",
    "#conda install -n base bedtools samtools khmer numpy scipy sra-tools trimmomatic jupyterlab \n",
    "#jupyter_contrib_nbextensions widgetsnbextension nodejs sra-tools r-essentials r-base r-rsqlite r-vegan bioconductor-biocinstaller "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OTU\t#Misc\t#Swit\n",
      "OTU2\t40\t37\n",
      "OTU4\t35\t40\n",
      "OTU5\t33\t38\n",
      "OTU6\t43\t38\n",
      "OTU7\t24\t21\n",
      "OTU8\t42\t46\n",
      "OTU10\t44\t30\n",
      "OTU14\t29\t40\n",
      "OTU15\t45\t38\n",
      "OTU18\t43\t29\n",
      "OTU21\t43\t30\n",
      "OTU23\t45\t38\n",
      "OTU29\t43\t38\n",
      "OTU32\t29\t42\n",
      "OTU41\t25\t21\n",
      "OTU47\t30\t37\n",
      "OTU48\t43\t30\n",
      "OTU50\t43\t30\n",
      "OTU66\t28\t42\n",
      "OTU79\t25\t21\n",
      "OTU81\t30\t41\n",
      "OTU84\t29\t40\n",
      "OTU86\t28\t39\n",
      "OTU89\t9\t12\n",
      "OTU90\t29\t40\n",
      "OTU132\t33\t40\n",
      "OTU152\t34\t41\n",
      "OTU179\t28\t42\n",
      "OTU180\t31\t43\n",
      "OTU190\t43\t38\n",
      "OTU192\t29\t42\n",
      "OTU199\t7\t10\n",
      "OTU233\t29\t42\n",
      "OTU246\t29\t42\n",
      "OTU275\t31\t40\n",
      "OTU292\t9\t12\n",
      "OTU343\t29\t43\n",
      "OTU389\t34\t40\n",
      "OTU416\t44\t31\n",
      "OTU430\t40\t37\n",
      "OTU487\t28\t42\n",
      "OTU519\t25\t22\n",
      "OTU537\t29\t40\n",
      "OTU632\t28\t42\n",
      "OTU649\t29\t40\n",
      "OTU842\t6\t12\n",
      "OTU995\t43\t38\n",
      "OTU1088\t3\t1\n",
      "OTU1221\t6\t8\n",
      "OTU1334\t1\t3\n",
      "OTU1409\t24\t13\n",
      "OTU1674\t35\t40\n",
      "OTU1807\t24\t20\n",
      "OTU2771\t28\t39\n",
      "OTU3963\t29\t40\n",
      "OTU3994\t38\t35\n",
      "OTU4223\t25\t21\n",
      "OTU6096\t43\t38\n",
      "OTU6270\t28\t39\n",
      "OTU7737\t42\t28\n",
      "OTU8259\t43\t38\n",
      "OTU9197\t34\t40\n",
      "OTU10438\t19\t18\n",
      "OTU13206\t22\t35\n",
      "OTU17186\t27\t23\n"
     ]
    }
   ],
   "source": [
    "from Bio.Blast import NCBIXML\n",
    "results = NCBIXML.parse(open(\"/home/GLBRCORG/dooley.shanek/GLBRC/data/mapping/CoreUnique.Miscanthus.blastout\",'r'))\n",
    "blastMap={}\n",
    "for res in results: blastMap[res.query]=[len(res.alignments),0]\n",
    "results = NCBIXML.parse(open(\"/home/GLBRCORG/dooley.shanek/GLBRC/data/mapping/CoreUnique.Switchgrass.blastout\",'r'))\n",
    "for res in results: blastMap[res.query][1]=len(res.alignments)\n",
    "\n",
    "print(\"OTU\\t#Misc\\t#Swit\")\n",
    "for otu in blastMap:\n",
    "    print(otu,blastMap[otu][0],blastMap[otu][1],sep=\"\\t\")\n",
    "#     print(res.query,len(res.alignments))\n",
    "#     for alignment in res.alignments:\n",
    "#         print(res.query)\n",
    "#         print(alignment.hit_def)\n",
    "#         for hsp in alignment.hsps: \n",
    "#             print(hsp.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Host Separated Mags\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'reads_file_name'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/bin/anaconda3/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2645\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2646\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2647\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'reads_file_name'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-408e78b5123a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mrawData\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"/mnt/bigdata/seq_facilities/jgi-globus/Seasonal*/Raw_Data/*.gz\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mfastqs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrawData\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mmetaGReadFiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmetaData\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"reads_file_name\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munique\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mfastq\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfastqs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mreadFileName\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfastq\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfastq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/bin/anaconda3/lib/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2798\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2799\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2800\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2801\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2802\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/bin/anaconda3/lib/python3.7/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2646\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2647\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2648\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2649\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2650\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'reads_file_name'"
     ]
    }
   ],
   "source": [
    "### Get the metadata and create links to the metaG data from JGI\n",
    "metaData = read_csv(\"metadata/GLBRC_MetaG_Metadata.tsv\",sep=\"\\t\")\n",
    "metaData.set_index(\"nucleic_acid_name\",inplace=True)\n",
    "rawData = \"/mnt/bigdata/seq_facilities/jgi-globus/Seasonal*/Raw_Data/*.gz\"\n",
    "fastqs = glob(rawData)\n",
    "metaGReadFiles = set(metaData[\"reads_file_name\"].unique())\n",
    "for fastq in fastqs:\n",
    "    readFileName = fastq[fastq.rfind(\"/\")+1:]\n",
    "    if readFileName not in metaGReadFiles:continue #it is a metaT file\n",
    "    nucleicAcidName = metaData[metaData[\"reads_file_name\"] == readFileName].index[0]\n",
    "    system(\"ln -s %s mapping/metaG/unpaired/%s.fastq.gz\" % (fastq,nucleicAcidName))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "225"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metaData = read_csv(\"metadata/GLBRC_MetaT_Metadata.tsv\",sep=\"\\t\")\n",
    "metaData.set_index(\"nucleic_acid_name\",inplace=True)\n",
    "rawData = \"/mnt/bigdata/seq_facilities/jgi-globus/Seasonal*\"\n",
    "fastqs = glob(rawData)\n",
    "len(fastqs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R1_MAIN_31MAY2016_LR1_MT_pilot_[G5RMTpilot_4_FD]\n",
      "2 G5R2_MAIN_31MAY2016_LR1\n",
      "3 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R2_MAIN_31MAY2016_LD1_MG_[G5R1MG_12_FD]\n",
      "4 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R3_MAIN_31MAY2016_LD1_MG_[G5R1MG_13_FD]\n",
      "5 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R4_MAIN_31MAY2016_LD1_MG_[G5R1MG_9_FD]\n",
      "6 G5R1_NF_31MAY2016_LR2\n",
      "7 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R1_NF_31MAY2016_LD2_MG_[G5R1MG_10_FD]\n",
      "8 G5R3_NF_31MAY2016_LR2\n",
      "9 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R3_NF_31MAY2016_LD1_MG_[G5R1MG_46_FD]\n",
      "10 G5R4_NF_31MAY2016_LR3\n",
      "11 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R4_NF_31MAY2016_LD1_MG_[G5R1MG_47_FD]\n",
      "12 G5R1_MAIN_12JUL2016_LR1\n",
      "13 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R1_MAIN_12JUL2016_LD2_MG_[G5R1MG_57_FD]\n",
      "14 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R2_MAIN_12JUL2016_LD1_MG_[G5R1MG_58_FD]\n",
      "15 G5R3_MAIN_12JUL2016_LR1\n",
      "16 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R3_MAIN_12JUL2016_LD1_MG_[G5R1MG_59_FD]\n",
      "17 G5R4_MAIN_12JUL2016_LR1\n",
      "18 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R4_MAIN_12JUL2016_LD1_MG_[G5R1MG_61_FD]\n",
      "19 G5R1_NF_12JUL2016_LR1\n",
      "20 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R1_NF_12JUL2016_LD1_MG_[G5R1MG_62_FD]\n",
      "21 G5R2_NF_12JUL2016_LR1\n",
      "22 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R2_NF_12JUL2016_LD1_MG_[G5R1MG_63_FD]\n",
      "23 G5R4_NF_12JUL2016_LR2\n",
      "24 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R4_NF_12JUL2016_LD1_MG_[G5R1MG_60_FD]\n",
      "25 G5R1_MAIN_12SEP2016_LR1\n",
      "26 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R1_MAIN_12SEP2016_LR1_MT_pilot_[G5RMTpilot_2_FD]\n",
      "27 G5R2_MAIN_12SEP2016_LR1\n",
      "28 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R2_MAIN_12SEP2016_LD1_MG_[G5R1MG_35_FD]\n",
      "29 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R3_MAIN_12SEP2016_LD1_MG_[G5R1MG_36_FD]\n",
      "30 G5R4_MAIN_12SEP2016_LR1\n",
      "31 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R4_MAIN_12SEP2016_LD1_MG_[G5R1MG_37_FD]\n",
      "32 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R1_NF_12SEP2016_LD1_MG_[G5R1MG_38_FD]\n",
      "33 G5R2_NF_12SEP2016_LR2\n",
      "34 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R2_NF_12SEP2016_LD1_MG_[G5R1MG_14_FD]\n",
      "35 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R3_NF_12SEP2016_LD1_MG_[G5R1MG_15_FD]\n",
      "36 G5R4_NF_12SEP2016_LR1\n",
      "37 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R4_NF_12SEP2016_LD1_MG_[G5R1MG_16_FD]\n",
      "38 G5R1_MAIN_15MAY2017_LR1\n",
      "39 G5R2_MAIN_15MAY2017_LR1\n",
      "40 /mnt/bigdata/seq_facilities/jgi-globus/Seasonal_dynamics_of_switchgrass_and_miscanthus_phyllosphere_microbiota__community_structure_and_function__G5R3_MAIN_15MAY2017_LD1_MG_[G5R3_MAIN_15MAY2_FD]\n",
      "41 G5R4_MAIN_15MAY2017_LR1\n",
      "42 G5R1_NF_15MAY2017_LR1\n",
      "43 G5R2_NF_15MAY2017_LR1\n",
      "44 G5R3_NF_15MAY2017_LR1\n",
      "45 G5R4_NF_15MAY2017_LR1\n",
      "46 G5R1_MAIN_05JUN2017_LR1\n",
      "47 G5R2_MAIN_05JUN2017_LR1\n",
      "48 G5R3_MAIN_05JUN2017_LR1\n",
      "49 G5R4_MAIN_05JUN2017_LR1\n",
      "50 G5R1_NF_05JUN2017_LR1\n",
      "51 G5R2_NF_05JUN2017_LR1\n",
      "52 G5R3_NF_05JUN2017_LR1\n",
      "53 G5R4_NF_05JUN2017_LR1\n",
      "54 G5R1_MAIN_26JUN2017_LR1\n",
      "55 G5R2_MAIN_26JUN2017_LR1\n",
      "56 G5R3_MAIN_26JUN2017_LR1\n",
      "57 G5R4_MAIN_26JUN2017_LR1\n",
      "58 G5R1_NF_26JUN2017_LR1\n",
      "59 G5R2_NF_26JUN2017_LR1\n",
      "60 G5R3_NF_26JUN2017_LR1\n",
      "61 G5R4_NF_26JUN2017_LR1\n",
      "62 G5R1_MAIN_17JUL2017_LR1\n",
      "63 G5R2_MAIN_17JUL2017_LR1\n",
      "64 G5R3_MAIN_17JUL2017_LR1\n",
      "65 G5R4_MAIN_17JUL2017_LR1\n",
      "66 G5R1_NF_17JUL2017_LR1\n",
      "67 G5R2_NF_17JUL2017_LR1\n",
      "68 G5R3_NF_17JUL2017_LR1\n",
      "69 G5R4_NF_17JUL2017_LR1\n",
      "70 G5R1_MAIN_07AUG2017_LR1\n",
      "71 G5R2_MAIN_07AUG2017_LR1\n",
      "72 G5R3_MAIN_07AUG2017_LR1\n",
      "73 G5R4_MAIN_07AUG2017_LR1\n",
      "74 G5R1_NF_07AUG2017_LR1\n",
      "75 G5R2_NF_07AUG2017_LR1\n",
      "76 G5R3_NF_07AUG2017_LR1\n",
      "77 G5R4_NF_07AUG2017_LR1\n",
      "78 G5R1_MAIN_28AUG2017_LR1\n",
      "79 G5R2_MAIN_28AUG2017_LR1\n",
      "80 G5R3_MAIN_28AUG2017_LR1\n",
      "81 G5R4_MAIN_28AUG2017_LR1\n",
      "82 G5R1_NF_28AUG2017_LR1\n",
      "83 G5R2_NF_28AUG2017_LR1\n",
      "84 G5R3_NF_28AUG2017_LR1\n",
      "85 G5R4_NF_28AUG2017_LR1\n",
      "86 G5R1_MAIN_18SEP2017_LR1\n",
      "87 G5R2_MAIN_18SEP2017_LR1\n",
      "88 G5R3_MAIN_18SEP2017_LR1\n",
      "89 G5R4_MAIN_18SEP2017_LR1\n",
      "90 G5R3_NF_18SEP2017_LR1\n",
      "91 G5R4_NF_18SEP2017_LR1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "23"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c=0\n",
    "from os import path\n",
    "files = set()\n",
    "for name in metaData.index: \n",
    "    found = False\n",
    "    for fastq in fastqs:\n",
    "        if name[:-4] in fastq:\n",
    "            c+=1\n",
    "            if path.exists(fastq+\"/Raw_Data\"):\n",
    "                print(c,fastq)\n",
    "                files.add(name)\n",
    "                found = True\n",
    "                break\n",
    "            else:print(c,name)\n",
    "                \n",
    "#     if not found:\n",
    "#         print(\"Missing \",name)\n",
    "#             system(\"ln -s %s mapping/metaT/unpaired/%s\" % (fastq,name))\n",
    "\n",
    "len(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/mnt/bigdata/linuxhome/dooley.shanek/GLBRC/data\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "pwd\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1><a id='outline'>Project Outline</a></h1>\n",
    "\n",
    "* [Sequencing Summaries](#seqSums)\n",
    "    *  [Adapter Trimming and QC (Trimmomatic) Report](#readbd)\n",
    "    *  [Host Plant Alignment Report](#pAlign)\n",
    "    *  [Switchgrass Fungal Alignments](#swAlign)\n",
    "    *  [Alignment Conclusions](#aConc)\n",
    "* [Annotation Report](#anno)   \n",
    "\n",
    "\n",
    "* [NMDS of annotated contigs](#nmds)\n",
    "  *  [All Crops](#anmds)\n",
    "  *  [Switchgrass](#sgnmds)\n",
    "  *  [Miscanthus](#msnmds)\n",
    "*  [Function Analysis](#metaT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#HIDDEN\n",
    "# For metaG\n",
    "metadataG = read_csv(\"metadata/GLBRC_MetaG_Metadata.tsv\",sep='\\t')\n",
    "metadataG.set_index(\"nucleic_acid_name\",inplace=True)\n",
    "metadataG.drop([\"source\",\"sampling ID\",\"sequencing_type\",\"height_mean_cm\",\"air_temp_c\",\"rep\",\"Sampling Time\",\"reads_file_name\",\"treatment\"],axis=1,inplace=True) \n",
    "for id in metadataG.index: metadataG.loc[id,\"type\"] = metadataG[metadataG.index == id].plot_name[0][0:2]\n",
    "metadataG['Date'] = to_datetime(metadataG.sampling_date)\n",
    "metadataG=metadataG.sort_values(by=[\"type\",\"Date\"])\n",
    "metadataG.drop([\"sampling_date\"],axis=1,inplace=True)\n",
    "metaG_Reads = read_csv(\"mapping/metaG/hostRemovalFlagstats/multiqc_data/multiqc_bowtie2.txt\",sep=\"\\t\")\n",
    "for id in metaG_Reads.index: metaG_Reads.at[id,\"Sample\"] = metaG_Reads.at[id,\"Sample\"].replace(\".stat\",\"\")\n",
    "metaG_Reads.set_index(\"Sample\",inplace=True)\n",
    "metadataG = merge(metadataG,metaG_Reads,left_index=True,right_index=True)\n",
    "metadataG.sort_values(\"Date\",inplace=True)\n",
    "metadataG[\"aligned\"] = (metadataG['paired_total']-metadataG['paired_aligned_mate_none_halved'])\n",
    "metadataG[\"percPlantAligned\"]= metadataG[\"overall_alignment_rate\"]\n",
    "metadataG[\"Plant Reads\"] = metadataG[\"aligned\"]\n",
    "metadataG[\"Total\"] = metadataG[\"paired_total\"]\n",
    "metadataG.drop(['aligned','total_reads','paired_total','paired_aligned_none','paired_total','paired_aligned_one', 'paired_aligned_multi',\n",
    "       'paired_aligned_discord_one', 'paired_aligned_mate_none',\n",
    "       'paired_aligned_mate_one', 'paired_aligned_mate_multi',\n",
    "       'overall_alignment_rate', 'paired_aligned_mate_multi_halved',\n",
    "       'paired_aligned_mate_none_halved', 'paired_aligned_mate_one_halved'],axis=1,inplace=True)\n",
    "metaG_Reads = read_csv(\"mapping/metaG/fungalRemovalFlagstats/multiqc_data/multiqc_bowtie2.txt\",sep=\"\\t\")\n",
    "for id in metaG_Reads.index: metaG_Reads.at[id,\"Sample\"] = metaG_Reads.at[id,\"Sample\"].replace(\".fungal.stat\",\"\")\n",
    "metaG_Reads.set_index(\"Sample\",inplace=True)\n",
    "metadataG = merge(metadataG,metaG_Reads,left_index=True,right_index=True)\n",
    "metadataG.sort_values(\"Date\",inplace=True)\n",
    "metadataG[\"Fungal Reads\"] = (metadataG['paired_total']-metadataG['paired_aligned_mate_none_halved'])\n",
    "metadataG[\"percFungalAligned\"]= metadataG[\"overall_alignment_rate\"]\n",
    "metadataG['percFungalAligned'] = Series([float(\"{0:.2f}%\".format(val).replace(\"%\",\"\")) for val in metadataG['percFungalAligned']], index = metadataG.index)\n",
    "metadataG['percPlantAligned']  = Series([float(\"{0:.2f}%\".format(val).replace(\"%\",\"\")) for val in metadataG['percPlantAligned']], index = metadataG.index)\n",
    "metadataG['Remaining'] = metadataG['paired_aligned_mate_none_halved']\n",
    "metadataG['percRemaining'] = (metadataG[\"Remaining\"]/metadataG['Total'])*100\n",
    "\n",
    "metadataG.drop(['total_reads','paired_total','paired_aligned_none','paired_total','paired_aligned_one', 'paired_aligned_multi',\n",
    "       'paired_aligned_discord_one', 'paired_aligned_mate_none',\n",
    "       'paired_aligned_mate_one', 'paired_aligned_mate_multi',\n",
    "       'overall_alignment_rate', 'paired_aligned_mate_multi_halved',\n",
    "       'paired_aligned_mate_none_halved', 'paired_aligned_mate_one_halved'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 id=\"seqSums\">Read QC and alignment</h2>\n",
    "\n",
    "#### Adapter trimming and QC\n",
    "\n",
    "![alt text](images/TrimReport.png \"Title\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4 id='plantAlign'> When the reads are aligned to plant reference genomes, how many of the reads are plant-aligned reads?</h4>\n",
    "\n",
    "<p><b><a href=\"scripts/hpc_scripts/NGS_Mapping.sh\">Methodology</a></b></p>\n",
    "    \n",
    "<p>QC'd reads were aligned to their respective plant reference genome: </p>\n",
    "    <ol>\n",
    "        <li>bowtie2 paired-end alignment to respective reference assembly</li>\n",
    "        <li>Report tables generated using <a href=\"https://multiqc.info/\">multiqc</a></li>\n",
    "        <li>Displayed below using pandas and nbinteract, two python libraries</li>\n",
    "    </ol> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9d560351eb948a09990b8d0b48c8364",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='Host_Plant', options=('Switchgrass & Miscanthus', 'Switchgrass', '…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#HIDDEN\n",
    "\n",
    "dataStr = \"\"\"G5R2_MAIN_09MAY2016_LD1,G5R2_MAIN,G5,2016-05-09,43.02,2444885.5,5682372,230.5,0.01,3101297.5,54.57751622033897\n",
    "G5R4_NF_09MAY2016_LD1,G5R4_NF,G5,2016-05-09,79.73,4575140.5,5737475,245.0,0.03,938323.0,16.354284768125353\n",
    "G5R1_NF_09MAY2016_LD1,G5R1_NF,G5,2016-05-09,37.16,2034146.5,5473795,487.0,0.01,3333201.0,60.89378575558639\n",
    "G5R2_NF_09MAY2016_LD1,G5R2_NF,G5,2016-05-09,61.07,4093313.5,6700488,383.5,0.02,2401160.5,35.83560630210815\n",
    "G5R4_MAIN_09MAY2016_LD1,G5R4_MAIN,G5,2016-05-09,85.16,4535606.0,5324879,283.5,0.05,569866.5,10.701961490580349\n",
    "G5R1_MAIN_09MAY2016_LD1,G5R1_MAIN,G5,2016-05-09,73.46,5016843.5,6828883,240.0,0.02,1572579.0,23.028348852952966\n",
    "G5R3_NF_09MAY2016_LD1,G5R3_NF,G5,2016-05-09,80.7,5001844.0,6196018,199.5,0.02,949029.5,15.316764735028206\n",
    "G6R1_MAIN_09MAY2016_LD1,G6R1_MAIN,G6,2016-05-09,85.88,5766456.0,6713739,39.0,0.01,559512.0,8.33383603384046\n",
    "G5R3_MAIN_09MAY2016_LD1,G5R3_MAIN,G5,2016-05-09,84.41,4590265.0,5436507,272.5,0.04,625518.5,11.505889719262754\n",
    "G6R3_MAIN_09MAY2016_LD1,G6R3_MAIN,G6,2016-05-09,86.34,6169902.0,7145090,63.0,0.01,564880.0,7.905848631717725\n",
    "G6R1_NF_09MAY2016_LD1,G6R1_NF,G6,2016-05-09,86.57,6516872.5,7527497,52.5,0.01,571074.5,7.586512488812683\n",
    "G6R2_MAIN_09MAY2016_LD1,G6R2_MAIN,G6,2016-05-09,86.06,5759999.0,6692593,115.5,0.02,549879.5,8.216239953632321\n",
    "G6R4_NF_09MAY2016_LD1,G6R4_NF,G6,2016-05-09,86.98,5748471.0,6607949,88.5,0.02,480675.5,7.274201117472305\n",
    "G6R2_NF_09MAY2016_LD1,G6R2_NF,G6,2016-05-09,86.22,11483052.5,13316083,228.5,0.02,1075775.5,8.078768358533061\n",
    "G6R4_MAIN_09MAY2016_LD1,G6R4_MAIN,G6,2016-05-09,86.36,10680511.5,12361687,570.5,0.06,985639.5,7.973341340870384\n",
    "G6R3_NF_09MAY2016_LD1,G6R3_NF,G6,2016-05-09,86.58,5920682.5,6837832,62.5,0.01,523727.5,7.65926246798693\n",
    "G6R1_MAIN_31MAY2016_LD1,G6R1_MAIN,G6,2016-05-31,86.0,4589765.0,5336461,204.0,0.05,447497.0,8.385651089739062\n",
    "G5R1_MAIN_31MAY2016_LD1,G5R1_MAIN,G5,2016-05-31,87.81,5542677.0,6310890,394.5,0.08,491297.5,7.784916232100385\n",
    "G5R4_MAIN_31MAY2016_LD1,G5R4_MAIN,G5,2016-05-31,88.47,5921339.0,6691560,427.5,0.09,475789.5,7.1102926671807465\n",
    "G5R4_NF_31MAY2016_LD1,G5R4_NF,G5,2016-05-31,87.97,5218316.5,5931777,626.5,0.14,456660.5,7.698544635106816\n",
    "G5R2_MAIN_31MAY2016_LD1,G5R2_MAIN,G5,2016-05-31,88.29,10553216.5,11950005,619.0,0.07,879644.0,7.36103457697298\n",
    "G5R1_NF_31MAY2016_LD1,G5R1_NF,G5,2016-05-31,87.65,6821004.0,7776723,938.0,0.16,601802.0,7.738503737371126\n",
    "G5R2_NF_31MAY2016_LD1,G5R2_NF,G5,2016-05-31,88.12,8249743.5,9358299,439.0,0.06,674956.0,7.212379087267889\n",
    "G5R3_NF_31MAY2016_LD1,G5R3_NF,G5,2016-05-31,87.5,5036978.0,5755740,714.0,0.15,474196.0,8.238662622008638\n",
    "G6R2_NF_31MAY2016_LD1,G6R2_NF,G6,2016-05-31,86.3,6707432.0,7771064,210.0,0.03,628542.0,8.088236051073572\n",
    "G6R3_MAIN_31MAY2016_LD1,G6R3_MAIN,G6,2016-05-31,85.16,5393305.5,6331750,557.0,0.09,589557.0,9.311122517471473\n",
    "G6R1_NF_31MAY2016_LD1,G6R1_NF,G6,2016-05-31,85.65,5390104.5,6292542,689.5,0.13,545324.5,8.66620357877627\n",
    "G6R4_MAIN_31MAY2016_LD1,G6R4_MAIN,G6,2016-05-31,84.24,5567014.5,6607123,572.0,0.08,676111.0,10.233062105851518\n",
    "G6R2_MAIN_31MAY2016_LD1,G6R2_MAIN,G6,2016-05-31,85.91,5781888.5,6729977,440.5,0.07,603103.5,8.961449645370259\n",
    "G6R4_NF_31MAY2016_LD1,G6R4_NF,G6,2016-05-31,85.77,6503491.5,7581116,545.5,0.08,699856.5,9.231576195378095\n",
    "G5R3_MAIN_31MAY2016_LD1,G5R3_MAIN,G5,2016-05-31,88.3,5812968.5,6582349,242.0,0.05,471430.0,7.162032885220762\n",
    "G6R3_NF_31MAY2016_LD1,G6R3_NF,G6,2016-05-31,85.55,4907993.0,5736908,393.5,0.08,502712.5,8.762777789011084\n",
    "G6R3_NF_20JUN2016_LD1,G6R3_NF,G6,2016-06-20,84.6,5671552.0,6703466,2010.5,0.31,646680.5,9.64695726061712\n",
    "G6R3_MAIN_20JUN2016_LD1,G6R3_MAIN,G6,2016-06-20,85.86,6135909.0,7145612,849.0,0.14,592992.0,8.298687362258123\n",
    "G6R4_NF_20JUN2016_LD1,G6R4_NF,G6,2016-06-20,83.87,10446119.5,12452913,2776.5,0.19,1446949.5,11.619365685763643\n",
    "G6R2_MAIN_20JUN2016_LD1,G6R2_MAIN,G6,2016-06-20,85.41,11384326.0,13325217,3304.0,0.28,1180202.0,8.856906420360735\n",
    "G6R4_MAIN_20JUN2016_LD1,G6R4_MAIN,G6,2016-06-20,81.91,5485149.0,6695207,1731.5,0.2,851777.5,12.722198133679811\n",
    "G6R2_NF_20JUN2016_LD1,G6R2_NF,G6,2016-06-20,85.47,5712932.0,6682956,2443.0,0.41,593585.0,8.882072543946123\n",
    "G6R1_MAIN_20JUN2016_LD1,G6R1_MAIN,G6,2016-06-20,86.14,5177962.0,6010405,647.5,0.13,482787.5,8.032528590003503\n",
    "G5R4_NF_20JUN2016_LD1,G5R4_NF,G5,2016-06-20,87.94,6100780.5,6936916,2196.0,0.41,537134.0,7.7431238896362595\n",
    "G5R4_MAIN_20JUN2016_LD1,G5R4_MAIN,G5,2016-06-20,86.83,6715795.5,7732527,5536.0,0.81,680666.0,8.802633343536984\n",
    "G5R3_NF_20JUN2016_LD1,G5R3_NF,G5,2016-06-20,81.38,5141417.0,6317596,1620.0,0.18,911401.0,14.426389405083833\n",
    "G5R1_MAIN_20JUN2016_LD1,G5R1_MAIN,G5,2016-06-20,77.82,5013160.5,6441413,2211.5,0.19,1180776.5,18.33101681261549\n",
    "G5R3_MAIN_20JUN2016_LD1,G5R3_MAIN,G5,2016-06-20,88.48,5700839.5,6442699,994.5,0.22,456179.5,7.080565148239891\n",
    "G5R2_NF_20JUN2016_LD1,G5R2_NF,G5,2016-06-20,87.78,5487295.0,6249934,3378.0,0.68,493303.0,7.89293134935505\n",
    "G5R2_MAIN_20JUN2016_LD1,G5R2_MAIN,G5,2016-06-20,88.04,5589304.5,6347927,2927.5,0.6,482960.5,7.608160900401028\n",
    "G6R1_NF_20JUN2016_LD1,G6R1_NF,G6,2016-06-20,85.51,6103073.5,7136929,1316.0,0.21,616980.0,8.644894743943789\n",
    "G5R1_NF_20JUN2016_LD1,G5R1_NF,G5,2016-06-20,86.61,5298116.5,6116500,3308.0,0.6,552031.0,9.025275893076106\n",
    "G5R4_NF_12JUL2016_LD1,G5R4_NF,G5,2016-07-12,84.76,5476482.0,6460341,9528.5,1.35,695774.5,10.769934590140057\n",
    "G5R2_NF_12JUL2016_LD1,G5R2_NF,G5,2016-07-12,87.96,5869056.5,6671812,4146.0,0.8,513375.0,7.694686241159074\n",
    "G5R2_MAIN_12JUL2016_LD1,G5R2_MAIN,G5,2016-07-12,88.65,5987816.0,6753075,6761.5,1.44,464321.5,6.875704771530007\n",
    "G5R1_MAIN_12JUL2016_LD1,G5R1_MAIN,G5,2016-07-12,88.09,8850117.0,10042351,8072.0,1.11,721231.0,7.18189396088625\n",
    "G5R4_MAIN_12JUL2016_LD1,G5R4_MAIN,G5,2016-07-12,86.08,6006044.5,6976158,27967.0,4.09,655552.0,9.397034872203296\n",
    "G5R3_NF_12JUL2016_LD1,G5R3_NF,G5,2016-07-12,87.92,5650799.0,6426349,3666.5,0.74,489209.5,7.6125573011985495\n",
    "G5R1_NF_12JUL2016_LD1,G5R1_NF,G5,2016-07-12,87.65,5863094.0,6688179,15945.5,2.99,517994.5,7.744925786226714\n",
    "G6R2_NF_12JUL2016_LD1,G6R2_NF,G6,2016-07-12,85.86,15344005.5,17866546,4512.5,0.3,1492239.5,8.352143161862399\n",
    "G6R4_NF_12JUL2016_LD1,G6R4_NF,G6,2016-07-12,85.0,11963642.5,14071595,10343.0,0.8,1286317.0,9.141230969197165\n",
    "G6R4_MAIN_12JUL2016_LD1,G6R4_MAIN,G6,2016-07-12,85.15,5534405.0,6499072,4525.5,0.76,588370.5,9.053146356895262\n",
    "G6R1_NF_12JUL2016_LD1,G6R1_NF,G6,2016-07-12,85.63,5643997.5,6590819,2754.0,0.49,559495.0,8.48900569109848\n",
    "G6R1_MAIN_12JUL2016_LD1,G6R1_MAIN,G6,2016-07-12,83.61,4497926.5,5378728,2752.0,0.49,562593.0,10.459591933260057\n",
    "G6R3_MAIN_12JUL2016_LD1,G6R3_MAIN,G6,2016-07-12,86.07,5089800.0,5912717,2432.0,0.51,475522.0,8.042360221197802\n",
    "G6R3_NF_12JUL2016_LD1,G6R3_NF,G6,2016-07-12,84.78,4832968.0,5700114,4311.0,0.8,533441.0,9.358426866550388\n",
    "G5R3_MAIN_12JUL2016_LD1,G5R3_MAIN,G5,2016-07-12,88.14,5450573.0,6182923,4988.5,1.07,461855.5,7.46985689454648\n",
    "G6R2_MAIN_12JUL2016_LD1,G6R2_MAIN,G6,2016-07-12,85.62,10251109.0,11970111,2263.0,0.22,1018829.0,8.511441539681629\n",
    "G6R1_MAIN_01AUG2016_LD1,G6R1_MAIN,G6,2016-08-01,82.98,4695413.5,5657378,10126.5,1.59,627861.5,11.098100568850093\n",
    "G5R3_NF_01AUG2016_LD1,G5R3_NF,G5,2016-08-01,82.23,4856813.5,5905510,24475.5,3.06,775344.5,13.12917089294574\n",
    "G5R4_NF_01AUG2016_LD1,G5R4_NF,G5,2016-08-01,84.25,11083140.5,13151939,105331.5,6.98,1403094.5,10.668347077947974\n",
    "G5R2_MAIN_01AUG2016_LD1,G5R2_MAIN,G5,2016-08-01,78.43,10000184.5,12746203,205100.5,9.13,2041289.5,16.01488302045715\n",
    "G6R4_NF_01AUG2016_LD1,G6R4_NF,G6,2016-08-01,81.26,10655178.0,13108055,21298.5,1.23,1704468.5,13.003214435703846\n",
    "G6R2_NF_01AUG2016_LD1,G6R2_NF,G6,2016-08-01,83.35,5351872.5,6419666,3630.5,0.52,701156.5,10.922009026637834\n",
    "G6R2_MAIN_01AUG2016_LD1,G6R2_MAIN,G6,2016-08-01,82.29,5006352.5,6082749,8431.5,1.15,723326.5,11.89144086004535\n",
    "G6R4_MAIN_01AUG2016_LD1,G6R4_MAIN,G6,2016-08-01,80.62,4765084.5,5908281,6744.0,0.83,809106.0,13.694440057945789\n",
    "G5R1_MAIN_01AUG2016_LD1,G5R1_MAIN,G5,2016-08-01,82.4,4881644.5,5924069,65453.5,8.16,736542.5,12.433050661631388\n",
    "G6R3_MAIN_01AUG2016_LD1,G6R3_MAIN,G6,2016-08-01,84.67,4607519.5,5440967,3466.5,0.67,510451.5,9.381631978286213\n",
    "G5R2_NF_01AUG2016_LD1,G5R2_NF,G5,2016-08-01,84.58,13356404.5,15787307,140160.0,7.88,1639224.0,10.383176814133025\n",
    "G5R4_MAIN_01AUG2016_LD1,G5R4_MAIN,G5,2016-08-01,80.67,5088918.0,6306715,71330.0,7.39,893281.0,14.163966502370887\n",
    "G5R1_NF_01AUG2016_LD1,G5R1_NF,G5,2016-08-01,82.15,5200676.5,6329412,57607.5,6.64,809665.5,12.792112442672398\n",
    "G6R1_NF_01AUG2016_LD1,G6R1_NF,G6,2016-08-01,84.32,4999734.5,5928912,5400.5,0.93,577330.5,9.737545438353614\n",
    "G5R3_MAIN_01AUG2016_LD1,G5R3_MAIN,G5,2016-08-01,81.65,4948744.5,6060102,47449.0,5.51,814192.0,13.435285412687772\n",
    "G6R3_NF_01AUG2016_LD1,G6R3_NF,G6,2016-08-01,77.96,4453617.0,5711239,7273.0,0.77,939309.0,16.44667645671981\n",
    "G5R3_MAIN_22AUG2016_LD1,G5R3_MAIN,G5,2016-08-22,52.0,2735946.5,5255407,147876.0,6.21,2234480.0,42.5177345922019\n",
    "G5R1_NF_22AUG2016_LD1,G5R1_NF,G5,2016-08-22,75.96,4626496.0,6088961,76749.0,6.23,1155024.0,18.96914760991243\n",
    "G5R2_NF_22AUG2016_LD1,G5R2_NF,G5,2016-08-22,79.02,3729827.0,4718274,47488.5,5.86,763407.5,16.179804309796335\n",
    "G5R4_NF_22AUG2016_LD1,G5R4_NF,G5,2016-08-22,78.15,9359890.0,11972027,163507.0,7.59,1990638.0,16.62740987804321\n",
    "G5R2_MAIN_22AUG2016_LD1,G5R2_MAIN,G5,2016-08-22,69.45,9572001.5,13770800,326894.0,8.78,3395956.0,24.66055712086444\n",
    "G6R1_MAIN_22AUG2016_LD1,G6R1_MAIN,G6,2016-08-22,80.3,4487068.5,5586492,7945.0,1.01,776060.0,13.891723106378745\n",
    "G6R3_MAIN_22AUG2016_LD1,G6R3_MAIN,G6,2016-08-22,79.46,3760759.0,4730626,4235.0,0.59,712830.0,15.068407436986142\n",
    "G5R1_MAIN_22AUG2016_LD1,G5R1_MAIN,G5,2016-08-22,73.38,7126087.5,9695253,147605.0,6.71,2051174.0,21.156477298735783\n",
    "G6R3_NF_22AUG2016_LD1,G6R3_NF,G6,2016-08-22,79.41,7336045.0,9235360,11875.0,0.83,1411208.0,15.280487171046934\n",
    "G6R1_NF_22AUG2016_LD1,G6R1_NF,G6,2016-08-22,81.46,7315217.5,8977689,8550.0,0.72,1184033.0,13.188616803277547\n",
    "G6R2_NF_22AUG2016_LD1,G6R2_NF,G6,2016-08-22,80.98,7858611.0,9700669,9664.0,0.72,1332356.0,13.734681597733106\n",
    "G6R2_MAIN_22AUG2016_LD1,G6R2_MAIN,G6,2016-08-22,79.22,8667328.5,10933944,17419.0,1.04,1662195.0,15.202153952864583\n",
    "G6R4_MAIN_22AUG2016_LD1,G6R4_MAIN,G6,2016-08-22,78.0,9370495.5,12007829,15616.0,0.78,1987906.0,16.555082521578214\n",
    "G5R3_NF_22AUG2016_LD1,G5R3_NF,G5,2016-08-22,60.95,3610024.5,5920203,188165.5,8.84,1940423.5,32.77630006944018\n",
    "G5R4_MAIN_22AUG2016_LD1,G5R4_MAIN,G5,2016-08-22,62.04,3661130.5,5898471,193594.0,9.39,1867030.0,31.65277916938135\n",
    "G6R4_NF_22AUG2016_LD1,G6R4_NF,G6,2016-08-22,80.3,8217143.0,10229978,10435.0,0.71,1465380.0,14.324370981051961\n",
    "G5R2_NF_12SEP2016_LD1,G5R2_NF,G5,2016-09-12,57.81,5163234.0,8925694,110375.5,3.14,3405613.5,38.155167542154146\n",
    "G5R2_MAIN_12SEP2016_LD1,G5R2_MAIN,G5,2016-09-12,65.95,6260428.0,9486264,166688.0,5.71,2754756.0,29.03941952279633\n",
    "G5R1_MAIN_12SEP2016_LD1,G5R1_MAIN,G5,2016-09-12,42.27,3368118.5,7957864,248828.5,5.62,4178289.5,52.5051634458694\n",
    "G5R4_NF_12SEP2016_LD1,G5R4_NF,G5,2016-09-12,52.3,3952036.0,7547720,178799.0,5.26,3220462.0,42.668011001997954\n",
    "G5R1_NF_12SEP2016_LD1,G5R1_NF,G5,2016-09-12,61.53,5621047.5,9128071,197461.5,6.1,3041559.5,33.32094480860195\n",
    "G6R2_MAIN_12SEP2016_LD1,G6R2_MAIN,G6,2016-09-12,78.51,7481589.5,9526453,14193.0,0.92,1534811.0,16.11104363817257\n",
    "G5R3_MAIN_12SEP2016_LD1,G5R3_MAIN,G5,2016-09-12,55.23,4388447.0,7938601,161060.0,4.8,3191794.0,40.20600103217179\n",
    "G5R3_NF_12SEP2016_LD1,G5R3_NF,G5,2016-09-12,47.96,3916354.5,8156444,102477.5,2.53,3944008.5,48.3545096367976\n",
    "G6R2_NF_12SEP2016_LD1,G6R2_NF,G6,2016-09-12,80.18,7916339.0,9870656,15662.5,1.09,1416937.5,14.35504894507518\n",
    "G6R1_MAIN_12SEP2016_LD1,G6R1_MAIN,G6,2016-09-12,77.05,6288105.5,8157259,7489.5,0.52,1445479.5,17.72016188280892\n",
    "G6R3_NF_12SEP2016_LD1,G6R3_NF,G6,2016-09-12,79.43,6106488.5,7685321,10643.0,0.92,1148741.0,14.94720910161072\n",
    "G6R4_MAIN_12SEP2016_LD1,G6R4_MAIN,G6,2016-09-12,69.66,6753666.0,9689783,17239.0,0.7,2434399.0,25.123359315683334\n",
    "G6R3_MAIN_12SEP2016_LD1,G6R3_MAIN,G6,2016-09-12,73.95,5500075.5,7433032,19400.0,1.23,1554624.0,20.915072072876857\n",
    "G6R1_NF_12SEP2016_LD1,G6R1_NF,G6,2016-09-12,72.38,6529576.0,9016718,14827.0,0.71,2073333.0,22.9943201062737\n",
    "G6R4_NF_12SEP2016_LD1,G6R4_NF,G6,2016-09-12,79.05,7784763.0,9844371,14631.0,0.93,1554474.0,15.790485750689404\n",
    "G5R4_MAIN_12SEP2016_LD1,G5R4_MAIN,G5,2016-09-12,35.27,2683194.5,7595791,150212.5,3.14,4630635.5,60.96317684359667\n",
    "G5R1_MAIN_03OCT2016_LD1,G5R1_MAIN,G5,2016-10-03,19.82,1693715.0,8528146,79877.0,1.18,6673938.0,78.25778311018597\n",
    "G5R2_NF_03OCT2016_LD1,G5R2_NF,G5,2016-10-03,29.27,2608174.0,8899698,76623.0,1.24,6085159.0,68.3748931705323\n",
    "G5R2_MAIN_03OCT2016_LD1,G5R2_MAIN,G5,2016-10-03,23.52,2090804.0,8873165,84764.0,1.27,6589822.0,74.26687095303649\n",
    "G5R4_MAIN_03OCT2016_LD1,G5R4_MAIN,G5,2016-10-03,24.99,2891414.5,11550331,81760.5,0.96,8431181.5,72.99515052858658\n",
    "G5R3_MAIN_03OCT2016_LD1,G5R3_MAIN,G5,2016-10-03,16.95,1343183.5,7906366,59621.5,0.92,6434206.5,81.38007398089084\n",
    "G5R1_NF_03OCT2016_LD1,G5R1_NF,G5,2016-10-03,32.84,2786797.5,8471997,80805.0,1.46,5468798.0,64.55146289593823\n",
    "G5R3_NF_03OCT2016_LD1,G5R3_NF,G5,2016-10-03,55.85,5278523.5,9441689,50607.5,1.29,3857580.5,40.85689011785921\n",
    "G5R4_NF_03OCT2016_LD1,G5R4_NF,G5,2016-10-03,10.73,842740.5,7838486,87026.0,1.25,6865762.0,87.59040967860375\n",
    "G6R3_NF_03OCT2016_LD1,G6R3_NF,G6,2016-10-03,82.11,7555021.5,9198716,8543.0,0.76,1117304.0,12.146303897196088\n",
    "G6R4_NF_03OCT2016_LD1,G6R4_NF,G6,2016-10-03,67.6,6157032.5,9101083,23241.5,0.92,2515567.5,27.640309400540573\n",
    "G6R4_MAIN_03OCT2016_LD1,G6R4_MAIN,G6,2016-10-03,72.75,7814311.5,10735815,10646.0,0.44,2383477.0,22.201174293707556\n",
    "G6R3_MAIN_03OCT2016_LD1,G6R3_MAIN,G6,2016-10-03,77.79,5921213.0,7608735,12459.5,0.97,1268258.5,16.668454085994583\n",
    "G6R1_MAIN_03OCT2016_LD1,G6R1_MAIN,G6,2016-10-03,82.58,7837373.5,9488885,3797.5,0.33,1131924.5,11.92895161022607\n",
    "G6R1_NF_03OCT2016_LD1,G6R1_NF,G6,2016-10-03,75.2,6498248.0,8637188,14738.0,0.84,1729924.0,20.02878714692791\n",
    "G6R2_NF_03OCT2016_LD1,G6R2_NF,G6,2016-10-03,75.81,6956577.0,9172480,15094.0,0.85,1769670.0,19.293255477253698\n",
    "G6R2_MAIN_03OCT2016_LD1,G6R2_MAIN,G6,2016-10-03,70.74,6526185.0,9219877,12187.0,0.54,2247638.0,24.378177713216783\n",
    "G6R4_MAIN_07NOV2016_LD1,G6R4_MAIN,G6,2016-11-07,26.19,1990040.0,7583071,32984.0,0.6,5425824.0,71.55180269313053\n",
    "G6R2_NF_07NOV2016_LD1,G6R2_NF,G6,2016-11-07,28.51,2105794.0,7370105,52325.0,1.02,5067900.0,68.76292807225948\n",
    "G6R1_MAIN_07NOV2016_LD1,G6R1_MAIN,G6,2016-11-07,18.58,1286592.0,6905794,43214.0,0.78,5486478.0,79.44746107399091\n",
    "G6R1_NF_07NOV2016_LD1,G6R1_NF,G6,2016-11-07,24.89,1710699.0,6858763,53420.0,1.06,4982370.0,72.642399219801\n",
    "G6R3_MAIN_07NOV2016_LD1,G6R3_MAIN,G6,2016-11-07,51.38,4029039.5,7832562,33285.0,0.94,3519722.0,44.937046141479634\n",
    "G6R2_MAIN_07NOV2016_LD1,G6R2_MAIN,G6,2016-11-07,52.97,4631958.5,8733100,29281.0,0.77,3762898.0,43.08776952055971\n",
    "G6R4_NF_07NOV2016_LD1,G6R4_NF,G6,2016-11-07,30.18,2318829.5,7669227,54941.5,1.06,5137738.5,66.991608150339\n",
    "G6R3_NF_07NOV2016_LD1,G6R3_NF,G6,2016-11-07,41.43,8686844.5,20875176,138953.0,1.2,11428845.0,54.7484964917182\"\"\"\n",
    "\n",
    "header = [\"sample_id\",\"plot_name\",\"type\",\"Date\",\"percPlantAligned\",\"Plant Reads\",\"Total\",\"Fungal Reads\",\"percFungalAligned\",\"Remaining\",\"percRemaining\"]\n",
    "data = []\n",
    "for line in dataStr.split(\"\\n\"):\n",
    "    rec = line.split(\",\")\n",
    "    data.append(rec)\n",
    "\n",
    "metadataG= DataFrame(data,columns=header)\n",
    "metadataG.set_index(\"sample_id\",inplace=True)\n",
    "\n",
    "@interact\n",
    "def graph(Host_Plant=[\"Switchgrass & Miscanthus\",\"Switchgrass\",\"Miscanthus\"],By=[\"Percentage\",\"Count\"]):\n",
    "    G5Data = metadataG[metadataG['type'] == 'G5']\n",
    "    G6Data = metadataG[metadataG['type'] == 'G6']\n",
    "    if By == \"Percentage\":\n",
    "        if (Host_Plant == \"Switchgrass & Miscanthus\"):\n",
    "            ax = metadataG.groupby(['Date','plot_name']).sum()['percPlantAligned'].unstack().iplot(title=\"% Reads Aligning to Respective Plant Host Assembly G5=Switchgrass G6=Miscanthus\");\n",
    "        elif Host_Plant == \"Switchgrass\":\n",
    "            ax = G5Data.groupby(['Date','plot_name']).sum()['percPlantAligned'].unstack().iplot(title=\"% Reads Aligning to Switchgrass Assembly\",fontsize=12);\n",
    "        else: # \"Miscanthus\":\n",
    "            ax = G6Data.groupby(['Date','plot_name']).sum()['percPlantAligned'].unstack().iplot(title=\"% Reads Aligning to Micanthus Assembly\",fontsize=12);\n",
    "        ylim = 100\n",
    "        ylab = \"% Reads\"\n",
    "    else:\n",
    "        colors = [\"#74C476\", \"#f26d07\",\"#794955\"]\n",
    "        if (Host_Plant == \"Switchgrass & Miscanthus\"):\n",
    "            ax = metadataG.loc[:,['Plant Reads','Fungal Reads', 'Remaining']].iplot(kind='bar', barmode = 'stack', color=colors, title=\"Reads Counts By Source\")\n",
    "        elif Host_Plant == \"Switchgrass\":\n",
    "            ax = G5Data.loc[:,['Plant Reads','Fungal Reads', 'Remaining']].iplot(kind='bar', barmode = 'stack', color=colors, title=\"Switchgrass Reads Counts\")\n",
    "        else:\n",
    "            ax = G5Data.loc[:,['Plant Reads','Fungal Reads', 'Remaining']].iplot(kind='bar', barmode = 'stack', color=colors, title=\"Miscanthus Reads Counts By Source\")\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What reads are fungal reads?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33e2c026204046c99255b0528f54c4ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "interactive(children=(Dropdown(description='Samples', options=('Switchgrass', 'Miscanthus', 'Switchgrass & Mis…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#HIDDEN\n",
    "@interact\n",
    "def fungalAlign(Samples=[\"Switchgrass\", \"Miscanthus\", \"Switchgrass & Miscanthus\"]):\n",
    "    if Samples == \"Switchgrass\":\n",
    "        G5Data = metadataG[metadataG['type'] == 'G5']\n",
    "        return G5Data.groupby(['Date','plot_name']).sum()['percFungalAligned'].unstack().iplot(yTitle=\"Percentage of Reads\",title=\"Switchgrass Samples Perctages of Reads Aligning to Combined Fungal Assemblies\");\n",
    "    elif Samples == \"Miscanthus\":\n",
    "        G6Data = metadataG[metadataG['type'] == 'G6']\n",
    "        return G6Data.groupby(['Date','plot_name']).sum()['percFungalAligned'].unstack().iplot(yTitle=\"Percentage of Reads\",title=\"Miscanthus Samples Perctages of Reads Aligning to Combined Fungal Assemblies\");\n",
    "    else:\n",
    "        return metadataG.groupby(['Date','plot_name']).sum()['percFungalAligned'].unstack().iplot(yTitle=\"Percentage of Reads\",title=\"All Samples Perctages of Reads Aligning to Combined Fungal Assemblies\");"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h4><a id=\"aConc\">Alignment &amp; Annotation Observations</a></h4>\n",
    "<ol>\n",
    "    <li>The percentage of plant reads sequenced is high thoughout the season and tapers off towards the end of the season. This may be do to senescence of plant cells.</li>\n",
    "    <li>Alignment to the 6 selected fungal assemblies is low, but picks up during the warmer months. Overall, reads aligning to the combined fungal assembly were &lt;10% of the non-plant reads.</li>\n",
    "    <li>The overlap between which contigs have both a prokaryotic and a eukaryotic annotation suggest either shared gene similarity or eukarotic contamination.</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 id=\"anno\">Assembly and MAGs</h2>\n",
    "<p>The fittered reads (All-(plant+fungal)) were used to generate a metagenomic assembly. The assembly was then annotated with a set of prokaryotic peptides and a set of eukaryotic peptides. The diagram below shows numbers for contigs that had only prokayotic, both, and only eukarotic respectively.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAD5CAYAAABBAHxqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2deZwdZZW/n9N7d3rNvi8QCCGEENYAAkECioAwIjIsorjM4DLCqD9UBmiaRRFHBzfGUUEQEFkEVESEAAkEEvYkEEII2ZPuTtJJel9ud9/z++OtS27a7r6d7ltd91afJ5/6pG9V3apTb9361vuec973FVXFMAzD6JmMoA0wDMNIdUwoDcMwEmBCaRiGkQATSsMwjASYUBqGYSTAhNIwDCMBJpQhQ0RWicj8gM79dxH5XC/b7xaRmwfTpjAgIpeIyNNB2zGUMaEcBETkYhF5XUQaRaTKE5SPJOG4/yQ8qjpLVRcN9Nj9QVXPVNV7PNs+LyJLgrCjJ0RkmHcPnhzEc+5XOYjIVBFREcmKrVPV+1X1DH8sNPqCCaXPiMg3gduB7wNjgMnAHcC5Qdo1RPk00AacISLjgjbGSCNU1RafFqAEaAQu6GWfXJyQVnrL7UCut20+sBX4FrADqAIu97b9G9AORLxz/NVbvxFY4P2dD9wD7AFWA1cDW+PO/R1gG9AArAFO68a+aUAtkOF9/i2wI277fcBV3t+LgC8BM4FWoNOzrdbbfjfwS+Bv3jlfAQ7spWweBqqBOuAFYFbctv06lved54BbgDeBb3fZthH4NrDSO9+DQF6i+xB3n38P7AQ2AdfiKiE9lcNZwFtAPbAFuCHuWJsB9fZvBI4HPg8sidvnBOA1z87XgBPiti0CbgJe8srlaWBk0M9Cui+BGxDmBfg40AFk9bLPjcAyYDQwCngZuMnbNt/7/o1ANvAJoBko87bfDdzc5Xgb2SuUtwKLgTJgoicCW71tM7yHdLz3eWpPQuM9vEd5f68B1gMz47bN9f5eBHzJ+3ufhzvO3t3AsUAWcD/wx17K5gtAEXtfJssHcKzJQBQ4FCd4K7spt1eB8cBw3Ivlij7eh98Df/ZsnQq8D3yxl3KYD8zGienhwHbgvLj7oPG/mfhjeLbtAT7rXfdF3ucRcfdgHXAw7kW5CLg16Gch3RdrevvLCKBGVTt62ecS4EZV3aGqO4EK3EMQo93b3q6qT+JqGTP6eP7PAN9X1T2quhX4Wdy2TpwAHSoi2aq6UVXX9XCcxcApIjLW+/yI93kaUAys6KM9AI+q6qtemdwPHNHTjqp6l6o2qGobcAMwR0RK+nMs4DKcOL4LPADMEpG5Xfb5mapWqupu4K9djtftfRCRTOBC4HuerRuBH7PvPex6XYtU9W1VjarqSs+eU3qxPZ6zgLWqeq+qdqjqA8B7wDlx+/xOVd9X1RbgIXovF6MPmFD6yy5gZLxjvhvG45prMTZ56z48RhehbQYK+3j+8bhaY4wP/1bVD4CrcAK0Q0T+KCLj6Z7FuFrQybgm8CLcg30K8KKqRvtoD7imdIwer0VEMkXkVhFZJyL1uBofwMj9PZbHZTgxRVUrcdfUNULf2/F6ug8jgRz++R5O6MkQETlORJ4XkZ0iUgdc0eW6eqPr76W78+1PuRh9wITSX5bifFTn9bJPJTAl7vNkb11fSDT0UxWuyR1j0j5fVv2Dqn7EO78CP+zhOIuBk3BiuRhYApyIE8rF/bQtERfjAl4LcD7Aqd562d8DicgJwEHA90SkWkSqgeOAixK8xPpCDa622fUebvP+7q4c/gD8BZikqiXAr9h7XYnKrevvpev5DB8wofQRVa0Drgd+KSLniUiBiGSLyJkicpu32wPAtSIySkRGevvf18dTbAcO6GX7QzhxKBORCcDXYxtEZIaIfFREcnFi3oJrjnd3HWu97ZcCL6hqvXfu8+lZKLcDE0Ukp4/X0pUiXIR6F1CAyxroL58DnsH5J4/wlsO84545gOOiqp24cr5FRIpEZArwTfbew+7KoQjYraqtInIs7qUQYyfOl9rTfX0SONhLOcsSkQu963piINdh9I4Jpc+o6k9wD861uIdgC06wHvd2uRl4HRdoeRsXke1rUvadOB9jrYg83s32G3HR2g3AQpxvsc3blosL9tTgmmqjgWt6OddiXPNzc9xnwUVvu+M5YBVQLSI1fbyeeH6Pa1JuA97FBbz2GxHJw/lqf66q1XHLBuBe/rn53R/+A2jCBbmW4GqMd3nbuiuHrwI3ikgD7sX4UOxAqtqMi8y/5N3XefEnUtVdwNm4gNQuXCbD2aranzI2+oio2sC9QwUR+Qrwr6ra18CBYRhYjTLUiMg4ETlRRDJEZAauFvJY0HYZRroxUEe2kdrkAP/H3qTxP+J6BRmGsR9Y09swDCMB1vQ2DMNIgAmlYRhGAkwoDcMwEmBCaRiGkQATSsMwjASYUBqGYSTAhNIwDCMBJpSGYRgJMKE0DMNIgAmlYRhGAkwoDcMwEmBCaRiGkQATSsMwjASYUBqGYSTAhNIwDCMBJpSGYRgJMKE0DMNIgAmlYRhGAkwoDcMwEmBCaRiGkQCbhdEw+oFUiACZuMqGxG1SoF3Lbda+MGGzMBpGHJ4AFgHFQIn3fz6QB+R6/+cB2QkO1Q5E4pZWoBFo8JZGoEHLtTX5V2EkGxNKY8giFZILjIpbhgOFDK5LqhXYBdTE/V9nNdLUwoQyLIhkAaW4GlC2t+Tg3Cs57G0ixpqJgmsmtgDNQJP3v/tbtXMwzR8MpEIKgYnAeJwwlgRrUY90ANXANqASqDHhDBYTynRDJB8ow4li/FKY5DO1ES+crqm4A9iOaluSz+ULUiGZwDhgkreUBmtRv2kDqoCtwEYt1+aA7RlymFCmMiK5uAd8PHvFMTdQmxx7cDWe7UA1qvUB2/MhUiEZuDKbDkwhfAFLxZX9emCDiebgYEKZaogMByZ7yxj2jaimKs3ECyfsQjU6mAZIhYzDieMBpMbLZDBQXJm/D3yg5doRsD2hxYQyaJxvcTx7xTHZTeggaAXWAe+jutOvk0iF5AAzgFk43+xQJgKsBVZrue4O2piwYUIZBCKZuNrPNJxIhq15GM8eXI1nLZqcZqJUSClwGHAQidN0hiLVwCpgvQWBkoMJ5WAiUoCr/czE5eINJRQXjFgDbOpPVF0qZDRwFM4HaSSmDlgOrNXywXWFhA0TysFAZCQwGzgQ6zYKrpm4DliD6o5EO0uFjAKOxgSyvzTiBHONlocv7WswMKH0CxEBpuIEcmywxqQ0VcAbqFZ23SAVMgInkFMG3apw0gS8DrxvTfL9w4Qy2ciHAYbDcF3hjL5RBbyOapVUSAFwLHBwwDaFld3AUi3XbUEbki6YUCYLkQzgUJwPbaikpySVqKCvzWPLF09i+KqCUET/U52NOMFsCNqQVMeEMhmITAKOJ317fgTO7pG0vXU80lJIDsCKAlp+OYasqhyLavtMJ645vtKa4z1jQjkQREpxAmlBhn7Snk3n20fTVjmFfGTf5PoOiD5RRsvvR5LfnmFBMJ/ZASzWct0TtCGpiAllf3DN7LnAEbgxCY1+UDWRlhXHkd2R03se6a5MIj8fS/SNwiGXUjXYdAJvAissnWhfTCj3F5GxwMlYM3tArJpL04ZDGLY/33l9GM0/HUtObVaoE/RTgRpgoZanTh/+oDGh7Csumj0POCRoU9KZSA6dr51MZM8o8vvz/Wah8/ZxRJYW9e/7Rp+J4JriG4I2JBUwoewLzhd5BlaLHBC1ZbS9dgoZbfkDD9AsKqL5Z2PJM9+l77wDLBvqTXETykSITAPmY32KB8SmA2l+5yjyNDN5wrYji8jNE2BDnouUG76xA9cUbwzakKAwoewJ17PmGFzAxugnUUFXzKNl21QK/Dh+B0TvGk3rX8v8Ob7xIc3AU1quNUEbEgQmlN3hBsw9DTdtgNFPWvPpWHYqHY0l/kerny2m+adjyVdJi/E705UO4Fkt101BGzLYmFB2xQ2cewY2vuGA2D2StldPITNR6k8yeT+PlusmktOcaSlbPqLAy1quq4I2ZDAxoYxHZDou9cfSTwbAnhG0Lf0oWdGswResXZlErpsEW3LNb+kzK7VclwVtxGBhQhlDZB5weNBmpDu1w2l7+bRgRDJGq9B5/UQ6VhdYn3ufeVfLdUnQRgwGJpQAIh/BDWhhDIDaMtqWLiCrM0CRjBERojdPIPLWMOvN4zOrtVxfDNoIvzGhFDkBNySaMQDqSom8vIDMzuzgRTJGB0RvG0+bJaf7zntari8EbYSfDG2hFDkOmBO0GelOfQmRlxeQMZiBm77SCfrzsbQ8W2LpQz4TarEcur0aRI7GRHLANBSnrkgCZIJ8o5r8U+uw+a/95RCpkOOCNsIvhqZQihwJHBm0GelOYxGRl05PXZGMkQFyZTX5xzTSErQtIWeOVMisoI3wg6EnlCJzcPOwGAOgaRjt6SCSMTJBvreN3JnNtAVtS8g5QSpkatBGJJuhJZQihwGhbR4MFp0ZRF85lWh7bnqIZIxsyLhxK1lT2ogEbUuIEeA0qZAxQRuSTIaOUIocApwQtBlhYMU8WpuL0jNHMU/J/MFmMoZ30BG0LSEmE/iYVEho5j0aGkIpMhr4SNBmhIHNB9BcOSW9I8hFUbJu3kJndpQhPXSYz+QBp0uFhEJjQnERveIG3D2NoXCtPtNQTOSdo8ORwD0pQu63qmgN2o6QM4qQtOKGgnjMx+bXHjAdmURfPQWiSRxPMmhObKTgU7ssbchnDpUKOShoIwZKaH703eKCN1ODNiMMLD+e1thUsmHishry5zRZzdJnTpIKGR60EQMhvEIpMgo3x40xQDZOp7l6Unr7JXsiE+S7lWQVdtIZtC0hJgv4aDr7K9PW8F5xfskFhPX6BpH6EiKrjgyHX7InCqNkXV1p+ZU+Mxw4Kmgj+ktYhWQ+5pccMB2ZRF+dD8mc5yZVmdtMwZl7zF/pM3OkQkYFbUR/CN8DYH7JpLHqKFpaC8Lnl+yJL+4kd0yE9qDtCDEZwHypkJQZYaqvhEsoRUZgfsmkUF9CZMu0cPoleyJXybx2G52iDOEhtXynjDRsgodLKOEkwndNgbD8eKJkDL2JuqZGyDt3jw2e4TOHS4WUBW3E/hAeURGZAYwO2owwsPkAmuvLwh3A6Y1La8gttS6OfpIBnBi0EftDOITSRbmPDdqMMNCRRfTduUPHL9kduUrmf1TbwBk+M14q5ICgjegr4RBKN2yaDfefBN47nJZ0GTrNT45tomBWsyWi+8xx6RLYSX+hFCnDJgZLCq35dGyabi+cGN+oRiyw4ytFwOygjegL6S+UcAzhuI7AWTWXyFDImewr49vJXVBngR2fmSMVkh20EYlI74dCZAyWM5kUGoqJVE2y2mRXLqkhy2qVvpJLGsyCmt5CaaOVJ413jqJzKKYDJWJEJzkfr7Vapc/MTvVaZfoKpchkYGzQZoSBhmIiu8ZabbInLt5FdqbVKv0kD0jpScnSVyhtgrCksW6mddvrjdJOss+2JHS/OVwqJGWzLdJTKN3UDiODNiMMdGQSrZo8dJPL+8r5u8k2X6Wv5AGHBG1ET6SnUFo6UNLYOo3WzizSIpctSMo6yT6h0fIqfSZln+v0E0rXCydtMvpTnY0Hm0j2lU/vsmCXz5RKhYwP2ojuSD+hhIPBeo4kg9rhtDWWpOe0s0EwvY08mxPcd1IyqJOOQjkzaAPCwrqZNl3r/nLhLhssw2emSIUMC9qIrqSXUIqMw41nZwyQ9mw6qydYbXJ/mddAXoHNr+MnGaRgUCe9hNJqk0lj03TarLvi/pMNGafU2/w6PnNg0AZ0JX0eFJE8LIiTNDZNNz9vf1lQn0bPTXpSKhUyImgj4kmnGz6D9LI3Zdk1KpxzdA8W01vJHW4D+/pNStUq00l4Us5vka5UTrYgzkDIADmtzqLfPmNCud+IlAAlQZsRFmrGWrN7oJxab/mnPlMkFZIyU7ukh1DChKANCAvt2XQ2FZHSI7WkA5Mi1vweBKYEbUCMdBHKlMzWT0d2jCOCWA+TZHBcgzW/fSZlKkipL5Qiggll0tg+wQZ2SBbzGoO2IPSMkgpJiaBj6gsljAAb3SZZ7B5tze5kcUgLuTaikK8IKVKrTAehTImCCgPNBbS3FphQJosCJfPgVmt++0xKPP8mlEOIHRMs+JBs5jVamfrMxKANgFQXSpEMbLqHpLF9vDUTk82hzSn+DKU/xVIhgY9JkOo3eQw2pFrS2DPSeuMkmykRc2UMAqOCNiDVhdKa3UmivoRIR469dJLNsChZo9ptziGfCXzal1QXSmt2J4m64TY0mF8c2mJ+Sp+xGmUCioM2ICw0FVn/br84tMXK1mesRtkjLpCTciMdpytNhdYbxy+mt6bwcxQOioJOPE/lG1wI9nAni+bClL7Xac2odvP9DgKBti5T+eGxZncSaS2w0W78oriTrEzroeM3JpQ9UBS0AWFBQdtyrdbjF5kgo9stoOMzgeqBCeUQoKWADjLMjeEnEyKWVeAzVqPsARPKJNFUbA+x30yIWOTbZ6xG2QMmlEmi0VKDfGdsu/kofaYwyJOnslBaMCdJNBXZQ+w3RZ3m2vAZSw/6J0SysTEok0ZTkT3EflNkzg2/CXRgjNQUSks0TyqR3JS9z6GhMGovI5/JlAoJLHMjVR8gy/lLIlErTd8Z1pmyz1KYCKz5naq5dUn/0b0AZZ+FyxvdtLd6OrzwR3jum3DUPXDObhj7G/jBl2ATQB1kngqXboYpAno1PPj/4H2AyfCtRijJwo0a8w+4fS40APwnHPV7OAdgAmxZCXc+B8Mvgq8oZHRC5lnw3O/hBYCr4Oj74RNRyDgc3n4e/pTsa1efJxO76ik+t6We2bmZNPzhfCoAbljEOat28JHcLBoBzjyIxy6ZzTvLqxlRvoiKohy2A4wpZP2Pz+B+gMv/zLda2inJynDlWnEqtx9YRsN3FvKZbfXMAOiIktPWSdFjF3JV7PzbG8n7+pNUTC1j+Y9O5wGAZ9Yx+e7lfL5DyZlSwtu3LuDBDB9LoSDqs1BupIzHuJyI+/1yAC9wAc99uP1BTmc1n+YrfIsxNLKasTzJ52hkMofwOBfyzIf7LmIWy7gQJYOpLOEingJgHSN4jC/TwTCK2MwXuYs8OrmLz1Djyp8oObRTxHVcxQaG8whfQclAyeRgnuNf3O/aJ3KBZh+P3yOpKpRJrwPlQvQGeORy2LwJco+Aax+G1SfCtpPhf78Bl8bv/1U4CaAGbnwLij4O37gKvp+NC4zcBnfGRDXGEzD6fjjzZbhtBjS/5UXuj4K69+CHZdCxBXJnQflSWFEMHXfC+UvgljnQOBc+/wM45HvwXjKvPepzXWf+VF4uzOH5u97i8vj1R45j4fdOintAPYZls/O+T3FTd8f6wlzuPOPAfcv1hwt4KPb3j17m1G31TI7f/uOlnDum0L3EYty7kksuns19Zx7E+n9/gm/88R1mXTybVf25vr6QrT43vbOIMp9HmMtmasnlV1zLKlYziyo2UkYVh5LD7g/3L6WJ+fyRVczd5zgdCEu5iAu4nYns4WdcwypWMIsqnuJ8ZrGQM3md33AJT/IRPsVivrC3/HmEU9nllf846vg6PySfDurI5ZeUczQrmESdb6UQEKnaXEi6XcdB3eWwGWAKtI2Gqveh9HyoPg9Xu4lnPYyb5wnWXGjIh+a7Eswz/GM46WxYNMN768VqmSXQWYbruVEPWer1YX8ZRo6EHXNwta55sPovcGQyrxv8r1GedwhrRxbQ5Oc5Yryzg2OPn8Srsc/PrGNyUzvFh4zk3di6NTWUtEfJO+tg1mcIHD2epa9XdhGMJOO7g3Iidcx1v19KaWMYVeyiFIAn+Ayn8ieIy24YRwNHsYmMLjm0bzKNfHYynRry6GQSr7GSOUSBPcxgAW8CMIelbOaIf7JjE8cy0yv/PDrJ93oktZGF/8UQmB94yAhlPAthRCVMuhg29LTPDNi6GOY0Q8bTMKIKpqyFstj278LnRsN1Z8NZsV9iFYzeAGMmwtXj4bsVMCu2/wtQNgquPwJu/ST843ioOxl27oSxC2FEM2Qsgrm74s6RLDSgu/xWNade8ijXX/UUn9taT0FsfXM7Iy99lGsv/zPf/vN7TI//zj3L+dylj3JdxWLOinZJalq5neFNEUb8yyHuBdYRRf7wDhd85Wgeid9vUx2lBdnUxj6PHcaexognKj4hftco41nPCBqYxGw2sJDDyaeWOWzt03f3UEpBXM2ziD00UcZOCsmkhWwv53Y0e2jrUmYbGE4rIzghrsWzkTJu43p+xa3M4B8+1iYhQL1K1aa3b2yB3EvgiivgoWnQ2tN+d8BLH4dxU+G/ymDXJFiXjfsRPQB3ngi1myD3VLjiazDvV7CsEzKrYPRq+PFSKP0UXH0x3HAQtJwMe3bCjUuh5FPw1TfgjaOg4Uq4/zL4soBOh3XbU2CQ0mRw6eEsmlzCExkCFYs4979f5oLbP84904dT97Mz+e7kEpqeWcfkX7/JV+dN5IYxhbRefQJ3zhxF7Y4mcq95liv+93Xmfe0YlsWO+dc1HDOtjDdzMl3N6fZlnDJ9OG8fNpo9izbuPbd2kzUqEpJc0jpy+RNXcDQPkU2U5XyCL/DTPn+/O0EXuh/So2uZvcIxjOVNsuLWT2UPV3MjWyjhQb5KJW8w3rWkwkSq1ih96UnSAJmnwBUnwys/grd627cAoi/AQzvgpjVwRwsUHAk7AE7E1VamQNsCeHUlTAMYDntOhuVF0HkG7BoJ1YtgdPxxj4e6cVD5BzgI4BZYWQm3boMfToPt47xzJBMJoF/O9OE05GSiWRno+Yfy4o4mpgIU5tAxucQ1008/kM2FOexcsZ0xADNHuXIdPYy2I8by6sZaV64xVtdwzClT9ja7N9Vx4PJqTv3Mw3z/uQ18+oPdzPuv5/iXqaXUNrfvrQ1VN1E2LNvXmg46GELcRia/4wqm8Apn8BYbGEUrI/kV1/F9vk+EMu7kv9jWS2eN4eyhmeEffm6gjAJqGU0jneTT7mnCDsrI6VJmWziGw/aW/z5Moo4iKnnb/a59IrAeZqkqlElP3+0EToLLJkDVw7Aw0f7VkFPppSPcAjMzoPMCqGqGjLe97lQNkLkEDp8O2wA+Actfw0UH34bCXTDmBKh5EUp34CahWgMFG2D60Z5fNBbwWQMFT8L8/4QXk33tEsAQYGtqKIn9/Y8POKIsn0qADXsojHi9WN6sYmRjG6NnjmRnWwcZG2tduba0k/nuTg4fV+jKFWDZVsZEOin4xEGsj637+Znc+fAFfO+hC7jmo9N4ZPpwlt3yUR6bMZK6rAzanlzLtKjC65Ucf9R4lvt5vb4XcBT4HZdRTBWf8X6/s9nGtXyba7x/Oezhi9zCBOp7PM5cNtLCaNYxglYy2cIxzGYFGUAZa1jo+chXcDyT48rsPcbQTgHH7C1/NlFKkze5Wg0F7GE64//Z359EAmsVpGrTO+lvjl/A9BUwbyRsGw3XAXwFHmuBrDvgohYo/Ab8x/dhy3r46TtQ9Bm4UkCLofYeuAugFrIWwJVRyIxCxkxY/WtP3K6FVc/CoSPgBoHoF+CRWdB0C8z8FFyAu9FyLjx9kSeul8GFld7cxZfCE5/0oUaZ4fN7+OtP8qXqRg6OdFJ4wcP88KTJ/OWD3czY1cxEBApz2PXNedwH8PxGDlq4nnNF6BSIfnIG908qoXlXMznXPc+VUSVTlYyJxaz++rF7XxpPfcCxB43gtb6m+Hz2cO6/ezmfv2cF2ZNLWHXRYbzjz9U72v2uUb7KdKqZRwHbuM39fjmGxzi1h+vaRjF38190koeg3MICvko5ZbQyjwd4mKtQhCm8xGFUAfAxHuVxvsxyzqOYzZzJSx8e73WOZQKv7VO12sQ4Htz7u+YQnmb23pebDwQ2lJ1odw6doBEZAZwftBlhYfGZtDaUWpdQP9mSQ+Sr02w6YJ+5T8s1kDzKIdP0HspkWGn6TlOGjdA0CLQFdeJUFcrGoA0IE7mt9hD7TWOmlbHPdGq5BvbKT02hVO0AWoI2IywUNIYkNSaFaUzNJylMBFabhFQVSkfocrGCorDeRrbxm/pMexn5jAllD/Sc4mDsF8MaUvo+h4LqHHsZ+cygdJHtiVR+gKxGmSSGNdiwdX6zNSeln6UwEGjFKZVvrgllkshvJouoNQ39pDI7ZXOSw4IJZQ+YUCYJAcltdWM8GsmnE3RHttXafSZQPTChHCLkt1huql/UZtLu91B2htUoe6KRAPt2ho2CBitLv9iZbS+hQcCEsltUo1jiedIYZrmUvvFBniWb+0y9lmugrqPUFUqHpQgliWEN1jT0i3fzU/45SndqgjYg1W9wVdAGhIWS3RaV9YtVBd5QY4Zf7AzagFQXysqgDQgLRfXkZLcFN0xVWGnMoGN3lr2EfMaEMgE7wNJaksXwnUSCtiFsbMi13+cgYE3vXnEBHWt+J4nRVj9POqvzLZDjM7VaroG/4FNbKB1+jpg8pBhTaQPLJptlhdbs9pmUeP5NKIcQeS1k5TVZ8ztZNGXQsTaf3KDtCDl9m4bXZ1JfKFV3Y2NTJo2ROyygkyxW5dtLx2dSxvWW+kLpMO9akhi9zfIpk8WrhUFbEHp2pIJ/EtJHKK35nSRGVZNjIwklh1cKzefrMynz3JtQDjGy28ksbAh2tOgwsDGH1lrLn/SbTUEbECM9hFK1AdgTtBlhYWS1DeIwUJ4rsbQgn6nTcg08fzJGegil472gDQgLEzZZTWggdII+W2zRbp9ZF7QB8aSTUL4PFrFNBmW7yC2w5ne/WZNHa32WDdTrMyaU/UK1DVgftBlhYdr71vzuLwtLLBjmM7u1XFPK1ZY+Qul4N2gDwsKkdeRldJhY7i8RIfpCMXlB2xFyPgjagK6kl1Cq7gB2BW1GGMjqJGPcVmt+7y9Limhty0iz5ya9iAJrgjaiK+l4w1cHbUBYOPBdC+rsLw8NtzLzmfVarinXEy8dhXItNvRaUiiuI6eoltag7UgXVufRui3Xksx9JiXda+knlKrtpKAPI12ZtsbyAfvKIyOCtiD07NZyrQ7aiO5IP6F0pORbJ+yEAu4AABLASURBVB2ZuIH8zHYL6iSiJovIq4UWxPGZVUEb0BPpKZSqu4DtQZsRBjIUmbDRgjqJeGi45fD6TDPOrZaSpKdQOl4P2oCwcOB7ZKOWG9gTNVlEniolP2g7Qs4KLdeUfRmlr1CqbiNFBvVMd4Y1kj2qysb87In7RtKhYsPT+UgLKZ7Nkr5C6Xg1aAPCwmGvk23Dr/0z27OIPFtCQdB2hJyUrk1Cugulag0p1ic0XRnWRPbEjVar7Mo9oyzQ5TMtpEFwNr2F0vEaWIpLMpi5nFzr1riXzTm0vlhsvkmfeSvVa5MQBqFUrQdWBm1GGMhtI3Pa+5aADhAFvX2s+SV9ppY0qE1CGITS8SbQFLQRYeCgd8jPabWeTy8V0WIzLPrOMi3XtGgNhkMoVTuAZUGbEQayOsmY9cbQzhlsFjrvGGMi6TObtVw3B21EXwmHUAKorsNma0wKEzaTX1ozdAM7d4+irTHTBub1kU7g5aCN2B/CI5SOJdgo6Elh7lKyJDr0gmTv59Hy9zJLB/KZ5Vqu9UEbsT+ESyhVa0mzN1WqMqyR7CkfDK1aZbPQecsEsoO2I+TUAG8FbcT+Ei6hBFB9jxTuM5pOzHyL/ILGodMP/I6xRHbbFLR+EgUWpUsAJ57wCaXjRVzqgTEAMqNkHPc8GUMht3JZIc2LLWfSb97Qct0dtBH9IZxC6aLgCzF/5YAZ1kj24a+Fu1ZZm0n7/4y1KLfP7ACWB21EfwmnUAKo7sb8lUlh4kYKJmygOWg7/KADojdPINpsUW4/iQDPa7mm7VgC4RVKMH9lEpnzCvnD6sNXs/z1aFrXWGK53yzWcq0L2oiBEG6hdCzB/JUDJkORYxeFy1/5bDHNlgrkOyu1XDcEbcRACb9Qujl2zF+ZBIY1kT3nFSJB25EM1uXS+vOxFrzxmSpCMhRi+IUSYv7KxWDjLQ6UCZvJn7QuvfvV78mk/fqJZHfaYLx+0gw8m46pQN0xNIQSYl0cXwjajDAw+zUKCuvSc5Shxgw6vjMZ6rMseOMj7cBTWq6hCQAOHaEEUF2Dy7E0BkCGIvOeJyu3Jb1GGWoTOq+ZRLQqx3rf+EgUeEbLtSZoQ5LJ0BJKANXVWNrQgMlrIevEpyFdhmRrh2j5RNo35JETtC0h5wUt19DNZTX0hBJA9R1sWLYBU9BM9kf+kfpi2Qn6w/G0rSqwebl95nUt1/eDNsIPhqZQAqiuxKa8HTAFzWSf+Axkt6VmVkEHRH84ntZXiizC7TNva7m+GbQRfiGavsnyyUHkaODIoM1Id5oKaX/xY0hHTuoMKhERojdPIPLWMKtJ+sxKLddQt9BMKAFEjgPmBG1GutNYRGTJGWSkgli2Cp3XT6RjdYH1uvGZFVqurwRthN+YUMYQORI4Omgz0p2GYiIvnR6sWDZm0HHNJKIWuPGd5VquoUgoT4QJZTwik4GPgj1gA6G+hMhLp5PZmT34uYrV2bRdM4mMndmWAuQzr4fZJ9kVE8quiBQDHwPKgjYlnakrJfLKqUgkb/AEa0UBLTdNILctYwgHKf0nihvkYkgNNmNC2R0iWcB84ICALUlr2nLpfO0UIrUj/I84/6WUpt+MYZjf5xniRICntVyH3CR+JpS9ITIHOBasT3B/UdBVR9G88WB/RCwiRH8xhtbnS2wUIJ9pAP6u5TokR+IyoUyEyATgNLAUk4GwbTItK44jJ5rEPtZbcmi7aQIZ1iXRd7YCz2m5pmX//mRgQtkXRIqAM4ARQZuSzjQWEXllPrQUDixYFgV9spTm346mwEYA8hUF3gTeTOfRyZOBCWVfcX7Lo4DZDOUeTQOkI5PoGyfRunNc/5rKdZm0/2gcnSssidxvWnC1yG1BG5IKmFDuLyIjgJOBUUGbks6snUXzmsPIJ6NvNcIo6ItFtNwxhlyb38Z3KnEiGZph0gaKCWV/EBFgFnAMmH+sv9SMpvWtE8hsy++9DCuzabt9LFgvG99pB17Rcn03aENSDRPKgSAyDDgOmB60KelKRybR1XNp2XQgBV1rlxEh+uAIWh8eTr6aL9JvtuGGSGsI2pBUxIQyGYiMAU7AmuP9pr6EyFsnEG0oJS8KurSQll+PIWd3VvD9xkNOBFeLXB20IamMCWUyETkYl3dpOX39Q18/hnUXnsbo9XkUB23MEGAN8KqWa0vQhqQ6JpTJRiQTOBAXHbd0or6zEXgT1RqpkBxchsEsLMPAD6qBZVquO4I2JF0wofQTkfE4wZyM9e7piY14Atl1g1RIMU4wp2PllwzqcDXItJ9ne7AxoRwM3EAbhwEzsCg5QBOwFngfTdwlTiqkFDe48oGYYPaH3cByYN1QTxzvLyaUg4lIDnAITjQLA7ZmsOnA1R7fB7bRjx+eVEgZTjCnYU3yvlCD61WzMWhD0h0TyiBweZhTcQ/8RMLdj7waJ47rUY0k44BSIcOAQ4GZhLvs+oMCW4BVWq5bgjYmLJhQBo0TzTE4P+ZkYHiwBiWFBvY2rev9OolUfBg4OwwY6dd50oRmXBR7tZZrY9DGhA0TylRDpBCYhBPNCZAWeYS1uJqjW3wUx56QChmBC/ocyNBxa3TiRvZZC2zUco0GbE9oMaFMZVyq0XicYJbiRl0vJNiARhTYyV5h3I6m1vBbUiFjcaJ5AOFrmsfEcT2wScuT484weseEMt1woxiV4IQzJp6l3rpkDRbRiWvKNXn/NwONwA5gJ6qdSTqPr0iFCK631CScL3g06Rk1b8QNVLEVJ47tAdsz5DChDAvO11nkLTm4NKT4JSaiwl6xUNxwWjExdMKo2jZ4hg8eUiG5uNr5eJyAjiA1o+fNuNr6NmCblg++K8PYFxNKY8giFZKBCwKN8pYyXM18sGbhVFzgqwbYFfvfhjdLPUwoDaMLUiF5QHHcko/zdeYBuXH/Z/LPNVLFDVcWiVvagVacKDZ6/zcAjRaASQ9MKA1jgHi+UACs50s4MaE0DMNIQCo6sg3DMFIKE0rDMIwEmFAahmEkwITSMAwjASaUhmEYCRhUoRSRG0TkvsE8Z18RkVUiMj9oO3pDRCaLSKO4PuCGYQwSCYVSRDaKSIv3gG4Xkd+JG+EmbRGRu0Xk5vh1qjpLVRf141g53gtgrYg0eeV1l4hMTYKdG0VkQZyNm1W1UNOkr7VhhIW+1ijPUdVC3OjSxwDXdt1BHINSQxU3MESq8AjwSeBiXPe3OcAbwGlBGmUYRvLYL2FT1W3A33EDpSIii0TkFhF5CdeR/wARGS8ifxGR3SLygYh8ubtjiUi2iDwgIn/yamXHishSEakVkSoR+YW4qRNi+6uIfE1E1gJrReSXIvLjLsf8q4hc5f0907Ov1mtWf9Jb/2/AJcDVXi35r976D2tvIpIpIteIyDoRaRCRN0RkUjfXsAA4HThXVV9T1Q5VrVPVX6rqnd4+PZaHVxN9SER+751nlYgc7W27Fzcm5V89O68WkaleOWR5+0wTkRe87y70yuQ+b1ueiNwnIru8MnhN3PzjRgri/Va/FLQdXUkXd4+IXCIiT/t2AlXtdcHNc7LA+3sSsAq4yfu8CNiMm1Y0CzdKzWLgDlx/2CNwYxee5u1/A3Afru/s34C7gUxv21HAPO84U4HVwFVxdijwDG4E8Hzc/NmVQIa3fSROrMd4dnwAXIMb4OCjuL61M7x97wZu7uU6/x/wNm4yMMHVEkd0Uza3AosTlF+i8mgFPoHrN/wDYFl3Nnmfp3rlkOV9Xgr8t3eNHwHqgfu8bf8O/BU3x3imV77Fie63Lf1fvPvVguvPHVt+0cfvLgK+lCLXsCBJxxoH3AlUec/fe0AFMGyAx93nORiMpa81ysdFpBZY4j3434/bdreqrlLVDmCs98B+R1VbVXU58Fvgs3H7FwNPAeuAy9Xzt6nqG6q6TF2tbCPwf8ApXez4garuVtUWVX0VN/1mrIn7r8AiVd2OE9xC4FZVjajqc8ATwEV9vN4vAdeq6hp1rFDVXd3sNwL3I+gWrxaaqDyWqOqTXjncixPlhIjIZJwb5HrvGpcAf4nbpd2zb7qqdnrla8N1+c856vzIseXrg3nyVHFLichw3Is8HzheVYtwra9S3Cj0aUVfhfI8VS1V1Smq+lVVbYnbFj+B0Xhgt6o2xK3bhBsDMMY84HCciH3Y0VxEDhaRJ0SkWkTqcWLcdR6UrpMl3QNc6v19KU5oYnZsUd1nZJaudvTGJJyQJ2IX7q3ZE30pj+q4v5uBvD7+2GPHjh+SK7587gX+AfxRRCpF5DYRsalyA6JrxkdXN0qXfceJyEoR+bb3+XIRWe25WNaLyL/H7TtfRLaKyHdEpBr4nYi8IyLnxO2TLSI1InKE9/mTnpun1mvyz/TW98XdM1xcQLdSRPaIyOM9XPI3cbXIS72KD6q6RVWvVNWV3rFO8FxCdd7/J8TZvEhEbhKRl7zrflpEYnrwgvd/rWfn8SLyeRFZEvf9M0RkjXfsO0Rkccy1ISLTvc91Xrk8mOj+JSP4Ej+qRiUwXESK4tZNxg1AGuNpXBPz2S4+s//FVc0PUtViXLO562jUXUfwuA84V0Tm4Gbki920SmCS7Btcircj0UggW+jbW28hcKyITOxhe1/Kozd6s7PKO3ZB3LoP/aiq2q6qFap6KHACcDZwWR/PawSEuGyJxbgm+397q3fg7l8xcDnwPyJyZNzXxuJcUlOAfwN+z94KBDjXTpWqLheRg4EHgKtwY3A+iRPGHFX9LM6VFqsV39aNiffi3DmzcCPG/08Pl7IAeLRLZSX+Oofj3G8/w7V8fgL8TURGxO12sXe9o3HupW9760/2/i/17Fza5dgjcUHW73nHXoN7BmLchNOhMtzI9z/v4Ro+JKlRalXdArwM/MALJhwOfBG4v8t+twF/wIll7C1RhPOxNYrIIcBX+nC+rcBruJv3p7ia7iu40bqv9t6m84FzgD9627fj5lPpid8CN4nIQeI4vMsNjJ1/Ic5v+piIHCUiWSJSJCJXiMgX+loevdCjnaq6CXgduEFcMOx47xoBEJFTRWS254SvxzXFLa3Ifx73amqxpdtgZg8civNVlqvqr2MrVfVvqrrOcwMtxj3kJ8V9L+p9p817Bu4DPiEixd72z7K3tXUh8DdVfUZV23E+7nz2FZJuEZFxwJnAFaq6x3sZL+5h917dUsBZwFpVvddztz2AqyidE7fP71T1fe+aHsL5+PvCJ4BVqvqo5xL8Gfu23NpxL5XxnktsSXcHicePdJ6LcM7WSuAx3A18putOqnoTrga40Hu7fBv3BmkAfgMkrA573APMZu8PAXXzR38Sd1NrcMGUy1T1PW+XO4FDvR9yd02Hn+BuzNM4kbkT92Pqjk/j3soP4nym7wBH42qb0Mfy6IEfANd6dn67m+2XAMfjXAA3ezbEpnEYi3ur1uMCY4txD5DhLzE3VWz5zX589xJca+OR+JUicqaILBOXOVGLE4J4t9ROjZvgTVUrgZeA80WkFPccxF7O43Hun9i+UVwLqi9uqUk4d8+ePuzbF7fUpi7rErml+pq/PZ44N5Tn4tsat/1qXGv1Vc8F8YWER+xvFChVFlw1fDNe9HsoLzihrAjajqG60EvEGJdJ8Wjc53nsm8GwCNeK+jPwMHuzQXI9kfg0kO2texwvawOYD2zt5nwXAc8CXwYWxq2/Dngo7rPgxHm+93kDPWRa4IQvimvyJiqLm4GVPT2XuFruq13WvQx8Pq48vhS37fO4wCe42uA+Ue8u2z8HvNzlGrfQTVYBLtjaigt69ng9ad3X2wtOXAn8VnvwhYQZETlGRA4UkQwR+ThwLnv9tEZqsRw4WVxeYgnOf9aVduACYBhwr+djz8GJ5U6gQ0TOBM7ow/kex3UQuRLns4zxEHCWiJzmPT/fwrVCXva29+buqcLlUd8hImWeW+vk7vbFtcqKgXtEZAqAiEwQkZ94LqgngYNF5GLPZXUhzvXwRB+ubSdOsHtyn/0NmC0i53lBqK/hWlh4dlwQF1fYgxPdXt1SaSuUXqSuFveWuz1gc4JiLO7N24jzw3xFVd8K1CIjFjGOLY8BqHO3PIirZb1BD4Kgzm30KVwA4y6cr/0bOIHbg3NP/aW773Y5TgvwJ2Aa8Gjc+jW4QM/PcW6pc3DBm9j84IncPZ/FCfp7uCDTVT2cfzfO79kOvCIiDbgabh3wgbp0u7NxQr0L1xw+W1Vr+nBtzcAtwEuenfO6bK/BvXBu8459KM6fH3NLHePZ1IgryytVdUNv57SpIAwjpIjI9cDBqnppwp1DjFcz3wpcoqrP9+cYaVujNAyjZ7wA6ReBXyfaN4yIyMdEpFREctmbarisv8czoTSMkOGlJG0B/q6qLyTaP6Qcj+s0EnMvnKf7dpTZL6zpbRiGkQCrURqGYSTAhNIwDCMBJpSGYRgJMKE0DMNIgAmlYRhGAv4/IVQIqm2mGMoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#HIDDEN\n",
    "EukaryoticAnnotations, ProkaryoticAnnotations = set(), set()\n",
    "for line in open(\"annotations/EukAnnotatedContigIDs2.txt\"):  EukaryoticAnnotations.add(line.strip())\n",
    "for line in open(\"annotations/ProkAnnotatedContigIDs.txt\"): ProkaryoticAnnotations.add(line.strip())\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib_venn import venn2\n",
    "# First way to call the 2 group Venn diagram:\n",
    "venn2(subsets = (len(ProkaryoticAnnotations.difference(EukaryoticAnnotations)), len(ProkaryoticAnnotations.intersection(EukaryoticAnnotations)), len(EukaryoticAnnotations.difference(ProkaryoticAnnotations))), set_labels = ('Prokaryotic Contigs','Eukaryotic Contigs'))\n",
    "plt.title('Contigs with an Annotation')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUoAAAD5CAYAAABBAHxqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2deZwdZZW/n9N7d3rNvi8QCCGEENYAAkECioAwIjIsorjM4DLCqD9UBmiaRRFHBzfGUUEQEFkEVESEAAkEEvYkEEII2ZPuTtJJel9ud9/z++OtS27a7r6d7ltd91afJ5/6pG9V3apTb9361vuec973FVXFMAzD6JmMoA0wDMNIdUwoDcMwEmBCaRiGkQATSsMwjASYUBqGYSTAhNIwDCMBJpQhQ0RWicj8gM79dxH5XC/b7xaRmwfTpjAgIpeIyNNB2zGUMaEcBETkYhF5XUQaRaTKE5SPJOG4/yQ8qjpLVRcN9Nj9QVXPVNV7PNs+LyJLgrCjJ0RkmHcPnhzEc+5XOYjIVBFREcmKrVPV+1X1DH8sNPqCCaXPiMg3gduB7wNjgMnAHcC5Qdo1RPk00AacISLjgjbGSCNU1RafFqAEaAQu6GWfXJyQVnrL7UCut20+sBX4FrADqAIu97b9G9AORLxz/NVbvxFY4P2dD9wD7AFWA1cDW+PO/R1gG9AArAFO68a+aUAtkOF9/i2wI277fcBV3t+LgC8BM4FWoNOzrdbbfjfwS+Bv3jlfAQ7spWweBqqBOuAFYFbctv06lved54BbgDeBb3fZthH4NrDSO9+DQF6i+xB3n38P7AQ2AdfiKiE9lcNZwFtAPbAFuCHuWJsB9fZvBI4HPg8sidvnBOA1z87XgBPiti0CbgJe8srlaWBk0M9Cui+BGxDmBfg40AFk9bLPjcAyYDQwCngZuMnbNt/7/o1ANvAJoBko87bfDdzc5Xgb2SuUtwKLgTJgoicCW71tM7yHdLz3eWpPQuM9vEd5f68B1gMz47bN9f5eBHzJ+3ufhzvO3t3AsUAWcD/wx17K5gtAEXtfJssHcKzJQBQ4FCd4K7spt1eB8cBw3Ivlij7eh98Df/ZsnQq8D3yxl3KYD8zGienhwHbgvLj7oPG/mfhjeLbtAT7rXfdF3ucRcfdgHXAw7kW5CLg16Gch3RdrevvLCKBGVTt62ecS4EZV3aGqO4EK3EMQo93b3q6qT+JqGTP6eP7PAN9X1T2quhX4Wdy2TpwAHSoi2aq6UVXX9XCcxcApIjLW+/yI93kaUAys6KM9AI+q6qtemdwPHNHTjqp6l6o2qGobcAMwR0RK+nMs4DKcOL4LPADMEpG5Xfb5mapWqupu4K9djtftfRCRTOBC4HuerRuBH7PvPex6XYtU9W1VjarqSs+eU3qxPZ6zgLWqeq+qdqjqA8B7wDlx+/xOVd9X1RbgIXovF6MPmFD6yy5gZLxjvhvG45prMTZ56z48RhehbQYK+3j+8bhaY4wP/1bVD4CrcAK0Q0T+KCLj6Z7FuFrQybgm8CLcg30K8KKqRvtoD7imdIwer0VEMkXkVhFZJyL1uBofwMj9PZbHZTgxRVUrcdfUNULf2/F6ug8jgRz++R5O6MkQETlORJ4XkZ0iUgdc0eW6eqPr76W78+1PuRh9wITSX5bifFTn9bJPJTAl7vNkb11fSDT0UxWuyR1j0j5fVv2Dqn7EO78CP+zhOIuBk3BiuRhYApyIE8rF/bQtERfjAl4LcD7Aqd562d8DicgJwEHA90SkWkSqgeOAixK8xPpCDa622fUebvP+7q4c/gD8BZikqiXAr9h7XYnKrevvpev5DB8wofQRVa0Drgd+KSLniUiBiGSLyJkicpu32wPAtSIySkRGevvf18dTbAcO6GX7QzhxKBORCcDXYxtEZIaIfFREcnFi3oJrjnd3HWu97ZcCL6hqvXfu8+lZKLcDE0Ukp4/X0pUiXIR6F1CAyxroL58DnsH5J4/wlsO84545gOOiqp24cr5FRIpEZArwTfbew+7KoQjYraqtInIs7qUQYyfOl9rTfX0SONhLOcsSkQu963piINdh9I4Jpc+o6k9wD861uIdgC06wHvd2uRl4HRdoeRsXke1rUvadOB9jrYg83s32G3HR2g3AQpxvsc3blosL9tTgmmqjgWt6OddiXPNzc9xnwUVvu+M5YBVQLSI1fbyeeH6Pa1JuA97FBbz2GxHJw/lqf66q1XHLBuBe/rn53R/+A2jCBbmW4GqMd3nbuiuHrwI3ikgD7sX4UOxAqtqMi8y/5N3XefEnUtVdwNm4gNQuXCbD2aranzI2+oio2sC9QwUR+Qrwr6ra18CBYRhYjTLUiMg4ETlRRDJEZAauFvJY0HYZRroxUEe2kdrkAP/H3qTxP+J6BRmGsR9Y09swDCMB1vQ2DMNIgAmlYRhGAkwoDcMwEmBCaRiGkQATSsMwjASYUBqGYSTAhNIwDCMBJpSGYRgJMKE0DMNIgAmlYRhGAkwoDcMwEmBCaRiGkQATSsMwjASYUBqGYSTAhNIwDCMBJpSGYRgJMKE0DMNIgAmlYRhGAkwoDcMwEmBCaRiGkQCbhdEw+oFUiACZuMqGxG1SoF3Lbda+MGGzMBpGHJ4AFgHFQIn3fz6QB+R6/+cB2QkO1Q5E4pZWoBFo8JZGoEHLtTX5V2EkGxNKY8giFZILjIpbhgOFDK5LqhXYBdTE/V9nNdLUwoQyLIhkAaW4GlC2t+Tg3Cs57G0ixpqJgmsmtgDNQJP3v/tbtXMwzR8MpEIKgYnAeJwwlgRrUY90ANXANqASqDHhDBYTynRDJB8ow4li/FKY5DO1ES+crqm4A9iOaluSz+ULUiGZwDhgkreUBmtRv2kDqoCtwEYt1+aA7RlymFCmMiK5uAd8PHvFMTdQmxx7cDWe7UA1qvUB2/MhUiEZuDKbDkwhfAFLxZX9emCDiebgYEKZaogMByZ7yxj2jaimKs3ECyfsQjU6mAZIhYzDieMBpMbLZDBQXJm/D3yg5doRsD2hxYQyaJxvcTx7xTHZTeggaAXWAe+jutOvk0iF5AAzgFk43+xQJgKsBVZrue4O2piwYUIZBCKZuNrPNJxIhq15GM8eXI1nLZqcZqJUSClwGHAQidN0hiLVwCpgvQWBkoMJ5WAiUoCr/czE5eINJRQXjFgDbOpPVF0qZDRwFM4HaSSmDlgOrNXywXWFhA0TysFAZCQwGzgQ6zYKrpm4DliD6o5EO0uFjAKOxgSyvzTiBHONlocv7WswMKH0CxEBpuIEcmywxqQ0VcAbqFZ23SAVMgInkFMG3apw0gS8DrxvTfL9w4Qy2ciHAYbDcF3hjL5RBbyOapVUSAFwLHBwwDaFld3AUi3XbUEbki6YUCYLkQzgUJwPbaikpySVqKCvzWPLF09i+KqCUET/U52NOMFsCNqQVMeEMhmITAKOJ317fgTO7pG0vXU80lJIDsCKAlp+OYasqhyLavtMJ645vtKa4z1jQjkQREpxAmlBhn7Snk3n20fTVjmFfGTf5PoOiD5RRsvvR5LfnmFBMJ/ZASzWct0TtCGpiAllf3DN7LnAEbgxCY1+UDWRlhXHkd2R03se6a5MIj8fS/SNwiGXUjXYdAJvAissnWhfTCj3F5GxwMlYM3tArJpL04ZDGLY/33l9GM0/HUtObVaoE/RTgRpgoZanTh/+oDGh7Csumj0POCRoU9KZSA6dr51MZM8o8vvz/Wah8/ZxRJYW9e/7Rp+J4JriG4I2JBUwoewLzhd5BlaLHBC1ZbS9dgoZbfkDD9AsKqL5Z2PJM9+l77wDLBvqTXETykSITAPmY32KB8SmA2l+5yjyNDN5wrYji8jNE2BDnouUG76xA9cUbwzakKAwoewJ17PmGFzAxugnUUFXzKNl21QK/Dh+B0TvGk3rX8v8Ob7xIc3AU1quNUEbEgQmlN3hBsw9DTdtgNFPWvPpWHYqHY0l/kerny2m+adjyVdJi/E705UO4Fkt101BGzLYmFB2xQ2cewY2vuGA2D2StldPITNR6k8yeT+PlusmktOcaSlbPqLAy1quq4I2ZDAxoYxHZDou9cfSTwbAnhG0Lf0oWdGswResXZlErpsEW3LNb+kzK7VclwVtxGBhQhlDZB5weNBmpDu1w2l7+bRgRDJGq9B5/UQ6VhdYn3ufeVfLdUnQRgwGJpQAIh/BDWhhDIDaMtqWLiCrM0CRjBERojdPIPLWMOvN4zOrtVxfDNoIvzGhFDkBNySaMQDqSom8vIDMzuzgRTJGB0RvG0+bJaf7zntari8EbYSfDG2hFDkOmBO0GelOfQmRlxeQMZiBm77SCfrzsbQ8W2LpQz4TarEcur0aRI7GRHLANBSnrkgCZIJ8o5r8U+uw+a/95RCpkOOCNsIvhqZQihwJHBm0GelOYxGRl05PXZGMkQFyZTX5xzTSErQtIWeOVMisoI3wg6EnlCJzcPOwGAOgaRjt6SCSMTJBvreN3JnNtAVtS8g5QSpkatBGJJuhJZQihwGhbR4MFp0ZRF85lWh7bnqIZIxsyLhxK1lT2ogEbUuIEeA0qZAxQRuSTIaOUIocApwQtBlhYMU8WpuL0jNHMU/J/MFmMoZ30BG0LSEmE/iYVEho5j0aGkIpMhr4SNBmhIHNB9BcOSW9I8hFUbJu3kJndpQhPXSYz+QBp0uFhEJjQnERveIG3D2NoXCtPtNQTOSdo8ORwD0pQu63qmgN2o6QM4qQtOKGgnjMx+bXHjAdmURfPQWiSRxPMmhObKTgU7ssbchnDpUKOShoIwZKaH703eKCN1ODNiMMLD+e1thUsmHishry5zRZzdJnTpIKGR60EQMhvEIpMgo3x40xQDZOp7l6Unr7JXsiE+S7lWQVdtIZtC0hJgv4aDr7K9PW8F5xfskFhPX6BpH6EiKrjgyHX7InCqNkXV1p+ZU+Mxw4Kmgj+ktYhWQ+5pccMB2ZRF+dD8mc5yZVmdtMwZl7zF/pM3OkQkYFbUR/CN8DYH7JpLHqKFpaC8Lnl+yJL+4kd0yE9qDtCDEZwHypkJQZYaqvhEsoRUZgfsmkUF9CZMu0cPoleyJXybx2G52iDOEhtXynjDRsgodLKOEkwndNgbD8eKJkDL2JuqZGyDt3jw2e4TOHS4WUBW3E/hAeURGZAYwO2owwsPkAmuvLwh3A6Y1La8gttS6OfpIBnBi0EftDOITSRbmPDdqMMNCRRfTduUPHL9kduUrmf1TbwBk+M14q5ICgjegr4RBKN2yaDfefBN47nJZ0GTrNT45tomBWsyWi+8xx6RLYSX+hFCnDJgZLCq35dGyabi+cGN+oRiyw4ytFwOygjegL6S+UcAzhuI7AWTWXyFDImewr49vJXVBngR2fmSMVkh20EYlI74dCZAyWM5kUGoqJVE2y2mRXLqkhy2qVvpJLGsyCmt5CaaOVJ413jqJzKKYDJWJEJzkfr7Vapc/MTvVaZfoKpchkYGzQZoSBhmIiu8ZabbInLt5FdqbVKv0kD0jpScnSVyhtgrCksW6mddvrjdJOss+2JHS/OVwqJGWzLdJTKN3UDiODNiMMdGQSrZo8dJPL+8r5u8k2X6Wv5AGHBG1ET6SnUFo6UNLYOo3WzizSIpctSMo6yT6h0fIqfSZln+v0E0rXCydtMvpTnY0Hm0j2lU/vsmCXz5RKhYwP2ojuSD+hhIPBeo4kg9rhtDWWpOe0s0EwvY08mxPcd1IyqJOOQjkzaAPCwrqZNl3r/nLhLhssw2emSIUMC9qIrqSXUIqMw41nZwyQ9mw6qydYbXJ/mddAXoHNr+MnGaRgUCe9hNJqk0lj03TarLvi/pMNGafU2/w6PnNg0AZ0JX0eFJE8LIiTNDZNNz9vf1lQn0bPTXpSKhUyImgj4kmnGz6D9LI3Zdk1KpxzdA8W01vJHW4D+/pNStUq00l4Us5vka5UTrYgzkDIADmtzqLfPmNCud+IlAAlQZsRFmrGWrN7oJxab/mnPlMkFZIyU7ukh1DChKANCAvt2XQ2FZHSI7WkA5Mi1vweBKYEbUCMdBHKlMzWT0d2jCOCWA+TZHBcgzW/fSZlKkipL5Qiggll0tg+wQZ2SBbzGoO2IPSMkgpJiaBj6gsljAAb3SZZ7B5tze5kcUgLuTaikK8IKVKrTAehTImCCgPNBbS3FphQJosCJfPgVmt++0xKPP8mlEOIHRMs+JBs5jVamfrMxKANgFQXSpEMbLqHpLF9vDUTk82hzSn+DKU/xVIhgY9JkOo3eQw2pFrS2DPSeuMkmykRc2UMAqOCNiDVhdKa3UmivoRIR469dJLNsChZo9ptziGfCXzal1QXSmt2J4m64TY0mF8c2mJ+Sp+xGmUCioM2ICw0FVn/br84tMXK1mesRtkjLpCTciMdpytNhdYbxy+mt6bwcxQOioJOPE/lG1wI9nAni+bClL7Xac2odvP9DgKBti5T+eGxZncSaS2w0W78oriTrEzroeM3JpQ9UBS0AWFBQdtyrdbjF5kgo9stoOMzgeqBCeUQoKWADjLMjeEnEyKWVeAzVqPsARPKJNFUbA+x30yIWOTbZ6xG2QMmlEmi0VKDfGdsu/kofaYwyJOnslBaMCdJNBXZQ+w3RZ3m2vAZSw/6J0SysTEok0ZTkT3EflNkzg2/CXRgjNQUSks0TyqR3JS9z6GhMGovI5/JlAoJLHMjVR8gy/lLIlErTd8Z1pmyz1KYCKz5naq5dUn/0b0AZZ+FyxvdtLd6OrzwR3jum3DUPXDObhj7G/jBl2ATQB1kngqXboYpAno1PPj/4H2AyfCtRijJwo0a8w+4fS40APwnHPV7OAdgAmxZCXc+B8Mvgq8oZHRC5lnw3O/hBYCr4Oj74RNRyDgc3n4e/pTsa1efJxO76ik+t6We2bmZNPzhfCoAbljEOat28JHcLBoBzjyIxy6ZzTvLqxlRvoiKohy2A4wpZP2Pz+B+gMv/zLda2inJynDlWnEqtx9YRsN3FvKZbfXMAOiIktPWSdFjF3JV7PzbG8n7+pNUTC1j+Y9O5wGAZ9Yx+e7lfL5DyZlSwtu3LuDBDB9LoSDqs1BupIzHuJyI+/1yAC9wAc99uP1BTmc1n+YrfIsxNLKasTzJ52hkMofwOBfyzIf7LmIWy7gQJYOpLOEingJgHSN4jC/TwTCK2MwXuYs8OrmLz1Djyp8oObRTxHVcxQaG8whfQclAyeRgnuNf3O/aJ3KBZh+P3yOpKpRJrwPlQvQGeORy2LwJco+Aax+G1SfCtpPhf78Bl8bv/1U4CaAGbnwLij4O37gKvp+NC4zcBnfGRDXGEzD6fjjzZbhtBjS/5UXuj4K69+CHZdCxBXJnQflSWFEMHXfC+UvgljnQOBc+/wM45HvwXjKvPepzXWf+VF4uzOH5u97i8vj1R45j4fdOintAPYZls/O+T3FTd8f6wlzuPOPAfcv1hwt4KPb3j17m1G31TI7f/uOlnDum0L3EYty7kksuns19Zx7E+n9/gm/88R1mXTybVf25vr6QrT43vbOIMp9HmMtmasnlV1zLKlYziyo2UkYVh5LD7g/3L6WJ+fyRVczd5zgdCEu5iAu4nYns4WdcwypWMIsqnuJ8ZrGQM3md33AJT/IRPsVivrC3/HmEU9nllf846vg6PySfDurI5ZeUczQrmESdb6UQEKnaXEi6XcdB3eWwGWAKtI2Gqveh9HyoPg9Xu4lnPYyb5wnWXGjIh+a7Eswz/GM46WxYNMN768VqmSXQWYbruVEPWer1YX8ZRo6EHXNwta55sPovcGQyrxv8r1GedwhrRxbQ5Oc5Yryzg2OPn8Srsc/PrGNyUzvFh4zk3di6NTWUtEfJO+tg1mcIHD2epa9XdhGMJOO7g3Iidcx1v19KaWMYVeyiFIAn+Ayn8ieIy24YRwNHsYmMLjm0bzKNfHYynRry6GQSr7GSOUSBPcxgAW8CMIelbOaIf7JjE8cy0yv/PDrJ93oktZGF/8UQmB94yAhlPAthRCVMuhg29LTPDNi6GOY0Q8bTMKIKpqyFstj278LnRsN1Z8NZsV9iFYzeAGMmwtXj4bsVMCu2/wtQNgquPwJu/ST843ioOxl27oSxC2FEM2Qsgrm74s6RLDSgu/xWNade8ijXX/UUn9taT0FsfXM7Iy99lGsv/zPf/vN7TI//zj3L+dylj3JdxWLOinZJalq5neFNEUb8yyHuBdYRRf7wDhd85Wgeid9vUx2lBdnUxj6PHcaexognKj4hftco41nPCBqYxGw2sJDDyaeWOWzt03f3UEpBXM2ziD00UcZOCsmkhWwv53Y0e2jrUmYbGE4rIzghrsWzkTJu43p+xa3M4B8+1iYhQL1K1aa3b2yB3EvgiivgoWnQ2tN+d8BLH4dxU+G/ymDXJFiXjfsRPQB3ngi1myD3VLjiazDvV7CsEzKrYPRq+PFSKP0UXH0x3HAQtJwMe3bCjUuh5FPw1TfgjaOg4Uq4/zL4soBOh3XbU2CQ0mRw6eEsmlzCExkCFYs4979f5oLbP84904dT97Mz+e7kEpqeWcfkX7/JV+dN5IYxhbRefQJ3zhxF7Y4mcq95liv+93Xmfe0YlsWO+dc1HDOtjDdzMl3N6fZlnDJ9OG8fNpo9izbuPbd2kzUqEpJc0jpy+RNXcDQPkU2U5XyCL/DTPn+/O0EXuh/So2uZvcIxjOVNsuLWT2UPV3MjWyjhQb5KJW8w3rWkwkSq1ih96UnSAJmnwBUnwys/grd627cAoi/AQzvgpjVwRwsUHAk7AE7E1VamQNsCeHUlTAMYDntOhuVF0HkG7BoJ1YtgdPxxj4e6cVD5BzgI4BZYWQm3boMfToPt47xzJBMJoF/O9OE05GSiWRno+Yfy4o4mpgIU5tAxucQ1008/kM2FOexcsZ0xADNHuXIdPYy2I8by6sZaV64xVtdwzClT9ja7N9Vx4PJqTv3Mw3z/uQ18+oPdzPuv5/iXqaXUNrfvrQ1VN1E2LNvXmg46GELcRia/4wqm8Apn8BYbGEUrI/kV1/F9vk+EMu7kv9jWS2eN4eyhmeEffm6gjAJqGU0jneTT7mnCDsrI6VJmWziGw/aW/z5Moo4iKnnb/a59IrAeZqkqlElP3+0EToLLJkDVw7Aw0f7VkFPppSPcAjMzoPMCqGqGjLe97lQNkLkEDp8O2wA+Actfw0UH34bCXTDmBKh5EUp34CahWgMFG2D60Z5fNBbwWQMFT8L8/4QXk33tEsAQYGtqKIn9/Y8POKIsn0qADXsojHi9WN6sYmRjG6NnjmRnWwcZG2tduba0k/nuTg4fV+jKFWDZVsZEOin4xEGsj637+Znc+fAFfO+hC7jmo9N4ZPpwlt3yUR6bMZK6rAzanlzLtKjC65Ucf9R4lvt5vb4XcBT4HZdRTBWf8X6/s9nGtXyba7x/Oezhi9zCBOp7PM5cNtLCaNYxglYy2cIxzGYFGUAZa1jo+chXcDyT48rsPcbQTgHH7C1/NlFKkze5Wg0F7GE64//Z359EAmsVpGrTO+lvjl/A9BUwbyRsGw3XAXwFHmuBrDvgohYo/Ab8x/dhy3r46TtQ9Bm4UkCLofYeuAugFrIWwJVRyIxCxkxY/WtP3K6FVc/CoSPgBoHoF+CRWdB0C8z8FFyAu9FyLjx9kSeul8GFld7cxZfCE5/0oUaZ4fN7+OtP8qXqRg6OdFJ4wcP88KTJ/OWD3czY1cxEBApz2PXNedwH8PxGDlq4nnNF6BSIfnIG908qoXlXMznXPc+VUSVTlYyJxaz++rF7XxpPfcCxB43gtb6m+Hz2cO6/ezmfv2cF2ZNLWHXRYbzjz9U72v2uUb7KdKqZRwHbuM39fjmGxzi1h+vaRjF38190koeg3MICvko5ZbQyjwd4mKtQhCm8xGFUAfAxHuVxvsxyzqOYzZzJSx8e73WOZQKv7VO12sQ4Htz7u+YQnmb23pebDwQ2lJ1odw6doBEZAZwftBlhYfGZtDaUWpdQP9mSQ+Sr02w6YJ+5T8s1kDzKIdP0HspkWGn6TlOGjdA0CLQFdeJUFcrGoA0IE7mt9hD7TWOmlbHPdGq5BvbKT02hVO0AWoI2IywUNIYkNSaFaUzNJylMBFabhFQVSkfocrGCorDeRrbxm/pMexn5jAllD/Sc4mDsF8MaUvo+h4LqHHsZ+cygdJHtiVR+gKxGmSSGNdiwdX6zNSeln6UwEGjFKZVvrgllkshvJouoNQ39pDI7ZXOSw4IJZQ+YUCYJAcltdWM8GsmnE3RHttXafSZQPTChHCLkt1huql/UZtLu91B2htUoe6KRAPt2ho2CBitLv9iZbS+hQcCEsltUo1jiedIYZrmUvvFBniWb+0y9lmugrqPUFUqHpQgliWEN1jT0i3fzU/45SndqgjYg1W9wVdAGhIWS3RaV9YtVBd5QY4Zf7AzagFQXysqgDQgLRfXkZLcFN0xVWGnMoGN3lr2EfMaEMgE7wNJaksXwnUSCtiFsbMi13+cgYE3vXnEBHWt+J4nRVj9POqvzLZDjM7VaroG/4FNbKB1+jpg8pBhTaQPLJptlhdbs9pmUeP5NKIcQeS1k5TVZ8ztZNGXQsTaf3KDtCDl9m4bXZ1JfKFV3Y2NTJo2ROyygkyxW5dtLx2dSxvWW+kLpMO9akhi9zfIpk8WrhUFbEHp2pIJ/EtJHKK35nSRGVZNjIwklh1cKzefrMynz3JtQDjGy28ksbAh2tOgwsDGH1lrLn/SbTUEbECM9hFK1AdgTtBlhYWS1DeIwUJ4rsbQgn6nTcg08fzJGegil472gDQgLEzZZTWggdII+W2zRbp9ZF7QB8aSTUL4PFrFNBmW7yC2w5ne/WZNHa32WDdTrMyaU/UK1DVgftBlhYdr71vzuLwtLLBjmM7u1XFPK1ZY+Qul4N2gDwsKkdeRldJhY7i8RIfpCMXlB2xFyPgjagK6kl1Cq7gB2BW1GGMjqJGPcVmt+7y9Limhty0iz5ya9iAJrgjaiK+l4w1cHbUBYOPBdC+rsLw8NtzLzmfVarinXEy8dhXItNvRaUiiuI6eoltag7UgXVufRui3Xksx9JiXda+knlKrtpKAPI12ZtsbyAfvKIyOCtiD07NZyrQ7aiO5IP6F0pORbJ+yEAu4AABLASURBVB2ZuIH8zHYL6iSiJovIq4UWxPGZVUEb0BPpKZSqu4DtQZsRBjIUmbDRgjqJeGi45fD6TDPOrZaSpKdQOl4P2oCwcOB7ZKOWG9gTNVlEniolP2g7Qs4KLdeUfRmlr1CqbiNFBvVMd4Y1kj2qysb87In7RtKhYsPT+UgLKZ7Nkr5C6Xg1aAPCwmGvk23Dr/0z27OIPFtCQdB2hJyUrk1Cugulag0p1ic0XRnWRPbEjVar7Mo9oyzQ5TMtpEFwNr2F0vEaWIpLMpi5nFzr1riXzTm0vlhsvkmfeSvVa5MQBqFUrQdWBm1GGMhtI3Pa+5aADhAFvX2s+SV9ppY0qE1CGITS8SbQFLQRYeCgd8jPabWeTy8V0WIzLPrOMi3XtGgNhkMoVTuAZUGbEQayOsmY9cbQzhlsFjrvGGMi6TObtVw3B21EXwmHUAKorsNma0wKEzaTX1ozdAM7d4+irTHTBub1kU7g5aCN2B/CI5SOJdgo6Elh7lKyJDr0gmTv59Hy9zJLB/KZ5Vqu9UEbsT+ESyhVa0mzN1WqMqyR7CkfDK1aZbPQecsEsoO2I+TUAG8FbcT+Ei6hBFB9jxTuM5pOzHyL/ILGodMP/I6xRHbbFLR+EgUWpUsAJ57wCaXjRVzqgTEAMqNkHPc8GUMht3JZIc2LLWfSb97Qct0dtBH9IZxC6aLgCzF/5YAZ1kj24a+Fu1ZZm0n7/4y1KLfP7ACWB21EfwmnUAKo7sb8lUlh4kYKJmygOWg7/KADojdPINpsUW4/iQDPa7mm7VgC4RVKMH9lEpnzCvnD6sNXs/z1aFrXWGK53yzWcq0L2oiBEG6hdCzB/JUDJkORYxeFy1/5bDHNlgrkOyu1XDcEbcRACb9Qujl2zF+ZBIY1kT3nFSJB25EM1uXS+vOxFrzxmSpCMhRi+IUSYv7KxWDjLQ6UCZvJn7QuvfvV78mk/fqJZHfaYLx+0gw8m46pQN0xNIQSYl0cXwjajDAw+zUKCuvSc5Shxgw6vjMZ6rMseOMj7cBTWq6hCQAOHaEEUF2Dy7E0BkCGIvOeJyu3Jb1GGWoTOq+ZRLQqx3rf+EgUeEbLtSZoQ5LJ0BJKANXVWNrQgMlrIevEpyFdhmRrh2j5RNo35JETtC0h5wUt19DNZTX0hBJA9R1sWLYBU9BM9kf+kfpi2Qn6w/G0rSqwebl95nUt1/eDNsIPhqZQAqiuxKa8HTAFzWSf+Axkt6VmVkEHRH84ntZXiizC7TNva7m+GbQRfiGavsnyyUHkaODIoM1Id5oKaX/xY0hHTuoMKhERojdPIPLWMKtJ+sxKLddQt9BMKAFEjgPmBG1GutNYRGTJGWSkgli2Cp3XT6RjdYH1uvGZFVqurwRthN+YUMYQORI4Omgz0p2GYiIvnR6sWDZm0HHNJKIWuPGd5VquoUgoT4QJZTwik4GPgj1gA6G+hMhLp5PZmT34uYrV2bRdM4mMndmWAuQzr4fZJ9kVE8quiBQDHwPKgjYlnakrJfLKqUgkb/AEa0UBLTdNILctYwgHKf0nihvkYkgNNmNC2R0iWcB84ICALUlr2nLpfO0UIrUj/I84/6WUpt+MYZjf5xniRICntVyH3CR+JpS9ITIHOBasT3B/UdBVR9G88WB/RCwiRH8xhtbnS2wUIJ9pAP6u5TokR+IyoUyEyATgNLAUk4GwbTItK44jJ5rEPtZbcmi7aQIZ1iXRd7YCz2m5pmX//mRgQtkXRIqAM4ARQZuSzjQWEXllPrQUDixYFgV9spTm346mwEYA8hUF3gTeTOfRyZOBCWVfcX7Lo4DZDOUeTQOkI5PoGyfRunNc/5rKdZm0/2gcnSssidxvWnC1yG1BG5IKmFDuLyIjgJOBUUGbks6snUXzmsPIJ6NvNcIo6ItFtNwxhlyb38Z3KnEiGZph0gaKCWV/EBFgFnAMmH+sv9SMpvWtE8hsy++9DCuzabt9LFgvG99pB17Rcn03aENSDRPKgSAyDDgOmB60KelKRybR1XNp2XQgBV1rlxEh+uAIWh8eTr6aL9JvtuGGSGsI2pBUxIQyGYiMAU7AmuP9pr6EyFsnEG0oJS8KurSQll+PIWd3VvD9xkNOBFeLXB20IamMCWUyETkYl3dpOX39Q18/hnUXnsbo9XkUB23MEGAN8KqWa0vQhqQ6JpTJRiQTOBAXHbd0or6zEXgT1RqpkBxchsEsLMPAD6qBZVquO4I2JF0wofQTkfE4wZyM9e7piY14Atl1g1RIMU4wp2PllwzqcDXItJ9ne7AxoRwM3EAbhwEzsCg5QBOwFngfTdwlTiqkFDe48oGYYPaH3cByYN1QTxzvLyaUg4lIDnAITjQLA7ZmsOnA1R7fB7bRjx+eVEgZTjCnYU3yvlCD61WzMWhD0h0TyiBweZhTcQ/8RMLdj7waJ47rUY0k44BSIcOAQ4GZhLvs+oMCW4BVWq5bgjYmLJhQBo0TzTE4P+ZkYHiwBiWFBvY2rev9OolUfBg4OwwY6dd50oRmXBR7tZZrY9DGhA0TylRDpBCYhBPNCZAWeYS1uJqjW3wUx56QChmBC/ocyNBxa3TiRvZZC2zUco0GbE9oMaFMZVyq0XicYJbiRl0vJNiARhTYyV5h3I6m1vBbUiFjcaJ5AOFrmsfEcT2wScuT484weseEMt1woxiV4IQzJp6l3rpkDRbRiWvKNXn/NwONwA5gJ6qdSTqPr0iFCK631CScL3g06Rk1b8QNVLEVJ47tAdsz5DChDAvO11nkLTm4NKT4JSaiwl6xUNxwWjExdMKo2jZ4hg8eUiG5uNr5eJyAjiA1o+fNuNr6NmCblg++K8PYFxNKY8giFZKBCwKN8pYyXM18sGbhVFzgqwbYFfvfhjdLPUwoDaMLUiF5QHHcko/zdeYBuXH/Z/LPNVLFDVcWiVvagVacKDZ6/zcAjRaASQ9MKA1jgHi+UACs50s4MaE0DMNIQCo6sg3DMFIKE0rDMIwEmFAahmEkwITSMAwjASaUhmEYCRhUoRSRG0TkvsE8Z18RkVUiMj9oO3pDRCaLSKO4PuCGYQwSCYVSRDaKSIv3gG4Xkd+JG+EmbRGRu0Xk5vh1qjpLVRf141g53gtgrYg0eeV1l4hMTYKdG0VkQZyNm1W1UNOkr7VhhIW+1ijPUdVC3OjSxwDXdt1BHINSQxU3MESq8AjwSeBiXPe3OcAbwGlBGmUYRvLYL2FT1W3A33EDpSIii0TkFhF5CdeR/wARGS8ifxGR3SLygYh8ubtjiUi2iDwgIn/yamXHishSEakVkSoR+YW4qRNi+6uIfE1E1gJrReSXIvLjLsf8q4hc5f0907Ov1mtWf9Jb/2/AJcDVXi35r976D2tvIpIpIteIyDoRaRCRN0RkUjfXsAA4HThXVV9T1Q5VrVPVX6rqnd4+PZaHVxN9SER+751nlYgc7W27Fzcm5V89O68WkaleOWR5+0wTkRe87y70yuQ+b1ueiNwnIru8MnhN3PzjRgri/Va/FLQdXUkXd4+IXCIiT/t2AlXtdcHNc7LA+3sSsAq4yfu8CNiMm1Y0CzdKzWLgDlx/2CNwYxee5u1/A3Afru/s34C7gUxv21HAPO84U4HVwFVxdijwDG4E8Hzc/NmVQIa3fSROrMd4dnwAXIMb4OCjuL61M7x97wZu7uU6/x/wNm4yMMHVEkd0Uza3AosTlF+i8mgFPoHrN/wDYFl3Nnmfp3rlkOV9Xgr8t3eNHwHqgfu8bf8O/BU3x3imV77Fie63Lf1fvPvVguvPHVt+0cfvLgK+lCLXsCBJxxoH3AlUec/fe0AFMGyAx93nORiMpa81ysdFpBZY4j3434/bdreqrlLVDmCs98B+R1VbVXU58Fvgs3H7FwNPAeuAy9Xzt6nqG6q6TF2tbCPwf8ApXez4garuVtUWVX0VN/1mrIn7r8AiVd2OE9xC4FZVjajqc8ATwEV9vN4vAdeq6hp1rFDVXd3sNwL3I+gWrxaaqDyWqOqTXjncixPlhIjIZJwb5HrvGpcAf4nbpd2zb7qqdnrla8N1+c856vzIseXrg3nyVHFLichw3Is8HzheVYtwra9S3Cj0aUVfhfI8VS1V1Smq+lVVbYnbFj+B0Xhgt6o2xK3bhBsDMMY84HCciH3Y0VxEDhaRJ0SkWkTqcWLcdR6UrpMl3QNc6v19KU5oYnZsUd1nZJaudvTGJJyQJ2IX7q3ZE30pj+q4v5uBvD7+2GPHjh+SK7587gX+AfxRRCpF5DYRsalyA6JrxkdXN0qXfceJyEoR+bb3+XIRWe25WNaLyL/H7TtfRLaKyHdEpBr4nYi8IyLnxO2TLSI1InKE9/mTnpun1mvyz/TW98XdM1xcQLdSRPaIyOM9XPI3cbXIS72KD6q6RVWvVNWV3rFO8FxCdd7/J8TZvEhEbhKRl7zrflpEYnrwgvd/rWfn8SLyeRFZEvf9M0RkjXfsO0Rkccy1ISLTvc91Xrk8mOj+JSP4Ej+qRiUwXESK4tZNxg1AGuNpXBPz2S4+s//FVc0PUtViXLO562jUXUfwuA84V0Tm4Gbki920SmCS7Btcircj0UggW+jbW28hcKyITOxhe1/Kozd6s7PKO3ZB3LoP/aiq2q6qFap6KHACcDZwWR/PawSEuGyJxbgm+397q3fg7l8xcDnwPyJyZNzXxuJcUlOAfwN+z94KBDjXTpWqLheRg4EHgKtwY3A+iRPGHFX9LM6VFqsV39aNiffi3DmzcCPG/08Pl7IAeLRLZSX+Oofj3G8/w7V8fgL8TURGxO12sXe9o3HupW9760/2/i/17Fza5dgjcUHW73nHXoN7BmLchNOhMtzI9z/v4Ro+JKlRalXdArwM/MALJhwOfBG4v8t+twF/wIll7C1RhPOxNYrIIcBX+nC+rcBruJv3p7ia7iu40bqv9t6m84FzgD9627fj5lPpid8CN4nIQeI4vMsNjJ1/Ic5v+piIHCUiWSJSJCJXiMgX+loevdCjnaq6CXgduEFcMOx47xoBEJFTRWS254SvxzXFLa3Ifx73amqxpdtgZg8civNVlqvqr2MrVfVvqrrOcwMtxj3kJ8V9L+p9p817Bu4DPiEixd72z7K3tXUh8DdVfUZV23E+7nz2FZJuEZFxwJnAFaq6x3sZL+5h917dUsBZwFpVvddztz2AqyidE7fP71T1fe+aHsL5+PvCJ4BVqvqo5xL8Gfu23NpxL5XxnktsSXcHicePdJ6LcM7WSuAx3A18putOqnoTrga40Hu7fBv3BmkAfgMkrA573APMZu8PAXXzR38Sd1NrcMGUy1T1PW+XO4FDvR9yd02Hn+BuzNM4kbkT92Pqjk/j3soP4nym7wBH42qb0Mfy6IEfANd6dn67m+2XAMfjXAA3ezbEpnEYi3ur1uMCY4txD5DhLzE3VWz5zX589xJca+OR+JUicqaILBOXOVGLE4J4t9ROjZvgTVUrgZeA80WkFPccxF7O43Hun9i+UVwLqi9uqUk4d8+ePuzbF7fUpi7rErml+pq/PZ44N5Tn4tsat/1qXGv1Vc8F8YWER+xvFChVFlw1fDNe9HsoLzihrAjajqG60EvEGJdJ8Wjc53nsm8GwCNeK+jPwMHuzQXI9kfg0kO2texwvawOYD2zt5nwXAc8CXwYWxq2/Dngo7rPgxHm+93kDPWRa4IQvimvyJiqLm4GVPT2XuFruq13WvQx8Pq48vhS37fO4wCe42uA+Ue8u2z8HvNzlGrfQTVYBLtjaigt69ng9ad3X2wtOXAn8VnvwhYQZETlGRA4UkQwR+ThwLnv9tEZqsRw4WVxeYgnOf9aVduACYBhwr+djz8GJ5U6gQ0TOBM7ow/kex3UQuRLns4zxEHCWiJzmPT/fwrVCXva29+buqcLlUd8hImWeW+vk7vbFtcqKgXtEZAqAiEwQkZ94LqgngYNF5GLPZXUhzvXwRB+ubSdOsHtyn/0NmC0i53lBqK/hWlh4dlwQF1fYgxPdXt1SaSuUXqSuFveWuz1gc4JiLO7N24jzw3xFVd8K1CIjFjGOLY8BqHO3PIirZb1BD4Kgzm30KVwA4y6cr/0bOIHbg3NP/aW773Y5TgvwJ2Aa8Gjc+jW4QM/PcW6pc3DBm9j84IncPZ/FCfp7uCDTVT2cfzfO79kOvCIiDbgabh3wgbp0u7NxQr0L1xw+W1Vr+nBtzcAtwEuenfO6bK/BvXBu8459KM6fH3NLHePZ1IgryytVdUNv57SpIAwjpIjI9cDBqnppwp1DjFcz3wpcoqrP9+cYaVujNAyjZ7wA6ReBXyfaN4yIyMdEpFREctmbarisv8czoTSMkOGlJG0B/q6qLyTaP6Qcj+s0EnMvnKf7dpTZL6zpbRiGkQCrURqGYSTAhNIwDCMBJpSGYRgJMKE0DMNIgAmlYRhGAv4/IVQIqm2mGMoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "EukaryoticContigIDs, ProkaryoticContigIDs = set(), set()\n",
    "for line in open(\"annotations/EukAnnotatedContigIDs2.txt\"):  EukaryoticContigIDs.add(line.strip())\n",
    "for line in open(\"annotations/ProkAnnotatedContigIDs.txt\"): ProkaryoticContigIDs.add(line.strip())\n",
    "len(EukaryoticContigIDs), len(ProkaryoticAnnotations)\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib_venn import venn2\n",
    "# First way to call the 2 group Venn diagram:\n",
    "venn2(subsets = (len(ProkaryoticContigIDs.difference(EukaryoticContigIDs)), \n",
    "                 len(ProkaryoticContigIDs.intersection(EukaryoticContigIDs)), \n",
    "                 len(EukaryoticContigIDs.difference(ProkaryoticContigIDs))), set_labels = ('Prokaryotic Contigs','Eukaryotic Contigs'))\n",
    "plt.title('Contigs with an Annotation')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2 id=\"magsReport\">MAGs Report</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pylab inline\n",
    "from pandas import Series, DataFrame, pivot_table\n",
    "import numpy as np\n",
    "import numpy\n",
    "from ipywidgets import interact,SelectMultiple\n",
    "from IPython.display import display\n",
    "\n",
    "d = { 'Class'   : Series( ['a',  'b', 'b','a','a',  'b', 'b','a','a',  'b', 'b','a','a','b','b','b']),\n",
    "  'Area'   : Series( ['North','East', 'South', 'West','North','East', 'South', 'West','North','East', 'South', 'West','South', 'West','South', 'West']),\n",
    "  'Type' : Series( ['square', 'round','square', 'round', 'round', 'square', 'round', 'square', 'round', 'square','round', 'square',]),\n",
    "  'Web'  : Series( ['Y','N','N','Y','Y','N','N','Y','Y','N','N','Y','Y','N','N','Y']),\n",
    "  'Agent'   : Series( ['Mike',  'John', 'Pete','Mike',  'John', 'Pete','Mike',  'John', 'Pete','Mike',  'John', 'Pete','John', 'Pete','John', 'Pete']),\n",
    "  'Income'   : Series( [20., 40., 90., 20.]),\n",
    "  'Profit' : Series( [1., 2., 3., 4.,1., 2., 3., 4.,1., 2., 3., 4.,1., 2., 3., 4.]),\n",
    "  'Stock' : Series( [20., 23., 33., 43.,12., 21., 310., 41.,11., 21., 31., 41.,11., 22., 34., 54.] )\n",
    " }\n",
    "df = DataFrame(d)\n",
    "\n",
    "\n",
    "def my_pivot( rows, values, aggfunc):\n",
    "    dfp = df\n",
    "    piv = pivot_table( dfp, rows=rows, values=values, aggfunc=aggfunc)\n",
    "\n",
    "\n",
    "i = interact( my_pivot,\n",
    "             rows    = SelectMultiple(options=list(df.columns)), \n",
    "             values  = SelectMultiple(options=['Profit', 'Stock']),\n",
    "             aggfunc = SelectMultiple(options={ 'sum' : numpy.sum, 'ave' : numpy.average }))\n",
    "i\n",
    "\n",
    "import chart_studio.plotly as py\n",
    "import plotly.graph_objects as go\n",
    "from ipywidgets import Text, Dropdown, Output, AppLayout, interact\n",
    "import numpy as np\n",
    "# py.offline.init_notebook_mode(connected=True)\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np \n",
    "  \n",
    "% matplotlib inline \n",
    "from plotly import __version__ \n",
    "import cufflinks as cf \n",
    "  \n",
    "from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot \n",
    "  \n",
    "# to get the connection \n",
    "init_notebook_mode(connected = True) \n",
    "  \n",
    "# plotly also serves online, \n",
    "# but we are using just a sample \n",
    "cf.go_offline \n",
    "  \n",
    "# creating dataframes \n",
    "df = pd.DataFrame(np.random.randn(100, 4), columns ='A B C D'.split()) \n",
    "  \n",
    "df2 = pd.DataFrame({'Category':['A', 'B', 'C'], 'Values':[32, 43, 50]}) \n",
    "df2.head() \n",
    "\n",
    "import plotly.graph_objects as go\n",
    "import plotly.io as pio\n",
    "\n",
    "fig = go.Figure(go.Scatter(x=[1, 2, 3, 4], y=[4, 3, 2, 1]))\n",
    "fig.update_layout(title_text='hello world')\n",
    "pio.show(fig)\n",
    "\n",
    "import chart_studio.plotly as py\n",
    "import plotly.graph_objects as go\n",
    "import plotly.io as pio\n",
    "\n",
    "# these two lines allow your code to show up in a notebook\n",
    "from plotly.offline import init_notebook_mode, iplot\n",
    "init_notebook_mode()\n",
    "\n",
    "\n",
    "trace0 = go.Scatter(\n",
    "    x=[1, 2, 3, 4],\n",
    "    y=[10, 15, 13, 17]\n",
    ")\n",
    "trace1 = go.Scatter(\n",
    "    x=[1, 2, 3, 4],\n",
    "    y=[16, 5, 11, 9]\n",
    ")\n",
    "data = [trace0, trace1]\n",
    "\n",
    "py.offline.iplot(data, filename = 'basic-line')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from matplotlib_venn import venn2\n",
    " \n",
    "# First way to call the 2 group Venn diagram:\n",
    "# venn2(subsets = (10, 5, 2), set_labels = ('Group A', 'Group B'))\n",
    "# plt.show()\n",
    " \n",
    "# Second way\n",
    "venn2([set(['A', 'B', 'C', 'D']), set(['D', 'E', 'F'])])\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "/mnt/research/ShadeLab/GLBRC/annotations/EukAnnotationFilteredGenes.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "mv Final.Contigs.Filtered.bed /mnt/gs18/scratch/users/dooleys1/bowtieDB/Final.contigs.filtered.bed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/mnt/bigdata/linuxhome/dooley.shanek/GLBRC/data\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "singleFrame = read_csv(\"mapping/metaG/mags/counts/G5R3_MAIN_12SEP2016_LD1.mags.krona.kegg.minpath.tab\",sep='\\t')\n",
    "\n",
    "from numpy import zeros\n",
    "cols = [\"Level1\",\"Level2\",\"Level3\"]\n",
    "cols += list(metadataG.index)\n",
    "allSampleCounts = DataFrame(zeros((157,139)),index=None,columns=cols)\n",
    "allSampleCounts[[\"Level1\",\"Level2\",\"Level3\"]] = singleFrame[[\"Level1\",\"Level2\",\"Level3\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>G5R3_MAIN_12SEP2016_LD1</th>\n",
       "      <th>Level1</th>\n",
       "      <th>Level2</th>\n",
       "      <th>Level3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.369913</td>\n",
       "      <td>Metabolism</td>\n",
       "      <td>Lipid metabolism</td>\n",
       "      <td>Glycerolipid metabolism</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.228278</td>\n",
       "      <td>Metabolism</td>\n",
       "      <td>Global and overview maps</td>\n",
       "      <td>Microbial metabolism in diverse environments</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.203689</td>\n",
       "      <td>Metabolism</td>\n",
       "      <td>Nucleotide metabolism</td>\n",
       "      <td>Purine metabolism</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.162954</td>\n",
       "      <td>Metabolism</td>\n",
       "      <td>Carbohydrate metabolism</td>\n",
       "      <td>Pyruvate metabolism</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.121414</td>\n",
       "      <td>Cellular Processes</td>\n",
       "      <td>Transport and catabolism</td>\n",
       "      <td>Peroxisome</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   G5R3_MAIN_12SEP2016_LD1              Level1                    Level2  \\\n",
       "0                 0.369913          Metabolism          Lipid metabolism   \n",
       "1                 0.228278          Metabolism  Global and overview maps   \n",
       "2                 0.203689          Metabolism     Nucleotide metabolism   \n",
       "3                 0.162954          Metabolism   Carbohydrate metabolism   \n",
       "4                 0.121414  Cellular Processes  Transport and catabolism   \n",
       "\n",
       "                                         Level3  \n",
       "0                       Glycerolipid metabolism  \n",
       "1  Microbial metabolism in diverse environments  \n",
       "2                             Purine metabolism  \n",
       "3                           Pyruvate metabolism  \n",
       "4                                    Peroxisome  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "singleFrame.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Level1</th>\n",
       "      <th>Level2</th>\n",
       "      <th>Level3</th>\n",
       "      <th>G5R2_MAIN_09MAY2016_LD1</th>\n",
       "      <th>G5R4_NF_09MAY2016_LD1</th>\n",
       "      <th>G5R1_NF_09MAY2016_LD1</th>\n",
       "      <th>G5R2_NF_09MAY2016_LD1</th>\n",
       "      <th>G5R4_MAIN_09MAY2016_LD1</th>\n",
       "      <th>G5R1_MAIN_09MAY2016_LD1</th>\n",
       "      <th>G5R3_NF_09MAY2016_LD1</th>\n",
       "      <th>...</th>\n",
       "      <th>G6R2_NF_03OCT2016_LD1</th>\n",
       "      <th>G6R2_MAIN_03OCT2016_LD1</th>\n",
       "      <th>G6R4_MAIN_07NOV2016_LD1</th>\n",
       "      <th>G6R2_NF_07NOV2016_LD1</th>\n",
       "      <th>G6R1_MAIN_07NOV2016_LD1</th>\n",
       "      <th>G6R1_NF_07NOV2016_LD1</th>\n",
       "      <th>G6R3_MAIN_07NOV2016_LD1</th>\n",
       "      <th>G6R2_MAIN_07NOV2016_LD1</th>\n",
       "      <th>G6R4_NF_07NOV2016_LD1</th>\n",
       "      <th>G6R3_NF_07NOV2016_LD1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Metabolism</td>\n",
       "      <td>Lipid metabolism</td>\n",
       "      <td>Glycerolipid metabolism</td>\n",
       "      <td>0.010871</td>\n",
       "      <td>0.036822</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.066581</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Metabolism</td>\n",
       "      <td>Global and overview maps</td>\n",
       "      <td>Microbial metabolism in diverse environments</td>\n",
       "      <td>0.050435</td>\n",
       "      <td>0.053198</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.058469</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Metabolism</td>\n",
       "      <td>Nucleotide metabolism</td>\n",
       "      <td>Purine metabolism</td>\n",
       "      <td>0.021613</td>\n",
       "      <td>0.072880</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000173</td>\n",
       "      <td>0.071321</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Metabolism</td>\n",
       "      <td>Carbohydrate metabolism</td>\n",
       "      <td>Pyruvate metabolism</td>\n",
       "      <td>0.035071</td>\n",
       "      <td>0.040630</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.033979</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cellular Processes</td>\n",
       "      <td>Transport and catabolism</td>\n",
       "      <td>Peroxisome</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.121764</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 139 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Level1                    Level2  \\\n",
       "0          Metabolism          Lipid metabolism   \n",
       "1          Metabolism  Global and overview maps   \n",
       "2          Metabolism     Nucleotide metabolism   \n",
       "3          Metabolism   Carbohydrate metabolism   \n",
       "4  Cellular Processes  Transport and catabolism   \n",
       "\n",
       "                                         Level3  G5R2_MAIN_09MAY2016_LD1  \\\n",
       "0                       Glycerolipid metabolism                 0.010871   \n",
       "1  Microbial metabolism in diverse environments                 0.050435   \n",
       "2                             Purine metabolism                 0.021613   \n",
       "3                           Pyruvate metabolism                 0.035071   \n",
       "4                                    Peroxisome                 0.000000   \n",
       "\n",
       "   G5R4_NF_09MAY2016_LD1  G5R1_NF_09MAY2016_LD1  G5R2_NF_09MAY2016_LD1  \\\n",
       "0               0.036822                    0.0                    0.0   \n",
       "1               0.053198                    0.0                    0.0   \n",
       "2               0.072880                    0.0                    0.0   \n",
       "3               0.040630                    0.0                    0.0   \n",
       "4               0.121764                    0.0                    0.0   \n",
       "\n",
       "   G5R4_MAIN_09MAY2016_LD1  G5R1_MAIN_09MAY2016_LD1  G5R3_NF_09MAY2016_LD1  \\\n",
       "0                      0.0                 0.000000               0.066581   \n",
       "1                      0.0                 0.000000               0.058469   \n",
       "2                      0.0                 0.000173               0.071321   \n",
       "3                      0.0                 0.000000               0.033979   \n",
       "4                      0.0                 0.000000               0.000000   \n",
       "\n",
       "   ...  G6R2_NF_03OCT2016_LD1  G6R2_MAIN_03OCT2016_LD1  \\\n",
       "0  ...                    0.0                      0.0   \n",
       "1  ...                    0.0                      0.0   \n",
       "2  ...                    0.0                      0.0   \n",
       "3  ...                    0.0                      0.0   \n",
       "4  ...                    0.0                      0.0   \n",
       "\n",
       "   G6R4_MAIN_07NOV2016_LD1  G6R2_NF_07NOV2016_LD1  G6R1_MAIN_07NOV2016_LD1  \\\n",
       "0                      0.0                    0.0                      0.0   \n",
       "1                      0.0                    0.0                      0.0   \n",
       "2                      0.0                    0.0                      0.0   \n",
       "3                      0.0                    0.0                      0.0   \n",
       "4                      0.0                    0.0                      0.0   \n",
       "\n",
       "   G6R1_NF_07NOV2016_LD1  G6R3_MAIN_07NOV2016_LD1  G6R2_MAIN_07NOV2016_LD1  \\\n",
       "0                    0.0                      0.0                      0.0   \n",
       "1                    0.0                      0.0                      0.0   \n",
       "2                    0.0                      0.0                      0.0   \n",
       "3                    0.0                      0.0                      0.0   \n",
       "4                    0.0                      0.0                      0.0   \n",
       "\n",
       "   G6R4_NF_07NOV2016_LD1  G6R3_NF_07NOV2016_LD1  \n",
       "0                    0.0                    0.0  \n",
       "1                    0.0                    0.0  \n",
       "2                    0.0                    0.0  \n",
       "3                    0.0                    0.0  \n",
       "4                    0.0                    0.0  \n",
       "\n",
       "[5 rows x 139 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "path = 'mapping/metaG/mags/counts/' \n",
    "all_files = glob(path + \"/*.tab\")\n",
    "\n",
    "for filename in all_files:\n",
    "    sampleID = filename[filename.rfind('/')+1:filename.rfind('.mags')]\n",
    "    df = read_csv(filename, index_col=None, header=0,sep='\\t')\n",
    "    allSampleCounts[sampleID] = df[sampleID]\n",
    "\n",
    "allSampleCounts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(allSampleCounts.Level3.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Interact with specification of arguments\n",
    "# @interact\n",
    "# def show_articles_more_than(column=['total_reads', 'paired_total', 'paired_aligned_none', 'paired_aligned_one'], \n",
    "#                             x=(0, 20000000, 50000)):\n",
    "#     return metadataG.loc[metadataG[column] > x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadataT = read_csv(\"metadata/GLBRC_MetaT_Metadata.tsv\",sep='\\t')\n",
    "metadataT.set_index(\"nucleic_acid_name\",inplace=True)\n",
    "\n",
    "#for metaT\n",
    "metadataT.drop([\"name\", \"day\", \"year\", \"time\", \"weather\", \"air_temp_c\", \"notes\", \"rep\", \"pseudorep\", \"MMPRNT_ID\", \"time_zone\", \"longitude\", \\\n",
    "               \"latitude\", \"altitude\", \"location\", \"country\", \"soil_name\", \"number_cores\", \"SPNL_date\", \"pH\", \"lime_index\", \"P_ppm\", \"K_ppm\", \\\n",
    "               \"Ca_ppm\", \"Mg_ppm\", \"organic_matter\", \"NO3N_ppm\", \"NH4_ppm\", \"soil_moisture_percent\", \"soil_temp_10cm\", \"LDMC_mg_per_g\", \\\n",
    "               \"nitrogen_percent\", \"carbon_percent\", \"carbon_per_nitrogen\", \"height_mean_cm\", \"mass_per_leaf_g\", \"date_of_extraction\",\\\n",
    "               \"nucleic_acid_type\", \"replicate_extraction\", \"source\", \"source_mass\", \"extraction_method\", \"elution_vol_ul\", \"concentration_ng_per_ul\",\\\n",
    "               \"ratio_260_280\", \"conc_ng_per_g_source\", \"extracted_by\", \"sequence_name\", \"sequencing_date\", \"conc_sent_ng_per_ul\", \\\n",
    "               \"sequencing_type\", \"sequencing_facility\", \"primers\", \"submitted_for_sequencing\", \"sequencing_successful\", \"duplicate_submitted\",\\\n",
    "               \"dup_sequencing_name\", \"exclude_from_analysis\", \"itemID_JGI\", \"sampleID_JGI\", \"barcode\", \"JGI_library\", \"HPCC_path\", \"JGI_taxonOID\",\\\n",
    "               \"JGI_rawdataname\", \"time_numeric\", \"date\", \"precipitation\", \"Air_temp_mean\", \"air_temp_max\", \"Air_Temp_Min\",\\\n",
    "               \"Air_Pressure\", \"RH\", \"AH\", \"Wind_Speed_Mean\", \"Wind_Direction_Mean\", \"Solar_Radiation\", \"PAR\", \"soil_temp_5_cm_bare_avg\", \\\n",
    "               \"soil_temp_5_cm_sod_avg\"],axis=1,inplace=True) \n",
    "\n",
    "for id in metadataT.index: metadataT.loc[id,\"type\"] = metadataT[metadataT.index == id].plot_name[0][0:2]\n",
    "metadataT['Date'] = to_datetime(metadataT.sampling_date)\n",
    "metadataT=metadataT.sort_values(by=[\"type\",\"Date\",\"treatment\",\"plot_name\"])\n",
    "metadataT.head()\n",
    "\n",
    "#Reads After host removal\n",
    "metaT_Reads = read_csv(\"mapping/metaT/fullAssembly/multiqc_data/multiqc_bowtie2.txt\",sep=\"\\t\")\n",
    "for id in metaT_Reads.index: metaT_Reads.at[id,\"Sample\"] = metaT_Reads.at[id,\"Sample\"].replace(\".stat\",\"\")\n",
    "metaT_Reads.set_index(\"Sample\",inplace=True)\n",
    "\n",
    "metadataT = merge(metadataT,metaT_Reads,left_index=True,right_index=True)\n",
    "metadataT.sort_values(\"Date\",inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15,7))\n",
    "# G5Data = metadataG[metadataG['type'] == 'G5']\n",
    "# G5Data.groupby(['Date','plot_name']).sum()[['aligned','total_reads']].unstack().plot(ax=ax);\n",
    "ax = metadataG.groupby(['Date','plot_name']).sum()['pLantAligned'].unstack().plot(ax=ax,title=\"% Reads Aligning to Respective Plant Host Assembly G5=Switchgrass G6=Miscanthus\"); #legend=false\n",
    "patches, labels = ax.get_legend_handles_labels()\n",
    "ax.set_ylabel(\"% Reads\",fontsize=14)\n",
    "ax.set_ylim(0,100)\n",
    "# ax.legend(patches, labels, loc='best')\n",
    "# Shrink current axis by 20%\n",
    "box = ax.get_position()\n",
    "ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])\n",
    "\n",
    "# Put a legend to the right of the current axis\n",
    "ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))\n",
    "\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('images/PlantAlignedReads.png')\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(15,7))\n",
    "G6Data = metadataG[metadataG['type'] == 'G6']\n",
    "ax = G6Data.groupby(['Date','plot_name']).sum()['pLantAligned'].unstack().plot(ax=ax,title=\"Seasonal Perctage of reads aligning to Micanthus Assembly\",fontsize=12);\n",
    "ax.set_ylabel(\"% Reads\",fontsize=14)\n",
    "ax.set_ylim(0,100)\n",
    "# fig = ax.get_figure()\n",
    "# fig.savefig('images/MiscanPlantReads.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove Human Disease Pathway related rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allSampleCounts = allSampleCounts[allSampleCounts.Level1 != \"Human Diseases\"]\n",
    "allSampleCounts.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalize by total reads and log transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import log, nan\n",
    "for sample in metadataG.index:\n",
    "    allSampleCounts[sample]= -log(allSampleCounts[sample].replace(0, nan)/metadataG.loc[sample,\"total_reads\"])\n",
    "for sample in metadataG.index: allSampleCounts[sample]= allSampleCounts[sample].replace(nan, 0)\n",
    "allSampleCounts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allSampleCounts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allSampleCounts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for level in [\"Level1\",\"Level2\",\"Level3\"]:\n",
    "    outfile = \"annotations/%s_CountTable_logged2.tsv\" % (level)\n",
    "    levelDF = allSampleCounts.groupby([level]).sum()\n",
    "    levelDF.to_csv(outfile,sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(15,7))\n",
    "# use unstack()\n",
    "data.groupby(['date','type']).count()['amount'].unstack().plot(ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "colors = [\"#74C476\", \"#f26d07\",\"#794955\"]\n",
    "G5_metadata = metadataG[metadataG.type == \"G5\"]\n",
    "G6_metadata = metadataG[metadataG.type == \"G6\"]\n",
    "\n",
    "ax = G5_metadata.loc[:,['Plant Reads','Fungal Reads', 'Remaining']].plot.bar(stacked=True, color=colors, figsize=(14,7))\n",
    "ax.set_ylabel(\"Read Count\",fontsize=12)\n",
    "ax.set_ylim(0,16000000)\n",
    "\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('images/MiscanthusBarCombined.png')\n",
    "ax = G6_metadata.loc[:,['Plant Reads','Fungal Reads', 'Remaining']].plot.bar(stacked=True, color=colors, figsize=(14,7))\n",
    "ax.set_ylabel(\"Read Count (Millions)\",fontsize=12)\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('images/SwitchgrassBarCombined.png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metaG_Reads = read_csv(\"mapping/metaG/fullAssembly/multiqc_data/multiqc_bowtie2.txt\",sep=\"\\t\")\n",
    "for id in metaG_Reads.index: metaG_Reads.at[id,\"Sample\"] = metaG_Reads.at[id,\"Sample\"].replace(\".fungal.stat\",\"\")\n",
    "metaG_Reads.set_index(\"Sample\",inplace=True)\n",
    "\n",
    "metadataG = merge(metadataG,metaG_Reads,left_index=True,right_index=True)\n",
    "metadataG.sort_values(\"Date\",inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadataG = metadataG[metadataG['type']==\"G5\"] # SWITCHGRASS Only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadataG[\"total_reads\"].plot(kind='bar',ax=plt.subplot(121),figsize=(30,10),title=\"SG MetaG after host removal\",color='orange')\n",
    "metadataT[\"total_reads\"].plot(kind='bar',ax=plt.subplot(122),figsize=(30,10),title=\"SG MetaT after host removal\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Creating links to the raw data files from JGI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir, path, system\n",
    "rawFiles = {}\n",
    "baseDir = \"/mnt/research/ShadeLab/Sequence/raw_sequence/GLBRC/metagenomes/\"\n",
    "dirs = listdir(baseDir)\n",
    "for dirName in dirs:\n",
    "    rawDir = path.join(baseDir,dirName,\"Raw_Data/\")\n",
    "    fastq = listdir(rawDir)[0]\n",
    "    rawFiles[fastq] = path.join(rawDir,fastq)\n",
    "metaGUnpaired = \"/mnt/research/ShadeLab/GLBRC/mapping/metaG/unpaired/%s.fastq.gz\"\n",
    "    \n",
    "for readFName,row in metadata.iterrows():\n",
    "    if readFName not in rawFiles:\n",
    "        print(\"No fastq for \"+readFileName)\n",
    "        continue\n",
    "    system(\"ln -s %s %s\" % (rawFiles[readFName],metaGUnpaired %(row.loc['nucleic_acid_name'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(8,5))\n",
    "G5Data = metadataG[metadataG['type'] == 'G5']\n",
    "G5Data.groupby(['Date','plot_name']).sum()['percFungalAligned'].unstack().plot(ax=ax,title=\"Seasonal Perctages of Reads Aligning to Combined Fungal Assemblies\");\n",
    "ax.set_ylabel(\"% Reads\",fontsize=14)\n",
    "ax.set_ylim(0,10)\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('images/SwitchgrassFungalAlign.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2><a id=\"analysis\">Sequence Mapping Overview</a></h2>\n",
    "<h4>Host organism reads removal</h4>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hostMapping = read_csv(\"mapping/metaG/fullAssembly/multiqc_data/multiqc_bowtie2.txt\",sep='\\t')\n",
    "hostMapping.set_index(\"Sample\",inplace=True)\n",
    "hostMapping['type'] = \"\"\n",
    "for id in hostMapping.index: hostMapping.at[id,\"type\"] = id[0:2]\n",
    "hostMapping[\"sample_name\"]=\"\"\n",
    "for id in hostMapping.index: hostMapping.at[id,\"sample_name\"] = id.replace(\".stat\",\"\")\n",
    "hostMapping.set_index(\"sample_name\",inplace=True)\n",
    "hostMapping.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SGData = hostMapping[hostMapping['type']=='G5']\n",
    "MiscanData = hostMapping[hostMapping['type']=='G6']\n",
    "\n",
    "sgTotal = SGData['total_reads'].sum()\n",
    "miscanTotatl = MiscanData['total_reads'].sum()\n",
    "\n",
    "sgBact = SGData['paired_aligned_none'].sum()\n",
    "miscanBact = MiscanData['paired_aligned_none'].sum()\n",
    "\n",
    "(sgTotal-sgBact)/sgTotal*100\n",
    "\n",
    "(miscanTotatl-miscanBact)/miscanTotatl*100\n",
    "\n",
    "comma(SGData['paired_aligned_none'].sum() +MiscanData['paired_aligned_none'].sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Alignment of non-host related reads to metagenomic assembly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seqMapping = read_csv(\"mapping/metaG/fullAssembly/multiqc_data/multiqc_bowtie2.txt\",sep='\\t')\n",
    "seqMapping.set_index(\"Sample\",inplace=True)\n",
    "seqMapping['type'] = \"\"\n",
    "for id in seqMapping.index: seqMapping.at[id,\"type\"] = id[0:2]\n",
    "seqMapping[\"sample_name\"]=\"\"\n",
    "for id in seqMapping.index: seqMapping.at[id,\"sample_name\"] = id.replace(\".stat\",\"\")\n",
    "seqMapping.set_index(\"sample_name\",inplace=True)\n",
    "seqMapping.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SGData = seqMapping[hostMapping['type']=='G5']\n",
    "MiscanData = seqMapping[hostMapping['type']=='G6']\n",
    "\n",
    "sgTotal = SGData['total_reads'].sum()\n",
    "miscanTotatl = MiscanData['total_reads'].sum()\n",
    "\n",
    "sgBact = SGData['paired_aligned_none'].sum()\n",
    "miscanBact = MiscanData['paired_aligned_none'].sum()\n",
    "\n",
    "print('SG %% aligned: %.2f' % ((sgTotal-sgBact)/sgTotal*100),comma(sgTotal))\n",
    "print('Miscanthus %% aligned: %.2f' % ((miscanTotatl-miscanBact)/miscanTotatl*100),comma(miscanTotatl))\n",
    "print(comma(SGData['paired_aligned_none'].sum() +MiscanData['paired_aligned_none'].sum()))\n",
    "\n",
    "#MetaG\n",
    "# metaG = read_csv(\"mapping/metaG/annotatedContigs/logs/multiqc_data/mqc_trimmomatic_plot_1.txt\",sep=\"\\t\")\n",
    "# metaG.set_index('Sample',inplace=True)\n",
    "# print(comma(metaG['Surviving Reads'].sum()))\n",
    "\n",
    "# metaT = read_csv(\"mapping/metaT/fullAssembly/logs/multiqc_data/multiqc_trimmomatic.txt\",sep=\"\\t\")\n",
    "# metaT.set_index('Sample',inplace=True)\n",
    "# print(comma(metaT['surviving'].sum())\n",
    "\n",
    "# metaT['Base'],metaG['Base'] = '',''\n",
    "\n",
    "# for sampleID in metaG.index:\n",
    "#     metaG.at[sampleID,'Base'] = sampleID[:sampleID.rfind('_')]\n",
    "# for sampleID in metaT.index:\n",
    "#     metaT.at[sampleID,'Base'] = sampleID[:sampleID.rfind('_')]\n",
    "\n",
    "# metaT.set_index(\"Base\",inplace=True)\n",
    "# metaG.set_index(\"Base\",inplace=True)\n",
    "\n",
    "# comma(metaT['surviving'].sum()-metaG['Surviving Reads'].sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for sampleID in ids.intersection(ids2):\n",
    "#     print(sampleID,'\\t',comma(int(metaT.at[sampleID,'surviving'])),'\\t',comma(int(metaG.at[sampleID,'Surviving Reads'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = read_csv(\"mapping/metaG/annotatedContigs/logs/multiqc_data/mqc_bowtie2_pe_plot_1.txt\",sep=\"\\t\")\n",
    "data[\"TotalSampleReads\"] = data.sum(axis=1)\n",
    "#data[[\"PE mapped uniquely\",\"PE mapped discordantly uniquely\",\"PE multimapped\"]]/data[\"TotalSampleReads\"] \n",
    "print(\"(%i, %i)\" % (data.shape, metadata.shape))\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Microbe Census (single copy gene counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "scc_path=\"mapping/metaG/fullAssembly/singleCopyGeneCounts/\"\n",
    "files = listdir(scc_path)\n",
    "metadata[\"genome_equivalents\"] = 0.0\n",
    "metadata[\"average_genome_size\"] = 0.0\n",
    "metadata[\"total_reads\"] = 0\n",
    "\n",
    "for fname in files:\n",
    "    sampleID = fname.replace(\".txt\",\"\")\n",
    "    metadata.at[sampleID,\"total_reads\"] = seqMapping.at[sampleID,\"total_reads\"]\n",
    "    for line in open(scc_path+fname):\n",
    "        if \"average_genome_size\" in line:\n",
    "#             print(line.strip().split(\"\\t\"))\n",
    "            ags = float(line.strip().split(\"\\t\")[1])\n",
    "            metadata.at[sampleID,\"average_genome_size\"] = ags\n",
    "        elif \"genome_equivalents\" in line: \n",
    "            ge = float(line.strip().split(\"\\t\")[1])\n",
    "            metadata.at[sampleID,\"genome_equivalents\"] = ge\n",
    "            \n",
    "            \n",
    "metadata.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = metadata.groupby(['Date']).mean()\n",
    "df[\"average_genome_size\"]=df[\"average_genome_size\"]/1000000.0\n",
    "df[\"total_reads\"]=df[\"total_reads\"]/25000.0\n",
    "df.reset_index().plot(x=\"Date\", y=[\"genome_equivalents\", \"average_genome_size\",\"total_reads\"], kind=\"bar\",figsize=(15,8));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For MetaT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seqMapping = read_csv(\"mapping/metaT/fullAssembly/multiqc_data/multiqc_bowtie2.txt\",sep='\\t')\n",
    "seqMapping.set_index(\"Sample\",inplace=True)\n",
    "seqMapping['type'] = \"\"\n",
    "for id in seqMapping.index: seqMapping.at[id,\"type\"] = id[0:2]\n",
    "seqMapping[\"sample_name\"]=\"\"\n",
    "for id in seqMapping.index: seqMapping.at[id,\"sample_name\"] = id.replace(\".stat\",\"\")\n",
    "seqMapping.set_index(\"sample_name\",inplace=True)\n",
    "seqMapping.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "scc_path=\"mapping/metaT/fullAssembly/singleCopyGeneCounts/\"\n",
    "files = listdir(scc_path)\n",
    "metadata[\"genome_equivalents\"] = 0.0\n",
    "metadata[\"average_genome_size\"] = 0.0\n",
    "metadata[\"total_reads\"] = 0\n",
    "\n",
    "for fname in files:\n",
    "    sampleID = fname.replace(\".txt\",\"\")\n",
    "    if sampleID not in seqMapping.index:\n",
    "        print(\"Missing:\",sampleID,fname)\n",
    "        continue\n",
    "    metadata.at[sampleID,\"total_reads\"] = seqMapping.at[sampleID,\"total_reads\"]\n",
    "    for line in open(scc_path+fname):\n",
    "        if \"average_genome_size\" in line:\n",
    "#             print(line.strip().split(\"\\t\"))\n",
    "            ags = float(line.strip().split(\"\\t\")[1])\n",
    "            metadata.at[sampleID,\"average_genome_size\"] = ags\n",
    "        elif \"genome_equivalents\" in line: \n",
    "            ge = float(line.strip().split(\"\\t\")[1])\n",
    "            metadata.at[sampleID,\"genome_equivalents\"] = ge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = metadata.groupby(['Date']).mean()\n",
    "df.sort_values(by=[\"month\"],inplace=True)\n",
    "df[\"average_genome_size\"]=df[\"average_genome_size\"]/10000.0\n",
    "df[\"total_reads\"]=df[\"total_reads\"]/25000.0\n",
    "df.reset_index().plot(x=\"Date\", y=[\"genome_equivalents\", \"average_genome_size\",\"total_reads\"], kind=\"bar\",figsize=(15,8));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=metadataG\n",
    "\n",
    "import cufflinks as cf\n",
    "\n",
    "@interact\n",
    "def scatter_plot(x=list(df.select_dtypes('number').columns), \n",
    "                 y=list(df.select_dtypes('number').columns)[1:],\n",
    "                 theme=list(cf.themes.THEMES.keys()), \n",
    "                 colorscale=list(cf.colors._scales_names.keys())):\n",
    "    print(x.title())\n",
    "    print(y.title())\n",
    "    return df.iplot(kind='scatter', x=x, y=y, mode='markers', \n",
    "             xTitle=x.title(), yTitle=y.title(), \n",
    "#              text='This is the title',\n",
    "             #title=f'{y.title()} vs {x.title()}',\n",
    "            theme=theme, colorscale=colorscale)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KEGG Annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotations = read_csv(\"annotations/MetagenomicAssemblyAnnotations.txt\",sep='\\t')\n",
    "#annotations = read_csv(\"annotations/Test.txt\",sep='\\t')\n",
    "\n",
    "annotations = annotations.sort_values('evalue',ascending = True).groupby('qseqid').head(1)\n",
    "annotations.set_index(\"qseqid\",inplace=True)\n",
    "print(annotations.shape[0])\n",
    "prin(annotations=annotations[annotations[\"evalue\"]<=1.0e-5])\n",
    "print(annotations.shape[0])\n",
    "annotations.to_csv(\"annotations/MetagenomicAssemblyAnnotations_tophit.tsv\",sep='\\t')\n",
    "annotations[annotations.index=='k127_13265132']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotations = read_csv(\"annotations/MetagenomicAssemblyAnnotations_tophit.tsv\",sep='\\t')\n",
    "annotations.set_index(\"qseqid\",inplace=True)\n",
    "annotations.sort_values(\"sseqid\",inplace=True)\n",
    "annotations.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ko_map = read_csv(\"annotations/ko/prokaryotes.dat\",sep='\\t')\n",
    "# ko_map.set_index(\"GENE\")\n",
    "print(ko_map.shape)\n",
    "ko_map.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(ko_map.GENE.unique()),ko_map.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ko_map[ko_map[\"GENE\"]==\"\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotations.shape,genesW_KO.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(annotations.sseqid.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "genesW_KO = set(annotations.sseqid.unique()).intersection(ko_map.GENE)\n",
    "len(genesW_KO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "genesW_KO = merge(annotations, ko_map, how='inner',left_on=\"sseqid\",right_on=\"GENE\")\n",
    "genesW_KO.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annotation_hierarchy = read_csv(\"annotations/KeggAnnotationTable_filtered.tsv\",sep='\\t')\n",
    "annotation_hierarchy[\"EC#\"] = \"\"\n",
    "for i in annotation_hierarchy.index:\n",
    "    function = annotation_hierarchy.at[i,'function']\n",
    "    ecNumIndex = function.rfind(\" [EC\")\n",
    "    if ecNumIndex != -1:\n",
    "        annotation_hierarchy.at[i,'EC#'] = function[ecNumIndex+2:-1]\n",
    "        annotation_hierarchy.at[i,'function'] = function[:ecNumIndex]\n",
    "annotation_hierarchy.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(annotation_hierarchy.level3.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for function in annotation_hierarchy.level3.unique():\n",
    "    \n",
    "    print(function)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(annotation_hierarchy.id.unique()),len(set(ko_map.KO_NUM.unique())),len(set(ko_map.KO_NUM.unique()).intersection(annotation_hierarchy.id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.merge(annotation_hierarchy, ko_map, left_on='id',right_on='KO_NUM', how='left')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniqGenes = set(df[\"GENE\"].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "foundGenes = set()\n",
    "for line in open(\"annotations/UniqFoundGenes.txt\"): foundGenes.add(line.strip())\n",
    "len(foundGenes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(foundGenes.difference(uniqGenes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ko_genes = set(ko_map['GENE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "koToContigMap = {}\n",
    "contigToKO = {}\n",
    "missing = 0\n",
    "for line in open(\"annotations/ContigsW_Annotations.txt\"):\n",
    "    contig,gene = line.strip().split('\\t')\n",
    "    if gene not in ko_genes:missing += 1; continue\n",
    "    subKO_Map = ko_map[ko_map[\"GENE\"] == gene]\n",
    "    for row in subKO_Map.iterrows():\n",
    "        ko = row[1]['KO_NUM']\n",
    "        try: koToContigMap[ko].add(contig)\n",
    "        except: koToContigMap[ko] =set([contig])\n",
    "        try: contigToKO[contig].add(ko)\n",
    "        except: contigToKO[contig] =set([ko])\n",
    "        \n",
    "from pickle import dump\n",
    "dump(koToContigMap,open('pickles/KO_ToContigMap.p','wb'))\n",
    "dump(koToContigMap,open('pickles/ContigToKO.p','wb'))\n",
    "len(contigToKO),missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metagenomeAnnotations = read_csv(\"annotations/ContigsW_Annotations.txt\",sep='\\t')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create the level 3 table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "genes = set()\n",
    "for line in open(\"annotations/UniqFoundGenes.txt\"):\n",
    "    genes.add(line.strip())\n",
    "len(genes),len(genes.difference(ko_map.GENE))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[df[\"level3\"]==\"Glycolysis / Gluconeogenesis [PATH:ko00010]\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.merge(annotation_hierarchy, ko_map, left_on='function',right_on='GENE', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_csv(\"annotations/ContigsW_Annotations.txt\",sep='\\t')\n",
    "\n",
    "df.head(), df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ko_dict = {}\n",
    "for row in ko_map.iterrows():\n",
    "    if row[1].values[1] in ko_dict:\n",
    "        print(row)\n",
    "        print()\n",
    "        print(ko_dict[row[1].values[1]])\n",
    "        break\n",
    "    ko_dict[row[1].values[1]]=row[1].values[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "annos = Annotation()\n",
    "missing = Counter()\n",
    "with open(\"annotations/MetagenomicAssemblyAnnotations_filtered_90PID.txt\",\"r\") as fh:\n",
    "    for i,line in enumerate(fh):\n",
    "        if i%500000==0:print(i,end=' ')\n",
    "        rec = line.strip().split(\"\\t\")\n",
    "        contigID, geneID = rec[0],rec[1]\n",
    "        val = ko_map.loc[ko_map[\"Gene\"] == geneID,['KoNum']]\n",
    "        try:koNum = val.values[0][0]\n",
    "        except: missing[geneID] += 1\n",
    "        annos[contigID] = koNum\n",
    "dump(annos,open(\"pickles/ContigAnnotationMap.p\",\"wb\"))\n",
    "dump(missing,open(\"pickles/NotInKO_Map.p\",\"wb\"))\n",
    "len(missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import randint\n",
    "class Annotation:\n",
    "    def __init__(self):\n",
    "        self.contigs = {} # ContigsName -> set of related Annotations\n",
    "        self.annotations = {} # KO number -> contigs with that KO\n",
    "    def __setitem__(self,key,value):\n",
    "        try: self.contigs[key].append(value)\n",
    "        except: self.contigs[key] = [value]\n",
    "        try: self.annotations[value].append(key)\n",
    "        except: self.annotations[value] = [key]\n",
    "    def __getitem__(self,key): return self.contigs[key][randint(0,len(self.contigs[key]))]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating the annotation fasta files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Bio.SeqIO import parse, write\n",
    "from Bio.SeqRecord import SeqRecord\n",
    "annotatedRegions = open(\"assemblies/AnnotatedContigs.fa\",\"w\")\n",
    "geneDescripts = open(\"annotations/AnnotationDescripts.bed\",\"w\")\n",
    "annotations = glob(\"mags/final.contigs.*/*.gbk\")\n",
    "seqs = set()\n",
    "hypoCounter = 0\n",
    "for fname in annotations:\n",
    "    for rec in parse(fname,'genbank'):\n",
    "        for feature in rec.features:\n",
    "            try:\n",
    "                product = feature.qualifiers['product'][0]\n",
    "                #if product == \"hypothetical protein\":continue\n",
    "                #subSeq = rec.seq[feature.location.start:feature.location.end]\n",
    "                hypoCounter += int(product == \"hypothetical protein\")\n",
    "                geneDescripts.write(\"%s\\t%i\\t%i\\t%s\\n\" % (rec.id,feature.location.start,feature.location.end,product))\n",
    "                if rec.id in seqs: continue\n",
    "                rec.seq = rec.seq.upper()\n",
    "                write(rec,annotatedRegions,\"fasta\")\n",
    "                seqs.add(rec.id)\n",
    "            except:pass\n",
    "annotatedRegions.close()\n",
    "geneDescripts.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2><a id=\"occon\">Calculate Abundance and occupancy of contigs in samples</a></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Individual Sample\n",
    "from collections import Counter\n",
    "seqPresenceCounter, seqMappedCounter = Counter(), Counter()\n",
    "missingFiles = set()\n",
    "for i, statFName in enumerate(metadata.index):\n",
    "    print(\"%i. %s\" % (i,statFName),end = '\\t')\n",
    "    try:\n",
    "        with open(\"mapping/metaG/stats/%s.tsv\" % statFName) as fh:\n",
    "            for line in fh:\n",
    "                rec = line.strip().split()\n",
    "                seqMappedCounter[rec[0]] += int(rec[2])\n",
    "                seqPresenceCounter[rec[0]] += int(int(rec[2])>0)\n",
    "        if i % 3 == 0:print()\n",
    "    except:\n",
    "        missingFiles.add(\"mapping/metaG/stats/%s.tsv\" % statFName)\n",
    "dump(seqPresenceCounter,open(\"pickles/seqPresenceCounter_genes.p\",\"wb\"))\n",
    "dump(seqMappedCounter,  open(\"pickles/seqMappedCounter_genes.p\",  \"wb\"))  \n",
    "seqPresenceDist = Series(load(open(\"pickles/seqPresenceCounter_genes.p\",\"rb\")))/136.0\n",
    "seqPresenceDist.plot.hist();\n",
    "\n",
    "dump(missingFiles,open(\"pickles/missingFiles_genes.p\",\"wb\")) \n",
    "len(missingFiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seqPresenceCounter = load(open(\"pickles/seqPresenceCounter_genes.p\",\"rb\"))\n",
    "smp = Series(seqPresenceCounter)\n",
    "totalReads = smp.sum()\n",
    "for i in range(10):\n",
    "    sub = smp[smp <= i ]\n",
    "    subSum = sub.sum()\n",
    "    print(i,len(sub),\"%.2f%%\" % (len(sub)/len(smp)*100),comma(subSum),\"%.2f%%\" % ((subSum/totalReads)*100))\n",
    "print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remove anything that is in less than 5 samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Before Number of genes:\",comma(len(smp)))\n",
    "keepers = set(smp[smp >= 5].index)\n",
    "print(\"After Number of genes:\",comma(len(keepers)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read the single copy number values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = glob(\"mapping/metaG/annotatedContigs/logs/singleCopyGeneCounts/*.txt\")\n",
    "metadata[\"SingleCopyCount\"] = 0.0\n",
    "for fname in files:\n",
    "    with open(fname) as fh:\n",
    "        lines = fh.readlines()[-1].strip().split('\\t')\n",
    "        sampleName = fname[fname.rfind('/')+1:-4]\n",
    "        metadata.at[sampleName,\"SingleCopyCount\"] = float(lines[-1])\n",
    "metadata.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read the count numbers and normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Bio.SeqIO import parse\n",
    "readLens={}\n",
    "for rec in parse(\"assemblies/AnnotatedContigs.fa\",\"fasta\"): readLens[rec.id] = len(rec.seq)/1000\n",
    "#RPKG = (reads mapped to gene)/(gene length in kb)/(genome equivalents)\n",
    "print(\"Number of reads: %s\" % (comma(len(readLens))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "readMap = {}\n",
    "for line in open(\"mapping/metaG/stats/G5R1_NF_09MAY2016_LD1.tsv\"):\n",
    "    rec = line.strip().split()\n",
    "    if rec[0] not in keepers:continue\n",
    "    readName = rec[0][:rec[0].rfind(\"_\")]\n",
    "    readName = readName[:readName.rfind(\"_\")]\n",
    "    readMap[rec[0]] = readName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allCounts,counter = {},0\n",
    "for sampleID in metadata.index:\n",
    "    counter += 1\n",
    "    print(\"%i. %s\" % (counter,sampleID),end='\\t')\n",
    "    allCounts[sampleID] = {}\n",
    "    for line in open(\"mapping/metaG/stats/%s.tsv\" % sampleID):\n",
    "        rec = line.strip().split()\n",
    "        if rec[0] not in keepers:continue\n",
    "        allCounts[sampleID][rec[0]] = float(rec[2])/readLens[readMap[rec[0]]]/metadata.at[sampleID,\"SingleCopyCount\"]\n",
    "    if counter % 3 == 0: print()\n",
    "allCounts = DataFrame(allCounts) \n",
    "dump(allCounts,open(\"pickles/allCounts_genes.p\",\"wb\"))  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allCounts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs, ys = [],[]\n",
    "for i in range(1,101):\n",
    "    xs.append(i)\n",
    "    ys.append(len(seqPresenceDist[seqPresenceDist >= i/100.0]))\n",
    "plt.plot(xs,ys)\n",
    "plt.xlabel('Percent of samples',fontsize=14)\n",
    "plt.ylabel('Contigs with >= 1 read',fontsize=14)\n",
    "plt.savefig(\"figures/GeneAbundance.png\",bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seqPresenceCounter = Series(load(open(\"pickles/seqPresenceCounter_genes.p\",\"rb\")))\n",
    "seqMappedCounter = Series(load(open(\"pickles/seqMappedCounter_genes.p\",\"rb\")))\n",
    "seqMappedCounter.describe()\n",
    "seqPresenceCounter.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allCounts = load(open(\"pickles/allCounts_genes.p\",\"rb\"))\n",
    "# allCounts[\"Average\"] = allCounts.sum(axis=1)/float(len(allCounts.columns))\n",
    "# allCounts[\"Average\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allCounts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Before Filtering:\",len(allCounts))\n",
    "allCounts = allCounts[allCounts[\"Average\"]>1]\n",
    "xs,ys=[],[]\n",
    "for i in range(1,30):\n",
    "    xs.append(i)\n",
    "    ys.append(len(allCounts[allCounts[\"Average\"] > i]))\n",
    "    xs.append(i+.5)\n",
    "    ys.append(len(allCounts[allCounts[\"Average\"] > i+.5]))\n",
    "    \n",
    "allCounts.drop(\"Average\",axis=1,inplace=True)\n",
    "print(\"After Filtering:\",len(allCounts))\n",
    "#allCounts = allCounts.transpose()\n",
    "#allCounts= DataFrame(allCounts,index=allCounts.index)\n",
    "dump(allCounts,open(\"pickles/allCountsFiltered_genes.p\",\"wb\")) \n",
    "plt.plot(xs,ys)\n",
    "# plt.xlabel('Percent of samples',fontsize=14)\n",
    "# plt.ylabel('Read Count',fontsize=14)\n",
    "plt.savefig(\"figures/GeneAvgCount.png\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sortedmeta = metadata[[\"sampling_date\",\"Date\",\"type\",\"nucleic_acid_name\",\"treatment\"]]\n",
    "sortedmeta = sortedmeta.sort_values(by=[\"type\",\"Date\"])\n",
    "counter = 0\n",
    "sampleOrder = {}  \n",
    "for id in sortedmeta.nucleic_acid_name:\n",
    "    counter+=1\n",
    "    sampleOrder[id] = counter\n",
    "\n",
    "allCounts = DataFrame(allCounts).transpose()\n",
    "allCounts['Rank'] = allCounts.index.to_series().map(sampleOrder)\n",
    "allCounts.sort_values('Rank',inplace = True)\n",
    "allCounts.drop('Rank', 1, inplace = True)\n",
    "dump(allCounts,open(\"pickles/allCountsFilteredSorted_genes.p\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allCounts = load(open(\"pickles/allCountsFilteredSorted_genes.p\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#allCounts = DataFrame(combined_abundance)\n",
    "seed = np.random.RandomState(seed=3)\n",
    "allCounts_matrix = allCounts.as_matrix()\n",
    "print(\"Calculating Distance\")\n",
    "similarities = euclidean_distances(allCounts_matrix)\n",
    "\n",
    "nsamples = len(allCounts)\n",
    "print(\"Running NMDS with %i samples\" % (nsamples))\n",
    "mds = manifold.MDS(n_components=2, max_iter=3000, eps=1e-9, random_state=seed, dissimilarity=\"precomputed\", n_jobs=1)\n",
    "pos = mds.fit(similarities).embedding_\n",
    "nmds = manifold.MDS(n_components=2, metric=False, max_iter=3000, eps=1e-12, dissimilarity=\"precomputed\", random_state=seed, n_jobs=1, n_init=1)\n",
    "npos = nmds.fit_transform(similarities, init=pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "fig = plt.figure(1)\n",
    "ax = plt.axes([0., 0., 1.5, 1.95])\n",
    "s = 100\n",
    "index = 0\n",
    "totalGroups = nsamples/4\n",
    "l1s, l2s = [], []\n",
    "dateColors = []\n",
    "for sample in sortedmeta.index:\n",
    "#     print(sample)\n",
    "    if sortedmeta[sortedmeta.index == sample].type[0] == \"G5\": \n",
    "        species = \"SG\"\n",
    "        if sortedmeta[sortedmeta.index == sample].treatment[0] == \"nitrogen free\": \n",
    "            treatment = \"NF\"\n",
    "            marker = '^'\n",
    "        else: \n",
    "            treatment = \"Fert\"\n",
    "            marker= \"v\"\n",
    "    else: \n",
    "        species = \"MC\"; \n",
    "        if sortedmeta[sortedmeta.index == sample].treatment[0] == \"nitrogen free\": \n",
    "            treatment = \"NF\"\n",
    "            marker = 'p'\n",
    "        else: \n",
    "            treatment = \"Fert\"\n",
    "            marker= \"s\"\n",
    "    lbl = \"%s_%s\" % (species,treatment)\n",
    "    date=sortedmeta[sortedmeta.index == sample].Date[0]\n",
    "    dateColors.append(scolor(index,72,species == \"SG\"))\n",
    "    plt.scatter(npos[index:index+1, 0], npos[index:index+1, 1], color=scolor(index,72,species == \"SG\"), s=s, lw=0,label=None,cmap='viridis',marker=marker)\n",
    "    index += 1\n",
    "\n",
    "for crop,marker in [['Switchgrass Unfertilized','^'],['Switchgrass Fertilized','v'],['Miscanthus Unfertilized','p'],['Miscanthus Fertilized','s'],]:\n",
    "    midval = 63/2\n",
    "    if \"Switchgrass\" not in crop: midval = 72 + 72/2\n",
    "    print(crop, scolor(midval,72, \"SG\" in crop))\n",
    "    plt.scatter([], [], c=scolor(midval,72, \"Switchgrass\" in crop), label=crop, marker=marker)\n",
    "plt.legend(scatterpoints=1, frameon=True, labelspacing=1, title='')\n",
    "#handles, labels = ax.get_legend_handles_labels()\n",
    "# plt.axis(aspect='equal')\n",
    "#cmap = mpl.colors.ListedColormap(dateColors)\n",
    "#cb1 = mpl.colorbar.ColorbarBase(ax, cmap=cmap, orientation='vertical')\n",
    "#cb1.set_label('Some Units')\n",
    "#psm = ax.pcolormesh(data, cmap=cmap, rasterized=True, vmin=-4, vmax=4)\n",
    "#fig.colorbar(cmap, ax=ax)\n",
    "#plt.colorbar(cmap, label='Time')\n",
    "# plt.clim(3, 7)\n",
    "# sort both labels and handles by labels\n",
    "# labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0]))\n",
    "# ax.legend(handles, labels)\n",
    "#plt.legend(handles, labels,scatterpoints=1, loc='best', shadow=False)\n",
    "plt.title(\"Switchgrass and Miscanthus Fertilized & Unfertilized NMDS\")\n",
    "# # plt.tight_layout()\n",
    "plt.savefig(\"figures/AllCombinedNMDS_genes.png\",bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allCounts = allCounts.transpose()\n",
    "allCounts.to_csv(\"stats/filteredCountTable.tsv\",sep='\\t')        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import cm\n",
    "from matplotlib.colors import ListedColormap, LinearSegmentedColormap\n",
    "viridis = cm.get_cmap('viridis', 256)\n",
    "newcolors = viridis(np.linspace(0, 1, 256))\n",
    "pink = np.array([248/256, 24/256, 148/256, 1])\n",
    "newcolors[:25, :] = pink\n",
    "newcmp = ListedColormap(newcolors)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Look at annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "annoMap = {}\n",
    "revMap = {}\n",
    "for line in open(\"annotations/annotationDescripts.tsv\"):\n",
    "    contID, function = line.strip().split(\"\\t\")\n",
    "#     if contID in annoMap:\n",
    "#         print(contID,function,annoMap[contID])\n",
    "#         break\n",
    "    annoMap[contID]= function\n",
    "    try:revMap[function].add(contID)\n",
    "    except:revMap[function] = set([contID])\n",
    "dist = {}\n",
    "for function,contigs in revMap.items():\n",
    "    dist[function]=len(contigs)\n",
    "dist = Series(dist)\n",
    "dist.plot.hist();\n",
    "dump(revMap,open(\"pickles/functionMap.p\",\"wb\"))\n",
    "dump(annoMap,open(\"pickles/annoMap.p\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist[dist>900]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allCounts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "revMap = load(open(\"pickles/functionMap.p\",\"rb\"))\n",
    "list(revMap.keys())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pickle import load,dump\n",
    "allCounts = load(open(\"pickles/allCounts_genes.p\",\"rb\"))\n",
    "annoMap = load(open(\"pickles/annoMap.p\",\"rb\"))\n",
    "revMap = load(open(\"pickles/functionMap.p\",\"rb\"))\n",
    "\n",
    "excCounter = 0\n",
    "geneCounts = {}\n",
    "for index,sample in enumerate(allCounts.columns):\n",
    "    geneCounts[sample]={}\n",
    "    sampleCounts = allCounts[sample]\n",
    "    print(\"%i. %s\" % (index+1,sample))\n",
    "    for function,contigs in revMap.items():\n",
    "        functCounts = sampleCounts[sampleCounts.index.isin(contigs)]\n",
    "        nReads = float(functCounts.sum(axis = 0))\n",
    "        geneCounts[sample][function] = nReads\n",
    "#         try: geneCounts[sample][function] = nReads/len(functCounts)\n",
    "#         except:\n",
    "#             excCounter +=1\n",
    "#             geneCounts[sample][function] = 0.0\n",
    "dump(geneCounts,open(\"pickles/pooledGeneCounts.p\",\"wb\"))\n",
    "print(\"There were %i genes with no representative contigs\" % (excCounter))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "geneCounts = DataFrame(load(open(\"pickles/pooledGeneCounts.p\",\"rb\")))\n",
    "print(len(geneCounts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "geneCounts.to_csv(\"tables/GeneCounts.tsv\",sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = DataFrame(geneCounts)\n",
    "remove = set()\n",
    "\n",
    "for function in df.index:\n",
    "    functionCount = float(df[df.index == function].sum(axis=1)) \n",
    "    if(functionCount == 0.0): remove.add(function)\n",
    "print(len(remove))\n",
    "df.drop(remove,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "distlist = []\n",
    "for function in df.index:distlist.append(float(df[df.index == function].sum(axis=1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist = Series(distlist)\n",
    "dist = dist[dist < dist.mean()*.001*dist.std()]\n",
    "dist.plot.hist()\n",
    "dist.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "diamondAnnos = {}\n",
    "for line in open(\"KEGG_tools_out/diamondAnnotations_0_KOtable.txt\"):\n",
    "    rec = line.split(\"\\t\")\n",
    "    try: diamondAnnos[rec[0]].add(rec[2])\n",
    "    except:  diamondAnnos[rec[0]] = set([rec[2]])\n",
    "print(len(diamondAnnos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist,conts = [],[]\n",
    "for contID, KOList in diamondAnnos.items(): conts.append(contID); dist.append(len(KOList))\n",
    "dist = Series(dist,index=conts)\n",
    "dist.plot.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(dist[dist>=2])/float(len(dist))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allCounts = load(open(\"pickles/allCounts_genes.p\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "medianReplicateGeneCounts = {}\n",
    "for crop in set(metadata.type):\n",
    "    for date in set(metadata.sampling_date):\n",
    "        for treatment in set(metadata.treatment):\n",
    "            replicateSamples = metadata.loc[(metadata.sampling_date==date) & (metadata.treatment==treatment) & (metadata.type == crop)]\n",
    "            if len(replicateSamples) == 0: continue\n",
    "#             print(crop,date,treatment)\n",
    "            replicateCounts = geneCounts[list(replicateSamples.nucleic_acid_name)]\n",
    "            groupID = list(replicateSamples.nucleic_acid_name)[1].replace(\"R1\",\"\").replace(\"R2\",\"\").replace(\"R3\",\"\").replace(\"R4\",\"\")\n",
    "            medianReplicateGeneCounts[groupID] = replicateCounts.median(axis=1)\n",
    "            #             \n",
    "#             sampleGroup[groupID] = {}\n",
    "#             for function, contigList in revMap.items():\n",
    "#                 functionCounts = sampleCounts[sampleCounts.index.isin(contigList)]\n",
    "#                 sampleGroup[groupID][function] = functionCounts.sum().median()\n",
    "medianReplicateGeneCounts = DataFrame(medianReplicateGeneCounts)\n",
    "medianReplicateGeneCounts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove = set()\n",
    "for function in medianReplicateGeneCounts.index:\n",
    "    if medianReplicateGeneCounts[medianReplicateGeneCounts.index == function].sum(axis=1).sum()== 0:remove.add(function)\n",
    "len(remove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(medianReplicateGeneCounts.shape)\n",
    "medianReplicateGeneCounts.drop(remove,inplace=True)\n",
    "print(medianReplicateGeneCounts.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dump(sampleGroup,open(\"pickles/deepFunctionReplicateCounts.p\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "medianReplicateGeneCounts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates=metadata[[\"sampling_date\",\"Date\",\"nucleic_acid_name\"]]\n",
    "dates=dates.sort_values(by=\"Date\")\n",
    "dates=dates[[\"sampling_date\",\"nucleic_acid_name\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata = metadata.sort_values(by=\"Date\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata.treatment.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "persistent = set()\n",
    "crop =\"G5\"\n",
    "treatment = 'standard fertilization'\n",
    "prevTimePoint = []\n",
    "seenGenes =set()\n",
    "for crop in metadata.type.unique():\n",
    "    for treatment in metadata.treatment.unique():\n",
    "        for date in dates.sampling_date.unique():\n",
    "            replicateSamples = metadata.loc[(metadata.sampling_date==date) & (metadata.treatment==treatment) & (metadata.type == crop)]\n",
    "            if len(replicateSamples) == 0: continue\n",
    "            groupID = list(replicateSamples.nucleic_acid_name)[1].replace(\"R1\",\"\").replace(\"R2\",\"\").replace(\"R3\",\"\").replace(\"R4\",\"\")\n",
    "            if len(prevTimePoint) == 0:\n",
    "                prevTimePoint = medianReplicateGeneCounts[groupID]\n",
    "                prevTimePoint = prevTimePoint[prevTimePoint>0]\n",
    "                persistent = set(prevTimePoint.index)\n",
    "        #         print(\"There are %i genes present at start %s\" % (len(prevTimePoint),str(date)))\n",
    "                continue\n",
    "            curTimePoint = medianReplicateGeneCounts[groupID]\n",
    "            curTimePoint = curTimePoint[curTimePoint>0]\n",
    "        #     print(\"At the next time point %s there are %i genes\"%(str(date),len(curTimePoint)))\n",
    "        #     print(\"\\t%i genes existed in the last timepoint\" % (len(set(curTimePoint.index).intersection(prevTimePoint.index))))\n",
    "        #     print(\"\\t%i genes are new\" % (len(set(curTimePoint.index).difference(prevTimePoint.index))))\n",
    "            seenGenes = seenGenes.union(set(curTimePoint.index).union(prevTimePoint.index))\n",
    "        #     print(\"\\t%i genes so far\\n\" % (len(seenGenes)))\n",
    "            persistent = persistent.intersection(curTimePoint.index)\n",
    "            prevTimePoint = curTimePoint\n",
    "        print(\"There are %i genes that persist over time for %s with %s\" % (len(persistent),crop,treatment))\n",
    "        persIncreasing = persistent\n",
    "        persDecreasing = persistent\n",
    "        prevTimePoint = []\n",
    "        seenGenes =set()\n",
    "        for i,date in enumerate(dates.sampling_date.unique()):\n",
    "            replicateSamples = metadata.loc[(metadata.sampling_date==date) & (metadata.treatment==treatment) & (metadata.type == crop)]\n",
    "            if len(replicateSamples) == 0: continue\n",
    "            groupID = list(replicateSamples.nucleic_acid_name)[1].replace(\"R1\",\"\").replace(\"R2\",\"\").replace(\"R3\",\"\").replace(\"R4\",\"\")\n",
    "            if len(prevTimePoint) == 0:\n",
    "                prevTimePoint = medianReplicateGeneCounts[groupID]\n",
    "                prevTimePointDec = prevTimePoint[prevTimePoint.index.isin(persistent)]\n",
    "                prevTimePointInc = prevTimePoint[prevTimePoint.index.isin(persistent)]\n",
    "                continue\n",
    "            prevTimePointInc = prevTimePointInc[prevTimePointInc.index.isin(persIncreasing)]\n",
    "            prevTimePointDec = prevTimePointDec[prevTimePointDec.index.isin(persDecreasing)]\n",
    "            curTimePoint = medianReplicateGeneCounts[groupID]\n",
    "            curIncreasing = curTimePoint[curTimePoint.index.isin(persIncreasing)]\n",
    "            curDecreasing = curTimePoint[curTimePoint.index.isin(persDecreasing)]\n",
    "            increasing = curIncreasing[curIncreasing >= prevTimePointInc]\n",
    "            decreasing = curDecreasing[curDecreasing <= prevTimePointDec]\n",
    "            persIncreasing = set(increasing.index)\n",
    "            persDecreasing = set(decreasing.index)\n",
    "            print(\"\\tAt time %s \\t %i genes are still increasing\\t\" % (str(date), len(persIncreasing)),end='\\t')\n",
    "            print(\"%i genes are still decreasing\" % ( len(persDecreasing)))\n",
    "            \n",
    "            #Add normalization by number of reads"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,date in enumerate(dates.sampling_date.unique()):print(date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "persistent = set()\n",
    "crop =\"G6\"\n",
    "prevTimePoint = []\n",
    "seenGenes =set()\n",
    "for date in dates.sampling_date.unique():\n",
    "    replicateSamples = metadata.loc[(metadata.sampling_date==date) & (metadata.treatment==treatment) & (metadata.type == crop)]\n",
    "    if len(replicateSamples) == 0: continue\n",
    "    groupID = list(replicateSamples.nucleic_acid_name)[1].replace(\"R1\",\"\").replace(\"R2\",\"\").replace(\"R3\",\"\").replace(\"R4\",\"\")\n",
    "    if len(prevTimePoint) == 0:\n",
    "        prevTimePoint = medianReplicateGeneCounts[groupID]\n",
    "        prevTimePoint = prevTimePoint[prevTimePoint>0]\n",
    "        persistent = set(prevTimePoint.index)\n",
    "        print(\"There are %i genes present at start %s\" % (len(prevTimePoint),str(date)))\n",
    "        continue\n",
    "    curTimePoint = medianReplicateGeneCounts[groupID]\n",
    "    curTimePoint = curTimePoint[curTimePoint>0]\n",
    "    print(\"At the next time point %s there are %i genes\"%(str(date),len(curTimePoint)))\n",
    "    print(\"\\t%i genes existed in the last timepoint\" % (len(set(curTimePoint.index).intersection(prevTimePoint.index))))\n",
    "    print(\"\\t%i genes are new\" % (len(set(curTimePoint.index).difference(prevTimePoint.index))))\n",
    "    seenGenes = seenGenes.union(set(curTimePoint.index).union(prevTimePoint.index))\n",
    "    print(\"\\t%i genes so far\\n\" % (len(seenGenes)))\n",
    "    persistent = persistent.intersection(curTimePoint.index)\n",
    "    prevTimePoint = curTimePoint\n",
    "print(\"There are %i genes that persist over time\" % (len(persistent)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What genes consistently go up over time?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "persIncreasing = persistent\n",
    "persDecreasing = persistent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# persistent\n",
    "persIncreasing = persistent\n",
    "persDecreasing = persistent\n",
    "prevTimePoint = []\n",
    "seenGenes =set()\n",
    "print(crop,treatment)\n",
    "for i,date in enumerate(dates.sampling_date.unique()):\n",
    "    replicateSamples = metadata.loc[(metadata.sampling_date==date) & (metadata.treatment==treatment) & (metadata.type == crop)]\n",
    "    if len(replicateSamples) == 0: continue\n",
    "    groupID = list(replicateSamples.nucleic_acid_name)[1].replace(\"R1\",\"\").replace(\"R2\",\"\").replace(\"R3\",\"\").replace(\"R4\",\"\")\n",
    "#     print(groupID)\n",
    "    if len(prevTimePoint) == 0:\n",
    "        prevTimePoint = medianReplicateGeneCounts[groupID]\n",
    "        prevTimePointDec = prevTimePoint[prevTimePoint.index.isin(persistent)]\n",
    "        prevTimePointInc = prevTimePoint[prevTimePoint.index.isin(persistent)]\n",
    "        continue\n",
    "    \n",
    "    prevTimePointInc = prevTimePointInc[prevTimePointInc.index.isin(persIncreasing)]\n",
    "    prevTimePointDec = prevTimePointDec[prevTimePointDec.index.isin(persDecreasing)]\n",
    "    curTimePoint = medianReplicateGeneCounts[groupID]\n",
    "    curIncreasing = curTimePoint[curTimePoint.index.isin(persIncreasing)]\n",
    "    curDecreasing = curTimePoint[curTimePoint.index.isin(persDecreasing)]\n",
    "    increasing = curIncreasing[curIncreasing >= prevTimePointInc]\n",
    "    decreasing = curDecreasing[curDecreasing <= prevTimePointDec]\n",
    "    persIncreasing = set(increasing.index)\n",
    "    persDecreasing = set(decreasing.index)\n",
    "    print(\"At time %i - %i genes are still increasing\" % (i, len(persIncreasing)),end='\\t')\n",
    "    print(\"At time %i - %i genes are still decreasing\" % (i, len(persDecreasing)))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "persIncreasing = persistent\n",
    "persDecreasing = persistent\n",
    "prevTimePoint = []\n",
    "seenGenes =set()\n",
    "print(crop,treatment)\n",
    "for i,date in enumerate(dates.sampling_date.unique()):\n",
    "    replicateSamples = metadata.loc[(metadata.sampling_date==date) & (metadata.treatment==treatment) & (metadata.type == crop)]\n",
    "    if len(replicateSamples) == 0: continue\n",
    "    groupID = list(replicateSamples.nucleic_acid_name)[1].replace(\"R1\",\"\").replace(\"R2\",\"\").replace(\"R3\",\"\").replace(\"R4\",\"\")\n",
    "#     print(groupID)\n",
    "    if len(prevTimePoint) == 0:\n",
    "        prevTimePoint = medianReplicateGeneCounts[groupID]\n",
    "        prevTimePointDec = prevTimePoint[prevTimePoint.index.isin(persistent)]\n",
    "        prevTimePointInc = prevTimePoint[prevTimePoint.index.isin(persistent)]\n",
    "        continue\n",
    "    \n",
    "    prevTimePointInc = prevTimePointInc[prevTimePointInc.index.isin(persIncreasing)]\n",
    "    prevTimePointDec = prevTimePointDec[prevTimePointDec.index.isin(persDecreasing)]\n",
    "    curTimePoint = medianReplicateGeneCounts[groupID]\n",
    "    curIncreasing = curTimePoint[curTimePoint.index.isin(persIncreasing)]\n",
    "    curDecreasing = curTimePoint[curTimePoint.index.isin(persDecreasing)]\n",
    "    increasing = curIncreasing[curIncreasing >= prevTimePointInc]\n",
    "    decreasing = curDecreasing[curDecreasing <= prevTimePointDec]\n",
    "    persIncreasing = set(increasing.index)\n",
    "    persDecreasing = set(decreasing.index)\n",
    "    print(\"At time %i - %i genes are still increasing\" % (i, len(persIncreasing)),end='\\t')\n",
    "    print(\"At time %i - %i genes are still decreasing\" % (i, len(persDecreasing)))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "startTime = medianReplicateGeneCounts['G5_NF_09MAY2016_LD1']\n",
    "startTime = startTime[startTime > 0]\n",
    "print(len(startTime))\n",
    "startTime.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "curTimePoint[curTimePoint.index.isin(persDecreasing)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seqMappedDist = Series(load(open(\"pickles/seqMappedCounter.p\",\"rb\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "presenceDist=presenceDist/float(numSamples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fh = open(\"assemblies/parts/final.contigs.%i.fa\" % (index),\"w\")\n",
    "from Bio.SeqIO import parse, write\n",
    "counter,index = 0, 0\n",
    "for rec in parse(\"assemblies/final.contigs.fa\",\"fasta\"): \n",
    "    if counter % 500000 == 0:\n",
    "        #fh.close()\n",
    "        index += 1\n",
    "        print(index,end=\" \")\n",
    "        #fh = open(\"assemblies/parts/final.contigs.%i.fa\" % (index),\"w\")\n",
    "    #write(rec,fh,\"fasta\")\n",
    "    counter+=1\n",
    "#fh.close()     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "#ls -laht pickles\n",
    "#head stats/11425.5.206700.GCCTTGT-AACAAGG.fastq.gz.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Initialize the dictionary for the counts\n",
    "seqCounter = {} #load(open(\"pickles/seqCounter.p\",\"rb\"))\n",
    "combined_abundance = seqCounter.copy()\n",
    "for crop in set(metadata.type):\n",
    "    for date in set(metadata.sampling_date):\n",
    "        for treatment in set(metadata.treatment):\n",
    "            sampleGroupMeta = metadata.loc[(metadata.sampling_date==date) & (metadata.treatment==treatment) & (metadata.type == crop)]\n",
    "            if len(sampleGroupMeta) == 0: continue\n",
    "            sampleAbundance = seqCounter.copy()\n",
    "            samplePresence = seqCounter.copy()\n",
    "            comFileName = \"stats/combined/%s_%s.tsv\" % (sampleGroupMeta.plot_name.unique()[0],date.replace(\"/\",\"_\"))\n",
    "            print(comFileName)\n",
    "            outfile = open(comFileName, \"w\")\n",
    "            nfiles = 4\n",
    "            for fstaName in sampleGroupMeta.index:\n",
    "                try:\n",
    "                    with open(\"stats/%s.tsv\" % (fstaName)) as fh:\n",
    "                        for line in fh:\n",
    "                            rec = line.strip().split()\n",
    "                            sampleAbundance[rec[0]] += int(rec[2])\n",
    "                            samplePresence[rec[0]]  += int(int(rec[2])>0)\n",
    "                except:\n",
    "                    print(\"\\t\\t\",comFileName,\"is missing file:\",fstaName)\n",
    "                    missingStats.append(fstaName)\n",
    "                    nfiles-=1\n",
    "                    \n",
    "            contigCounter = 0\n",
    "            for contName,presCount in samplePresence.items():\n",
    "                contigCounter += int(presCount >= 2)\n",
    "                outfile.write(\"%s\\t%s\\n\" % (contName,int(sampleAbundance[contName]/nfiles)))\n",
    "                combined_abundance[contName] += int(sampleAbundance[contName]/nfiles)\n",
    "            print(\"\\t\",contigCounter,\"Contigs Present\")\n",
    "            outfile.close()\n",
    "dump(combined_abundance,open(\"pickles/combined_abundance.p\",\"wb\"))\n",
    "dump(missingStats,open(\"pickles/missingStats.p\",\"wb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><a id=\"pltao\">Plot abundance and occupancy</a></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now that the files are combined, let's look at p/a and abundance\n",
    "fileNames = glob(\"stats/combined/*.tsv\")\n",
    "presAbs,abundance = {},{}\n",
    "for fname in fileNames:\n",
    "    print(fname)\n",
    "    for line in open(fname): \n",
    "        rec = line.strip().split()\n",
    "        try:presAbs[rec[0]] += int(int(rec[1]) > 0)\n",
    "        except:presAbs[rec[0]] = int(int(rec[1]) > 0)\n",
    "        try:abundance[rec[0]] += int(rec[1])\n",
    "        except:abundance[rec[0]] = int(rec[1])\n",
    "\n",
    "dump(presAbs,open(\"presence_absence_combined.p\",\"wb\"))    \n",
    "dump(abundance,open(\"abundance_combined.p\",\"wb\"))   \n",
    "dist_pres = []\n",
    "numSamples = float(len(fileNames))\n",
    "for contig, count in presAbs.items(): dist_pres.append(count/float(numSamples))\n",
    "plotter = Series(dist_pres)\n",
    "plotter.plot.hist();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><a id=\"pltao\">Abundance</a></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combinedFiles = glob(\"stats/combined/filtered/*.tsv\")\n",
    "# presAbs = load(open(\"pickles/presence_absence_combined.p\",\"rb\"))    \n",
    "# abundance = load(open(\"pickles/abundance_combined.p\",\"rb\"))   \n",
    "# dist_pres = []\n",
    "# numSamples = float(len(combinedFiles))\n",
    "# for contig, count in presAbs.items(): dist_pres.append(count/float(numSamples))\n",
    "# plotter = Series(dist_pres)\n",
    "ax = plotter.plot.hist()  # s is an instance of Series\n",
    "fig = ax.get_figure()\n",
    "fig.savefig('figures/Abundance_combined.png')\n",
    "ax;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3><a id=\"OXA\">Occupancy X Abundance</a></h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "xs, ys =[], []\n",
    "for i in range(1,1000):\n",
    "    xs.append(i/1000.0)\n",
    "    ys.append(len(plotter[plotter > i/1000.0]))\n",
    "plt.plot(xs,ys)\n",
    "plt.xlabel('Percent of samples',fontsize=14)\n",
    "plt.ylabel('Rank Abundance',fontsize=14)\n",
    "plt.savefig(\"figures/OccupencyAbundance.png\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "xs, ys =[], []\n",
    "for contName in presAbs:\n",
    "    ys.append(presAbs[contName]/numSamples)\n",
    "    xs.append(abundance[contName])\n",
    "plt.plot(xs,ys,'ro')\n",
    "plt.xlabel('Abundance',fontsize=14)\n",
    "plt.ylabel('% Presence',fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2><a id=\"pltao\">Filter the data by occupency and abundance</a></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numSamples = float(len(fileNames))\n",
    "# keeperContigs = set()\n",
    "# for contig, count in presAbs.items(): \n",
    "#     if count/float(numSamples) >= .25: keeperContigs.add(contig)\n",
    "print (\"Keeping %i contigs\" % (len(keeperContigs)))\n",
    "\n",
    "fileNames = glob(\"stats/combined/*.tsv\")\n",
    "presAbs,abundance = {},{}\n",
    "\n",
    "print(\"%i combined samples\"%(int(numSamples)))\n",
    "counter=0\n",
    "for fname in fileNames:\n",
    "    counter += 1\n",
    "    print(\"%i.\" % counter,fname)\n",
    "    ffh =  open(fname.replace(\"stats/combined/\",\"stats/combined/filtered/\"),\"w\")\n",
    "    for line in open(fname): \n",
    "        rec = line.strip().split()\n",
    "        if rec[0] in keeperContigs: \n",
    "            ffh.write(\"%s\\t%s\\n\" % (rec[0],rec[1]))\n",
    "            try:presAbs[rec[0]]     += int(int(rec[1]) > 0)\n",
    "            except:presAbs[rec[0]]   = int(int(rec[1]) > 0)\n",
    "            try:abundance[rec[0]]   += int(rec[1])\n",
    "            except:abundance[rec[0]] = int(rec[1])\n",
    "    ffh.close()\n",
    "    \n",
    "import matplotlib.pyplot as plt\n",
    "xs, ys =[], []\n",
    "for contName in presAbs:\n",
    "    xs.append(presAbs[contName]/numSamples)\n",
    "    ys.append(abundance[contName])\n",
    "plt.plot(xs,ys,'ro')\n",
    "plt.xlabel('Percent Abundance',fontsize=14)\n",
    "plt.ylabel('Abundance',fontsize=14)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fileNames = glob(\"stats/combined/filtered/*.tsv\")\n",
    "allCounts = {}\n",
    "counter=0\n",
    "for fname in fileNames:\n",
    "    sampleName = fname.replace(\"stats/combined/filtered/\",\"\").replace(\".tsv\",\"\")\n",
    "    allCounts[sampleName]={}\n",
    "    counter+=1\n",
    "    print(\"%i. %s\"%(counter,sampleName))\n",
    "    for line in open(fname):\n",
    "        rec=line.strip().split()\n",
    "        allCounts[sampleName][rec[0]]=int(rec[1])\n",
    "dump(allCounts,open(\"pickles/sampleDict.p\",\"wb\"))    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2><a id=\"nmds\">NMDS</a></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleOrder = {}\n",
    "counter = 0\n",
    "dates=metadata[[\"sampling_date\",\"Date\"]]\n",
    "dates=dates.sort_values(by=\"Date\")\n",
    "dates=dates[\"sampling_date\"]\n",
    "    \n",
    "for date in dates.unique():\n",
    "    for crop in set(metadata.type):\n",
    "        for treatment in set(metadata.treatment):\n",
    "            sampleGroupMeta = metadata.loc[(metadata.sampling_date==date) & (metadata.treatment==treatment) & (metadata.type == crop)]\n",
    "            if len(sampleGroupMeta) == 0: continue\n",
    "            idName = \"%s_%s\" % (sampleGroupMeta.plot_name.unique()[0],date.replace(\"/\",\"_\"))\n",
    "            counter+=1\n",
    "            sampleOrder[idName] = counter\n",
    "\n",
    "print(\"Loading sample counts\")\n",
    "allCounts = load(open(\"pickles/sampleDict.p\",\"rb\"))\n",
    "allCounts = DataFrame(allCounts).transpose()\n",
    "allCounts['Rank'] = allCounts.index.to_series().map(sampleOrder)\n",
    "allCounts.sort_values('Rank',inplace = True)\n",
    "allCounts.drop('Rank', 1, inplace = True)\n",
    "allCounts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = np.random.RandomState(seed=3)\n",
    "allCounts_matrix = allCounts.as_matrix()\n",
    "similarities = euclidean_distances(allCounts_matrix)\n",
    "\n",
    "nsamples = len(allCounts)\n",
    "mds = manifold.MDS(n_components=2, max_iter=3000, eps=1e-9, random_state=seed, dissimilarity=\"precomputed\", n_jobs=1)\n",
    "pos = mds.fit(similarities).embedding_\n",
    "nmds = manifold.MDS(n_components=2, metric=False, max_iter=3000, eps=1e-12, dissimilarity=\"precomputed\", random_state=seed, n_jobs=1, n_init=1)\n",
    "npos = nmds.fit_transform(similarities, init=pos)\n",
    "\n",
    "fig = plt.figure(1)\n",
    "ax = plt.axes([0., 0., 1.5, 1.95])\n",
    "s = 100\n",
    "counter = 0\n",
    "group = 0\n",
    "totalGroups = nsamples/4\n",
    "l1s, l2s = [], []\n",
    "for date in dates.unique():\n",
    "    group+=1\n",
    "    l1s.append(\"SG-Fert \"+date)\n",
    "    plt.scatter(npos[counter:counter+1, 0], npos[counter:counter+1, 1], color=scolor(group,totalGroups,True), s=s, lw=0, label=\"SG-Fert \"+date,marker=\"^\")\n",
    "    counter+=1\n",
    "    l1s.append(\"SG-NF \"+date)\n",
    "    plt.scatter(npos[counter:counter+1, 0], npos[counter:counter+1, 1], color=scolor(group,totalGroups,True), s=s, lw=0, label=\"SG-NF \"+date,marker=\"v\")\n",
    "    counter+=1\n",
    "    l2s.append(\"MC-Fert \"+date)\n",
    "    plt.scatter(npos[counter:counter+1, 0], npos[counter:counter+1, 1], color=scolor(group,totalGroups,False), s=s, lw=0, label=\"MC-Fert \"+date,marker=\"p\")\n",
    "    counter+=1\n",
    "    l2s.append(\"MC-NF \"+date)\n",
    "    plt.scatter(npos[counter:counter+1, 0], npos[counter:counter+1, 1], color=scolor(group,totalGroups,False), s=s, lw=0, label=\"MC-NF \"+date,marker=\"H\")\n",
    "    counter+=1\n",
    "\n",
    "handles, labels = ax.get_legend_handles_labels()\n",
    "# sort both labels and handles by labels\n",
    "# labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0]))\n",
    "# ax.legend(handles, labels)\n",
    "plt.legend(handles, labels,scatterpoints=1, loc='best', shadow=False)\n",
    "plt.title(\"Switchgrass(SG) and Miscanthus(MC) Fertilized(Fert) & Not Ferilized(NF) NMDS\")\n",
    "# plt.tight_layout()\n",
    "plt.savefig(\"figures/AllCombinedNMDS.png\",bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g6Data, g5Data = [],[]\n",
    "for key in allCounts.index:\n",
    "    if key[:2] == \"G5\": g5Data.append(key)\n",
    "    else: g6Data.append(key)\n",
    "g5Data = allCounts.drop(g5Data)\n",
    "g6Data = allCounts.drop(g6Data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g5Data_matrix = g5Data.as_matrix()\n",
    "similarities = euclidean_distances(g5Data_matrix)\n",
    "nsamples = len(g5Data)\n",
    "mds = manifold.MDS(n_components=2, max_iter=3000, eps=1e-9, random_state=seed, dissimilarity=\"precomputed\", n_jobs=1)\n",
    "pos = mds.fit(similarities).embedding_\n",
    "nmds = manifold.MDS(n_components=2, metric=False, max_iter=3000, eps=1e-12, dissimilarity=\"precomputed\", random_state=seed, n_jobs=1, n_init=1)\n",
    "npos = nmds.fit_transform(similarities, init=pos)\n",
    "\n",
    "fig = plt.figure(1)\n",
    "ax = plt.axes([0., 0., 1.5, 1.95])\n",
    "s = 100\n",
    "\n",
    "for index, sample in enumerate(g5Data.index):\n",
    "    species = \"MC\"; \n",
    "    if sortedmeta[sortedmeta.nucleic_acid_name == sample].treatment[0] == \"nitrogen free\": \n",
    "        treatment = \"NF\"\n",
    "        marker = 'p'\n",
    "    else: \n",
    "        treatment = \"Fert\"\n",
    "        marker= \"s\"\n",
    "            \n",
    "    lbl = \"%s_%s\" % (species,treatment)\n",
    "    plt.scatter(npos[index:index+1, 0], npos[index:index+1, 1], color=scolor(index+71,72,species == \"SG\"), s=s, lw=0, label=lbl,marker=marker)\n",
    "\n",
    "#plt.legend(scatterpoints=1, loc='best', shadow=False)\n",
    "plt.title(\"Miscanthus (Fert) and Not Ferilized (NF)\")\n",
    "plt.savefig(\"figures/Miscanthus_NMDS_genes.png\", bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g6Data_matrix = g6Data.as_matrix()\n",
    "similarities = euclidean_distances(g6Data_matrix)\n",
    "nsamples = len(g6Data)\n",
    "mds = manifold.MDS(n_components=2, max_iter=3000, eps=1e-9, random_state=seed, dissimilarity=\"precomputed\", n_jobs=1)\n",
    "pos = mds.fit(similarities).embedding_\n",
    "nmds = manifold.MDS(n_components=2, metric=False, max_iter=3000, eps=1e-12, dissimilarity=\"precomputed\", random_state=seed, n_jobs=1, n_init=1)\n",
    "npos = nmds.fit_transform(similarities, init=pos)\n",
    "\n",
    "fig = plt.figure(1)\n",
    "ax = plt.axes([0., 0., 1.5, 1.5])\n",
    "s = 100\n",
    "counter = 0\n",
    "group=0\n",
    "for index, sample in enumerate(g6Data.index):\n",
    "    species = \"SG\"; \n",
    "    if sortedmeta[sortedmeta.nucleic_acid_name == sample].treatment[0] == \"nitrogen free\": \n",
    "        treatment = \"NF\"\n",
    "        marker = '^'\n",
    "    else: \n",
    "        treatment = \"Fert\"\n",
    "        marker= \"v\"   \n",
    "    lbl = \"%s_%s\" % (species,treatment)\n",
    "    plt.scatter(npos[index:index+1, 0], npos[index:index+1, 1], color=scolor(index,72,species == \"SG\"), s=s, lw=0, label=lbl,marker=marker)\n",
    "\n",
    "#plt.legend(scatterpoints=1, loc='best', shadow=False)\n",
    "plt.title(\"Switchgrass Fertilized (Fert) and Not Ferilized (NF)\")\n",
    "plt.savefig(\"figures/Switchgrass_NMDS_genes.png\", bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1><a id=\"metaT\">Function Analysis</a></h1>\n",
    "\n",
    "[Home](#meta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "KO_Levels = DataFrame.from_csv(\"annotations/KO_Levels.tsv\",sep='\\t')\n",
    "KO_Levels['Level1'] = KO_Levels['Level1'].apply(lambda x: str(x).zfill(5))\n",
    "KO_Levels['Level2'] = KO_Levels['Level2'].apply(lambda x: str(x).zfill(5))\n",
    "KO_Levels['Level3'] = KO_Levels['Level3'].apply(lambda x: str(x).zfill(5))\n",
    "KO_Levels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contigKOs = {}\n",
    "koContigs = {}\n",
    "for line in open(\"annotations/KEGG_tools_out/diamondAnnotations_0_KOtable.txt\"):\n",
    "    rec = line.split('\\t')\n",
    "    try: contigKOs[rec[0]].add(rec[2])\n",
    "    except: contigKOs[rec[0]] = set([rec[2]])\n",
    "    try: koContigs[rec[2]].add(rec[0])\n",
    "    except: koContigs[rec[2]] = set([rec[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "allCounts = load(open(\"pickles/allCounts_genes.p\",\"rb\")) \n",
    "ko_mapper = load(open(\"pickles/koMap.p\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from numpy import log2\n",
    "levels = KO_Levels.Level1.unique()\n",
    "Level3DF = DataFrame(index=levels, columns=allCounts.columns)\n",
    "Level3DF = Level3DF.fillna(0)\n",
    "for level in levels:\n",
    "    selectedKOs = KO_Levels[KO_Levels.Level1 == level]\n",
    "    selectedContigs = set()\n",
    "    for ko in selectedKOs.index: selectedContigs = selectedContigs.union(koContigs[ko])\n",
    "    selectedCounts = allCounts[allCounts.index.isin(selectedContigs)]\n",
    "    countSums = selectedCounts.sum(axis=0)\n",
    "    countSums = readsForAllContigs\n",
    "    countSums = log2(countSums/readCounts[\"TotalSampleReads\"])\n",
    "    Level3DF.loc[level] = countSums\n",
    "Level3DF[\"Description\"] = Level3DF.index.to_series().map(ko_mapper.descripts)\n",
    "cols = Level3DF.columns.tolist()\n",
    "cols = cols[-1:] + cols[:-1]\n",
    "Level3DF = Level3DF[cols]\n",
    "Level3DF=Level3DF.set_index('Description')\n",
    "Level3DF.to_csv(\"annotations/Level1CountTable.tsv\",sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "readCounts = read_csv(\"mapping/metaG/flagstats/multiqc_data/mqc_bowtie2_pe_plot_1.txt\",sep=\"\\t\")\n",
    "readCounts[\"TotalSampleReads\"] = data.sum(axis=1)\n",
    "readCounts.set_index(\"Sample\",inplace=True)\n",
    "readCounts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KO_Mapper:\n",
    "    def __init__(self):\n",
    "        self.koToMap = {}\n",
    "        self.descripts = {}\n",
    "        self.mapToKO = {}\n",
    "        self.levelToMap = {}\n",
    "        self.ko = \"\"\n",
    "        self.KEGG_URL = \"https://www.kegg.jp/dbget-bin/www_bget?%s\"\n",
    "\n",
    "    def setItem(self, level, mapNum, descrip):\n",
    "        # print(self.ko, level, mapNum, descrip)\n",
    "        # Map descriptions\n",
    "        self.descripts[mapNum] = descrip\n",
    "        \n",
    "        # Map to KO\n",
    "        try: self.mapToKO[mapNum].add(self.ko)\n",
    "        except: self.mapToKO[mapNum] = set([self.ko])\n",
    "        \n",
    "        # KO to level to mapNum\n",
    "        try: self.koToMap[self.ko][level].add(mapNum)\n",
    "        except: \n",
    "            try: self.koToMap[self.ko][level] = set([mapNum])\n",
    "            except: self.koToMap[self.ko] = { level:set([mapNum]) }\n",
    "        \n",
    "        # Level to map\n",
    "        try: self.levelToMap[level].add(mapNum)\n",
    "        except: self.levelToMap[level] = set([mapNum])\n",
    "                \n",
    "    def _processKOInfo(self, koText):\n",
    "        rec = koText.strip().split('\\n')\n",
    "        while 'KEGG Orthology' not in rec[0] and len(rec)>0: rec = rec[1:]\n",
    "        if len(rec)==0:return\n",
    "        for line in rec[1:]:\n",
    "            level = line.count('\\xa0')\n",
    "            if (level == 0) or self.ko in str(line): break #Don't record any ribosome info\n",
    "            info = line.strip('\\xa0')\n",
    "            bIndex = info.find(\" \")\n",
    "            self.setItem(level,info[:bIndex],info[bIndex+1:])\n",
    "    \n",
    "    def mapKO(self,ko):\n",
    "        siteContent = BeautifulSoup(urlopen(self.KEGG_URL%(ko)).read(),features=\"lxml\")\n",
    "        self.ko = ko\n",
    "        found = False\n",
    "        for i, elm in enumerate(siteContent.find_all(\"td\", {\"class\": \"td41\"})):\n",
    "            if \"KEGG Orthology\" in str(elm.contents[0]):\n",
    "                pathway = elm.find(\"nobr\")\n",
    "                self._processKOInfo(pathway.text)\n",
    "                found = True\n",
    "                break\n",
    "        if not found: \n",
    "            for i, elm in enumerate(siteContent.find_all(\"td\", {\"class\": \"td40\"})):\n",
    "                if \"KEGG Orthology\" in str(elm.contents[0]):\n",
    "                    pathway = elm.find(\"nobr\")\n",
    "                    self._processKOInfo(pathway.text)\n",
    "                    found = True\n",
    "                    break\n",
    "        if not found: raise Exception(\"Unable to find \" + self.ko)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DataFrame.read_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#A custom function to help us find the raw fastq files\n",
    "from glob import glob\n",
    "rawFastqFiles = glob('jgi_transfer/metaT_raw/*/Raw_Data/*')\n",
    "\n",
    "def lookupFSTQ(name,rawFastqFiles):\n",
    "    fastqName = ''\n",
    "    found = False\n",
    "    for fastqName in rawFastqFiles:\n",
    "        if name in fastqName: \n",
    "            found = True\n",
    "            break\n",
    "    if found:\n",
    "        rawFastqFiles.remove(fastqName)\n",
    "        return fastqName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "ls metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import DataFrame, to_datetime\n",
    "#Collect Metadata\n",
    "metadata = DataFrame.from_csv(\"metadata/GLBRC_MetaT_Metadata.tsv\",sep='\\t')\n",
    "\n",
    "#Change the data to only include things we are interested in and format a few columns\n",
    "metadata['sampling_date'] = to_datetime(metadata.date) #Make date a format python can sort\n",
    "metadata.drop(['time','air_temp_c', 'day', 'month', 'year', 'weather', 'notes', 'rep', 'date_of_extraction', 'nucleic_acid_type', \n",
    "   'replicate_extraction', 'source', 'source_mass', 'extraction_method', 'elution_vol_ul', 'concentration_ng_per_ul', \n",
    "   'ratio_260_280', 'conc_ng_per_g_source', 'extracted_by', 'sequencing_date', 'conc_sent_ng_per_ul', 'sequencing_type', \n",
    "   'sequencing_facility', 'primers', 'submitted_for_sequencing', 'sequencing_successful', 'duplicate_submitted', 'dup_sequencing_name', \n",
    "   'exclude_from_analysis', 'itemID_JGI', 'sampleID_JGI', 'JGI_rawdataname', 'Air_Pressure', 'RH', 'AH', 'Wind_Speed_Mean', 'PAR', \n",
    "   'soil_temp_5_cm_bare_avg', 'soil_temp_5_cm_sod_avg', 'Year', 'date', 'pseudorep','MMPRNT_ID','time_zone','longitude', 'country',\n",
    "   'location','air_temp_max','Air_Temp_Min','latitude','altitude','plot_name', 'soil_name', 'number_cores', 'Air_temp_mean' ,\n",
    "   'Wind_Direction_Mean','time_numeric','precipitation', 'Solar_Radiation','pH','JGI_taxonOID','JGI_library','SPNL_date','lime_index',\n",
    "   'P_ppm','barcode','K_ppm', 'Ca_ppm', 'Mg_ppm', 'organic_matter', 'NO3N_ppm', 'NH4_ppm', 'soil_moisture_percent', 'soil_temp_10cm', \n",
    "   'plant_name', 'LDMC_mg_per_g', 'nitrogen_percent', 'carbon_percent', 'carbon_per_nitrogen', 'height_mean_cm', 'mass_per_leaf_g',\n",
    "   'name','plotID','sequence_name'],axis=1,inplace=True)\n",
    "metadata = metadata.rename(index=str, columns={'nucleic_acid_name':'name'})\n",
    "\n",
    "#Map Metadata to fastq files\n",
    "metadata['HPCC_path'] = metadata.apply(lambda row: lookupFSTQ(row['name'],rawFastqFiles), axis=1)\n",
    "\n",
    "#Sort the metadata for ordering purposes\n",
    "metadata.sort_values(by=['plant','sampling_date','treatment','name'],inplace=True) #,\"Date\",\"treatment\",\"plot_name\"])\n",
    "\n",
    "#Change the Identifier\n",
    "metadata.set_index('name',inplace=True)\n",
    "\n",
    "#Show me the top 10\n",
    "print(metadata.shape)\n",
    "metadata.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "rawDirs = listdir(\"jgi_transfer/metaT_raw/\") #*/Raw_Data/*\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import system\n",
    "# links = glob(\"mapping/metaT/unpaired/*.fastq.gz\")\n",
    "# for lnk in links: system(\"unlink \"+lnk)\n",
    "from os import listdir\n",
    "rawDirs = listdir(\"jgi_transfer/metaT_raw/\") #*/Raw_Data/*\n",
    "#Make Sym links to put fastqs in same dir\n",
    "from os import system\n",
    "for sample_name in metadata.index:\n",
    "    if metadata.loc[sample_name,'HPCC_path']:\n",
    "        pass\n",
    "#         system(\"ln -s /mnt/research/ShadeLab/GLBRC/%s mapping/metaT/unpaired/%s.fastq.gz\" % (metadata.loc[sample_name,'HPCC_path'],sample_name))\n",
    "    else: \n",
    "        sample_name = sample_name[:sample_name.rfind(\"_\")]\n",
    "        \n",
    "#         for fname in rawDirs:\n",
    "        print(\"Missing metadata for\",sample_name)\n",
    "# for fname in rawFastqFiles: print(\"Missing Metadata:\",fname[fname.find(\"function__\")+10:fname.find(\"_MT_\")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "ls jgi_transfer/metaT_raw/ |wc -l\n",
    "cd jgi_transfer/metaT_raw/*G5R4_NF_12JUL2016*\n",
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "ls mapping/metaT/unpaired/ |wc -l\n",
    "ls mapping/metaT/unpaired/*12JUL2016*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# N-Dimensional Scaling\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.collections import LineCollection\n",
    "from sklearn import manifold\n",
    "from sklearn.metrics import euclidean_distances\n",
    "from sklearn.decomposition import PCA\n",
    "from scipy.spatial import distance\n",
    "\n",
    "n_samples = 20\n",
    "seed = np.random.RandomState(seed=3)\n",
    "X_true = seed.randint(0, 20, 2 * n_samples).astype(np.float)\n",
    "print(X_true)\n",
    "X_true = X_true.reshape((n_samples, 2))\n",
    "\n",
    "# Center the data\n",
    "# X_true -= X_true.mean()\n",
    "\n",
    "similarities = euclidean_distances(X_true)\n",
    "#print(similarities)\n",
    "\n",
    "# Add noise to the similarities\n",
    "noise = np.random.rand(n_samples, n_samples)\n",
    "noise = noise + noise.T\n",
    "noise[np.arange(noise.shape[0]), np.arange(noise.shape[0])] = 0\n",
    "# similarities += noise\n",
    "\n",
    "mds = manifold.MDS(n_components=2, max_iter=3000, eps=1e-9, random_state=seed, dissimilarity=\"precomputed\", n_jobs=1)\n",
    "pos = mds.fit(similarities).embedding_\n",
    "\n",
    "nmds = manifold.MDS(n_components=2, metric=False, max_iter=3000, eps=1e-12, dissimilarity=\"precomputed\", random_state=seed, n_jobs=1, n_init=1)\n",
    "npos = nmds.fit_transform(similarities, init=pos)\n",
    "\n",
    "# Rescale the data\n",
    "# pos *= np.sqrt((X_true ** 2).sum()) / np.sqrt((pos ** 2).sum())\n",
    "# npos *= np.sqrt((X_true ** 2).sum()) / np.sqrt((npos ** 2).sum())\n",
    "\n",
    "# Rotate the data\n",
    "clf = PCA(n_components=2)\n",
    "X_true = clf.fit_transform(X_true)\n",
    "\n",
    "pos = clf.fit_transform(pos)\n",
    "npos = clf.fit_transform(npos)\n",
    "\n",
    "fig = plt.figure(1)\n",
    "ax = plt.axes([0., 0., 1., 1.])\n",
    "\n",
    "s = 100\n",
    "plt.scatter(X_true[:, 0], X_true[:, 1], color='navy', s=s, lw=0, label='True Position')\n",
    "plt.scatter(pos[:, 0], pos[:, 1], color='turquoise', s=s, lw=0, label='MDS')\n",
    "plt.scatter(npos[:, 0], npos[:, 1], color='darkorange', s=s, lw=0, label='NMDS')\n",
    "plt.legend(scatterpoints=1, loc='best', shadow=False)\n",
    "similarities = similarities.max() / similarities * 100\n",
    "similarities[np.isinf(similarities)] = 0\n",
    "\n",
    "# Plot the edges\n",
    "start_idx, end_idx = np.where(pos)\n",
    "# a sequence of (*line0*, *line1*, *line2*), where::\n",
    "#            linen = (x0, y0), (x1, y1), ... (xm, ym)\n",
    "segments = [[X_true[i, :], X_true[j, :]]\n",
    "            for i in range(len(pos)) for j in range(len(pos))]\n",
    "values = np.abs(similarities)\n",
    "lc = LineCollection(segments, zorder=0, cmap=plt.cm.Blues, norm=plt.Normalize(0, values.max()))\n",
    "lc.set_array(similarities.flatten())\n",
    "lc.set_linewidths(np.full(len(segments), 0.5))\n",
    "ax.add_collection(lc)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Bio.SeqIO import parse\n",
    "dist = []\n",
    "for rec in parse(\"assemblies/final.contigs.fa\",\"fasta\"): dist.append(len(rec.seq))\n",
    "dist = Series(dist)\n",
    "dist.plot.hist();\n",
    "print(dist.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist.plot.bar();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from GetOrfs import getOrfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "nfiles = 1\n",
    "fname = \"contigs2/%i.fasta\" % (counter)\n",
    "fh = open(\"contigs2/%i.fasta\" % (counter),\"w\")\n",
    "for rec in parse(\"/mnt/scratch/howead/glbrc/assembly/final.contigs.fa\",\"fasta\"): \n",
    "    write(rec,fh,\"fasta\")\n",
    "    counter+=1\n",
    "    if (counter % 5000 == 0):\n",
    "        fh.close()\n",
    "        getOrfs(fname)\n",
    "        nfiles+=1\n",
    "        fname = \"contigs2/%i.fasta\" % (counter)\n",
    "        fh = open(\"contigs2/%i.fasta\" % (counter),\"w\")\n",
    "fh.close()\n",
    "#pickle.dump(contigNames,open(\"SeqIDs.p\", \"wb\"))\n",
    "#contigNames = pickle.load(open(\"SeqIDs.p\", \"rb\"))\n",
    "#files = glob(\"bams/*.bam\")\n",
    "print(nfiles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "ls -alh pickles/G5R3_NF_31MAY2016_LD1_counts.p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "files = glob(\"stats/*.tsv\")\n",
    "print (\"Number of files:\",len(files))\n",
    "smallFile = open(\"testFile.txt\",\"w\")\n",
    "\n",
    "for statFile in files:\n",
    "    counter = 0\n",
    "    readsFileName = statFile.replace(\"stats/\",\"\").replace(\".tsv\",\"\")\n",
    "    sampleMeta = metadata[metadata.index == readsFileName]\n",
    "    curSampleID = sampleMeta.nucleic_acid_name.values[0]\n",
    "\n",
    "    fname = \"pickles/%s_counts.p\" % (curSampleID)\n",
    "    fh = open(fname,\"rb\")\n",
    "    data = pickle.load(fh)\n",
    "    print(data)\n",
    "    break\n",
    "\n",
    "smallFile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "from os import path\n",
    "sample_data = {}\n",
    "files = glob(\"stats/*.tsv\")\n",
    "print (\"Number of files:\",len(files))\n",
    "counter = 0\n",
    "for statFile in files:\n",
    "    readsFileName = statFile.replace(\"stats/\",\"\").replace(\".tsv\",\"\")\n",
    "    sampleMeta = metadata[metadata.index == readsFileName]\n",
    "    curSampleID = sampleMeta.nucleic_acid_name.values[0]\n",
    "    sample_data={}\n",
    "    counter+=1\n",
    "    print (\"%i. %s\" % (counter, curSampleID))\n",
    "#     if path.exists(\"pickles/%s_counts.p\" % (curSampleID)):continue#\n",
    "#     for line in open(statFile):\n",
    "#         rec = line.strip().split()\n",
    "#         sample_data[rec[0]] = int(rec[2]) #How can an unmapped read be connected to a contig?\n",
    "#     pickle.dump(sample_data,open(\"pickles/%s_counts.p\" % (curSampleID),\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Dimensional Reduction\n",
    "import numpy as np\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "from matplotlib.collections import LineCollection\n",
    "from sklearn import manifold\n",
    "from sklearn.metrics import euclidean_distances\n",
    "from sklearn.decomposition import PCA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def checkSkipRead(read,reads):\n",
    "    # 1. Read is unmapped\n",
    "    # 2. Read pair has already been seen \n",
    "    # 3. Read doesn't map well\n",
    "    return read.is_unmapped or read.qname in reads or read.mapping_quality <= 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = 0\n",
    "\n",
    "for bamFileName in files:\n",
    "    contigCounter = contigNames.copy()\n",
    "    samfile = pysam.AlignmentFile(bamFileName, \"rb\")\n",
    "    readsFileName = bamFileName.replace(\"bams/\",\"\").replace(\".sorted.bam\",\"\")\n",
    "    print readsFileName\n",
    "    sampleMeta = metadata[metadata.index == readsFileName]\n",
    "    nuceID = sampleMeta.nucleic_acid_name.get_values()[0]\n",
    "    countsFile = open(\"counts/%s.txt\" % (nuceID),\"w\")\n",
    "    reads=set()\n",
    "    for read in samfile:\n",
    "        if checkSkipRead(read,reads):continue \n",
    "        reads.add(read.qname)\n",
    "    countsFile.close()\n",
    "    break\n",
    "print coninues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print samFileName\n",
    "counter = 0\n",
    "for name in contigCounter:\n",
    "    print name, samfile.count(name)\n",
    "    counter +=1\n",
    "    if counter == 10: break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print \"Number of unmapped\",unmapped\n",
    "print coninues-unmapped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print read"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print read.reference_name\n",
    "print samFileName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "contname = contigNames.keys()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = glob(\"stats/*.tsv\")\n",
    "sampleStats = {}\n",
    "for statFile in files:\n",
    "    sampleFileName = statFile.replace(\"stats/\",\"\").replace(\".tsv\",\"\")\n",
    "    sampleMeta = metadata[metadata.index == sampleFileName]\n",
    "    nuceID = sampleMeta.nucleic_acid_name.get_values()[0]\n",
    "    for line in open(statFile):\n",
    "        rec = line.strip().split()\n",
    "        contigNames[rec[0]] = int(rec[2])\n",
    "    sampleStats[nuceID] = contigNames\n",
    "import pickle\n",
    "pickle.dump(sampleStats,open(\"sampleCounts.p\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "help(samfile.count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = 0\n",
    "for id in metadata.index:\n",
    "    if '11505.2.209522.CACCTTA-GTAAGGT.fastq.gz' in id: \n",
    "        counter += 1\n",
    "        print id,\n",
    "        if counter % 2 == 0:print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampleMeta = metadata[metadata.index == readsFileName]\n",
    "sampleMeta.nucleic_acid_name.get_values()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "samFileName"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rec.is_qcfail\n",
    "rec.is_read1\n",
    "dir(rec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print 10**(-(ord('3')-33)/10.0)\n",
    "print 10.0**(-(ord('3')-33.0)/10.0)\n",
    "print 10**(-(ord('3')-33)/10)\n",
    "print 10**(-(ord('3')-33.0)/10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "fh=open(\"CombinedStats.tsv\",\"w\")\n",
    "fh.write(\"Contig\")\n",
    "\n",
    "for fname in glob(\"stats/*.tsv\"):\n",
    "    print(fname)\n",
    "    readsFileName = fname.replace(\"stats/\",\"\").replace(\".tsv\",\"\")\n",
    "    sampleMeta = metadata[metadata.index == readsFileName]\n",
    "    curSampleID = sampleMeta.nucleic_acid_name.values[0]\n",
    "    fh.write(\"\\t\"+curSampleID)\n",
    "\n",
    "\n",
    "#Get the column identifier\n",
    "init_fname = \"stats/11425.3.206650.ACGGTCT-AAGACCG.fastq.gz.tsv\"\n",
    "for line in open(init_fname):\n",
    "    rec = line.strip().split()\n",
    "    fh.write(\"\\t\"+rec[0])\n",
    "fh.write(\"\\n\")\n",
    "\n",
    "#Read all the files\n",
    "for fname in glob(\"stats/*.tsv\"):\n",
    "    print(fname)\n",
    "    readsFileName = fname.replace(\"stats/\",\"\").replace(\".tsv\",\"\")\n",
    "    sampleMeta = metadata[metadata.index == readsFileName]\n",
    "    curSampleID = sampleMeta.nucleic_acid_name.values[0]\n",
    "    fh.write(curSampleID)\n",
    "    for line in open(fname):\n",
    "        rec = line.strip().split()\n",
    "        fh.write(\"\\t\"+rec[2])\n",
    "    fh.write(\"\\n\")\n",
    "fh.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import linecache\n",
    "from glob import glob\n",
    "# fh = open(\"CombinedStats_T.tsv\",\"w\")\n",
    "# fh.write(\"\\t\")\n",
    "fileNames = glob(\"stats/*.tsv\")\n",
    "\n",
    "# #Write the header line\n",
    "# for fname in fileNames:\n",
    "#     readsFileName = fname.replace(\"stats/\",\"\").replace(\".tsv\",\"\")\n",
    "#     sampleMeta = metadata[metadata.index == readsFileName]\n",
    "#     curSampleID = sampleMeta.nucleic_acid_name.values[0]\n",
    "#     fh.write(\"\\t\"+curSampleID)\n",
    "# fh.write(\"\\n\")\n",
    "\n",
    "#get the row starters\n",
    "pres_abs = {}\n",
    "total_count =\n",
    "init_fname = \"stats/11425.3.206650.ACGGTCT-AAGACCG.fastq.gz.tsv\"\n",
    "for line in open(init_fname): rows.append(line.strip().split()[0])\n",
    "print(\"Done getting contig names\")\n",
    "fh = open(\"combined/CombinedStats_T%i.tsv\" % 0 ,\"w\")\n",
    "val={0:\"\"}\n",
    "for i in range(1,12301484):\n",
    "    print(i,end=\" \")\n",
    "    fh.write(rows[i-1])\n",
    "    for fname in fileNames: fh.write(\"\\t\"+linecache.getline(fname, i))\n",
    "    fh.write(\"\\n\")\n",
    "    try:\n",
    "        val[i %  5000] = 1\n",
    "        print(i,end=\" \")\n",
    "        fh.close()\n",
    "        fh = open(\"combined/CombinedStats_T%i.tsv\" % i ,\"w\")\n",
    "    except:pass\n",
    "        \n",
    "print(\"Done\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Presence absence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import pickle\n",
    "fileNames = glob(\"stats/*.tsv\")\n",
    "presAbs,abundance = {},{}\n",
    "\n",
    "for fname in fileNames:\n",
    "    print(fname)\n",
    "    for line in open(fname): \n",
    "        rec = line.strip().split()\n",
    "        try:presAbs[rec[0]] += int(int(rec[2]) > 0)\n",
    "        except:presAbs[rec[0]] = int(int(rec[2]) > 0)\n",
    "        try:abundance[rec[0]] += int(rec[2])\n",
    "        except:abundance[rec[0]] = int(rec[2])\n",
    "\n",
    "pickle.dump(presAbs,open(\"presence_Absence.p\",\"wb\"))    \n",
    "pickle.dump(abundance,open(\"abundance.p\",\"wb\"))   \n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist_abu = []\n",
    "print(int(len(fileNames)*.7),len(fileNames))\n",
    "numSamples = float(len(fileNames))\n",
    "for contig, count in abundance.items(): dist_abu.append(count/numSamples)\n",
    "    \n",
    "from pandas import Series\n",
    "dist = Series(dist_abu)\n",
    "dist.plot.hist();\n",
    "dist.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dist.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Combining Sample Counts\n",
    "\n",
    "allCounts = load(open(\"pickles/allCounts_genes.p\",\"rb\")) \n",
    "\n",
    "combined_abundance = {}\n",
    "for crop in set(metadata.type):\n",
    "    for date in set(metadata.sampling_date):\n",
    "        for treatment in set(metadata.treatment):\n",
    "            sampleGroupMeta = metadata.loc[(metadata.sampling_date==date) & (metadata.treatment==treatment) & (metadata.type == crop)]\n",
    "            groupName = \"%s_%s_%s\" % (crop,treatment.replace(\" \",\"\").replace(\"f\",\"F\"),date)\n",
    "            if len(sampleGroupMeta) == 0: continue\n",
    "            print(groupName)\n",
    "            combined_abundance[groupName]= {}\n",
    "            for sampleName in sampleGroupMeta.nucleic_acid_name:\n",
    "                #print(sampleName,len(allCounts[sampleName]))\n",
    "                for rec,count in allCounts[sampleName].items():\n",
    "                    try:combined_abundance[groupName][rec] += count\n",
    "                    except:combined_abundance[groupName][rec] = count\n",
    "allCounts = DataFrame(combined_abundance)\n",
    "allCounts[\"Average\"] = allCounts.sum(axis=1)/float(len(metadata.index))\n",
    "allCounts[\"Average\"].head()\n",
    "#allCounts.drop(\"Average\",axis=1,inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fname in metadata.index:\n",
    "    sampMet = metadata[metadata.index == fname]\n",
    "    sampleID = sampMet.nucleic_acid_name.unique()[0]\n",
    "    #print(\"mv mapping/flagstats/%s.stat mapping/flagstats/%s.stat\" % (fname,sampleID))\n",
    "    \n",
    "    res = os.system(\"mv mapping/flagstats/%s.stat mapping/flagstats/%s.txt\" % (sampleID,sampleID))\n",
    "    print(fname,res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "v= MultiqcModule()\n",
    "v = FlagstatReportMixin()\n",
    "v.parse_samtools_flagstats()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from collections import OrderedDict\n",
    "import logging\n",
    "import re\n",
    "\n",
    "from multiqc import config\n",
    "from multiqc.plots import bargraph\n",
    "from multiqc.modules.base_module import BaseMultiqcModule\n",
    "# from multiqc.utils import config as mqcConfig\n",
    "# def load_config():\n",
    "#     my_search_patterns = {\n",
    "#         'my_plugin/my_mod': {'fn': '*_somefile.txt'},\n",
    "#         'my_plugin/my_other_mod': {'fn': '*other_file.txt'},\n",
    "#     }\n",
    "#     mqcConfig.update_dict(config.sp, my_search_patterns)\n",
    "\n",
    "\n",
    "# Initialise the logger\n",
    "log = logging.getLogger(__name__)\n",
    "\n",
    "class MultiqcModule(BaseMultiqcModule):\n",
    "    \"\"\" Bowtie 2 module, parses stderr logs. \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "\n",
    "        # Initialise the parent object\n",
    "        super(MultiqcModule, self).__init__(name=\"Bowtie 2\", anchor=\"stat\",\n",
    "        href='http://bowtie-bio.sourceforge.net/bowtie2/',\n",
    "        info=\"is an ultrafast and memory-efficient tool for aligning sequencing\"\\\n",
    "                \" reads to long reference sequences.\")\n",
    "\n",
    "        # Find and load any Bowtie 2 reports\n",
    "        self.bowtie2_data = dict()\n",
    "        self.num_se = 0\n",
    "        self.num_pe = 0\n",
    "        print(dir(self.find_log_files))\n",
    "        print(\"\")\n",
    "        print(help(self.find_log_files))\n",
    "        for f in self.find_log_files('mapping/flagstats', filehandles=True):\n",
    "            self.parse_bowtie2_logs(f)\n",
    "\n",
    "        # Filter to strip out ignored sample names\n",
    "        self.bowtie2_data = self.ignore_samples(self.bowtie2_data)\n",
    "\n",
    "        if len(self.bowtie2_data) == 0:\n",
    "            raise UserWarning\n",
    "\n",
    "        log.info(\"Found {} reports\".format(len(self.bowtie2_data)))\n",
    "\n",
    "        # Write parsed report data to a file\n",
    "        self.write_data_file(self.bowtie2_data, 'multiqc_bowtie2')\n",
    "\n",
    "        # Basic Stats Table\n",
    "        # Report table is immutable, so just updating it works\n",
    "        self.bowtie2_general_stats_table()\n",
    "\n",
    "        # Alignment Rate Plot\n",
    "        self.bowtie2_alignment_plot()\n",
    "\n",
    "\n",
    "    def parse_bowtie2_logs(self, f):\n",
    "        \"\"\"\n",
    "        Warning: This function may make you want to stab yourself.\n",
    "        Parse logs from bowtie2. These miss several key bits of information\n",
    "        such as input files, so we try to look for logs from other wrapper tools\n",
    "        that may have logged this info. If not found, we default to using the filename.\n",
    "        Note that concatenated logs only parse if we have the command printed in there.\n",
    "        The bowtie log uses the same strings mulitple times in different contexts to mean\n",
    "        different things, making parsing very messy. Handle with care.\n",
    "        Example single-end output from bowtie2:\n",
    "            Time loading reference: 00:00:08\n",
    "            Time loading forward index: 00:00:16\n",
    "            Time loading mirror index: 00:00:09\n",
    "            [samopen] SAM header is present: 25 sequences.\n",
    "            Multiseed full-index search: 00:58:04\n",
    "            38377305 reads; of these:\n",
    "              38377305 (100.00%) were unpaired; of these:\n",
    "                2525577 (6.58%) aligned 0 times\n",
    "                27593593 (71.90%) aligned exactly 1 time\n",
    "                8258135 (21.52%) aligned >1 times\n",
    "            93.42% overall alignment rate\n",
    "            Time searching: 00:58:37\n",
    "            Overall time: 00:58:37\n",
    "        Example paired-end output from bowtie2:\n",
    "            Time loading reference: 00:01:07\n",
    "            Time loading forward index: 00:00:26\n",
    "            Time loading mirror index: 00:00:09\n",
    "            Multiseed full-index search: 01:32:55\n",
    "            15066949 reads; of these:\n",
    "              15066949 (100.00%) were paired; of these:\n",
    "                516325 (3.43%) aligned concordantly 0 times\n",
    "                11294617 (74.96%) aligned concordantly exactly 1 time\n",
    "                3256007 (21.61%) aligned concordantly >1 times\n",
    "                ----\n",
    "                516325 pairs aligned concordantly 0 times; of these:\n",
    "                  26692 (5.17%) aligned discordantly 1 time\n",
    "                ----\n",
    "                489633 pairs aligned 0 times concordantly or discordantly; of these:\n",
    "                  979266 mates make up the pairs; of these:\n",
    "                    592900 (60.55%) aligned 0 times\n",
    "                    209206 (21.36%) aligned exactly 1 time\n",
    "                    177160 (18.09%) aligned >1 times\n",
    "            98.03% overall alignment rate\n",
    "            Time searching: 01:34:37\n",
    "            Overall time: 01:34:37\n",
    "        \"\"\"\n",
    "\n",
    "        # Regexes\n",
    "        regexes = {\n",
    "            'unpaired': {\n",
    "                'unpaired_aligned_none': r\"(\\d+) \\([\\d\\.]+%\\) aligned 0 times\",\n",
    "                'unpaired_aligned_one': r\"(\\d+) \\([\\d\\.]+%\\) aligned exactly 1 time\",\n",
    "                'unpaired_aligned_multi': r\"(\\d+) \\([\\d\\.]+%\\) aligned >1 times\"\n",
    "            },\n",
    "            'paired': {\n",
    "                'paired_aligned_none': r\"(\\d+) \\([\\d\\.]+%\\) aligned concordantly 0 times\",\n",
    "                'paired_aligned_one': r\"(\\d+) \\([\\d\\.]+%\\) aligned concordantly exactly 1 time\",\n",
    "                'paired_aligned_multi': r\"(\\d+) \\([\\d\\.]+%\\) aligned concordantly >1 times\",\n",
    "                'paired_aligned_discord_one': r\"(\\d+) \\([\\d\\.]+%\\) aligned discordantly 1 time\",\n",
    "                'paired_aligned_discord_multi': r\"(\\d+) \\([\\d\\.]+%\\) aligned discordantly >1 times\",\n",
    "                'paired_aligned_mate_one': r\"(\\d+) \\([\\d\\.]+%\\) aligned exactly 1 time\",\n",
    "                'paired_aligned_mate_multi': r\"(\\d+) \\([\\d\\.]+%\\) aligned >1 times\",\n",
    "                'paired_aligned_mate_none': r\"(\\d+) \\([\\d\\.]+%\\) aligned 0 times\"\n",
    "            }\n",
    "        }\n",
    "\n",
    "        # Go through log file line by line\n",
    "        s_name = f['s_name']\n",
    "        parsed_data = {}\n",
    "\n",
    "        for l in f['f']:\n",
    "            # Attempt in vain to find original bowtie2 command, logged by another program\n",
    "            btcmd = re.search(r\"bowtie2 .+ -[1U] ([^\\s,]+)\", l)\n",
    "            if btcmd:\n",
    "                s_name = self.clean_s_name(btcmd.group(1), f['root'])\n",
    "                log.debug(\"Found a bowtie2 command, updating sample name to '{}'\".format(s_name))\n",
    "\n",
    "            # Total reads\n",
    "            total = re.search(r\"(\\d+) reads; of these:\", l)\n",
    "            if total:\n",
    "                parsed_data['total_reads'] = int(total.group(1))\n",
    "\n",
    "            # Single end reads\n",
    "            unpaired = re.search(r\"(\\d+) \\([\\d\\.]+%\\) were unpaired; of these:\", l)\n",
    "            if unpaired:\n",
    "                parsed_data['unpaired_total'] = int(unpaired.group(1))\n",
    "                self.num_se += 1\n",
    "\n",
    "                # Do nested loop whilst we have this level of indentation\n",
    "                l = f['f'].readline()\n",
    "                while l.startswith('    '):\n",
    "                    for k, r in regexes['unpaired'].items():\n",
    "                        match = re.search(r, l)\n",
    "                        if match:\n",
    "                            parsed_data[k] = int(match.group(1))\n",
    "                    l = f['f'].readline()\n",
    "\n",
    "            # Paired end reads\n",
    "            paired = re.search(r\"(\\d+) \\([\\d\\.]+%\\) were paired; of these:\", l)\n",
    "            if paired:\n",
    "                parsed_data['paired_total'] = int(paired.group(1))\n",
    "                self.num_pe += 1\n",
    "\n",
    "                # Do nested loop whilst we have this level of indentation\n",
    "                l = f['f'].readline()\n",
    "                while l.startswith('    '):\n",
    "                    for k, r in regexes['paired'].items():\n",
    "                        match = re.search(r, l)\n",
    "                        if match:\n",
    "                            parsed_data[k] = int(match.group(1))\n",
    "                    l = f['f'].readline()\n",
    "\n",
    "            # Overall alignment rate\n",
    "            overall = re.search(r\"([\\d\\.]+)% overall alignment rate\", l)\n",
    "            if overall:\n",
    "                parsed_data['overall_alignment_rate'] = float(overall.group(1))\n",
    "\n",
    "                # End of log section\n",
    "                # Save half 'pairs' of mate counts\n",
    "                m_keys = ['paired_aligned_mate_multi', 'paired_aligned_mate_none', 'paired_aligned_mate_one']\n",
    "                for k in m_keys:\n",
    "                    if k in parsed_data:\n",
    "                        parsed_data['{}_halved'.format(k)] = float(parsed_data[k]) / 2.0\n",
    "                # Save parsed data\n",
    "                if s_name in self.bowtie2_data:\n",
    "                    log.debug(\"Duplicate sample name found! Overwriting: {}\".format(s_name))\n",
    "                self.add_data_source(f, s_name)\n",
    "                self.bowtie2_data[s_name] = parsed_data\n",
    "                # Reset in case we find more in this log file\n",
    "                s_name = f['s_name']\n",
    "                parsed_data = {}\n",
    "\n",
    "\n",
    "    def bowtie2_general_stats_table(self):\n",
    "        \"\"\" Take the parsed stats from the Bowtie 2 report and add it to the\n",
    "        basic stats table at the top of the report \"\"\"\n",
    "\n",
    "        headers = OrderedDict()\n",
    "        headers['overall_alignment_rate'] = {\n",
    "            'title': '% Aligned',\n",
    "            'description': 'overall alignment rate',\n",
    "            'max': 100,\n",
    "            'min': 0,\n",
    "            'suffix': '%',\n",
    "            'scale': 'YlGn'\n",
    "        }\n",
    "        self.general_stats_addcols(self.bowtie2_data, headers)\n",
    "\n",
    "    def bowtie2_alignment_plot (self):\n",
    "        \"\"\" Make the HighCharts HTML to plot the alignment rates \"\"\"\n",
    "\n",
    "        half_warning = ''\n",
    "        for s_name in self.bowtie2_data:\n",
    "            if 'paired_aligned_mate_one_halved' in self.bowtie2_data[s_name] or 'paired_aligned_mate_multi_halved' in self.bowtie2_data[s_name] or 'paired_aligned_mate_none_halved' in self.bowtie2_data[s_name]:\n",
    "                half_warning = '<em>Please note that single mate alignment counts are halved to tally with pair counts properly.</em>'\n",
    "        description_text = 'This plot shows the number of reads aligning to the reference in different ways.'\n",
    "\n",
    "        # Config for the plot\n",
    "        config = {\n",
    "            'ylab': '# Reads',\n",
    "            'cpswitch_counts_label': 'Number of Reads'\n",
    "        }\n",
    "\n",
    "        # Two plots, don't mix SE with PE\n",
    "        if self.num_se > 0:\n",
    "            sekeys = OrderedDict()\n",
    "            sekeys['unpaired_aligned_one'] = { 'color': '#20568f', 'name': 'SE mapped uniquely' }\n",
    "            sekeys['unpaired_aligned_multi'] = { 'color': '#f7a35c', 'name': 'SE multimapped' }\n",
    "            sekeys['unpaired_aligned_none'] = { 'color': '#981919', 'name': 'SE not aligned' }\n",
    "            config['id'] = 'bowtie2_se_plot'\n",
    "            config['title'] = 'Bowtie 2: SE Alignment Scores'\n",
    "            self.add_section(\n",
    "                description = description_text,\n",
    "                helptext = '''\n",
    "                There are 3 possible types of alignment:\n",
    "                * **SE Mapped uniquely**: Read has only one occurence in the reference genome.\n",
    "                * **SE Multimapped**: Read has multiple occurence.\n",
    "                * **SE No aligned**: Read has no occurence.\n",
    "                ''',\n",
    "                plot = bargraph.plot(self.bowtie2_data, sekeys, config)\n",
    "            )\n",
    "\n",
    "        if self.num_pe > 0:\n",
    "            pekeys = OrderedDict()\n",
    "            pekeys['paired_aligned_one'] = { 'color': '#20568f', 'name': 'PE mapped uniquely' }\n",
    "            pekeys['paired_aligned_discord_one'] = { 'color': '#5c94ca', 'name': 'PE mapped discordantly uniquely' }\n",
    "            pekeys['paired_aligned_mate_one_halved'] = { 'color': '#95ceff', 'name': 'PE one mate mapped uniquely' }\n",
    "            pekeys['paired_aligned_multi'] = { 'color': '#f7a35c', 'name': 'PE multimapped' }\n",
    "            pekeys['paired_aligned_discord_multi'] = { 'color': '#dce333', 'name': 'PE discordantly multimapped' }\n",
    "            pekeys['paired_aligned_mate_multi_halved'] = { 'color': '#ffeb75', 'name': 'PE one mate multimapped' }\n",
    "            pekeys['paired_aligned_mate_none_halved'] = { 'color': '#981919', 'name': 'PE neither mate aligned' }\n",
    "            config['id'] = 'bowtie2_pe_plot'\n",
    "            config['title'] = 'Bowtie 2: PE Alignment Scores'\n",
    "            self.add_section(\n",
    "                description = \"<br>\".join([description_text,half_warning]),\n",
    "                helptext = '''\n",
    "                There are 6 possible types of alignment:\n",
    "                * **PE mapped uniquely**: Pair has only one occurence in the reference genome.\n",
    "                * **PE mapped discordantly uniquely**: Pair has only one occurence but not in proper pair.\n",
    "                * **PE one mate mapped uniquely**: One read of a pair has one occurence.\n",
    "                * **PE multimapped**: Pair has multiple occurence.\n",
    "                * **PE one mate multimapped**: One read of a pair has multiple occurence.\n",
    "                * **PE neither mate aligned**: Pair has no occurence.\n",
    "                ''',\n",
    "                plot = bargraph.plot(self.bowtie2_data, pekeys, config)\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fh = open(\"figures/metadata.html\",\"w\")\n",
    "# fh.write(metadata.head(25).to_html())\n",
    "# fh.close()\n",
    "#A custom function to help us find the raw fastq files\n",
    "from glob import glob\n",
    "rawFastqFiles = glob('jgi_transfer/metaT_raw/*/Raw_Data/*')\n",
    "\n",
    "def lookupFSTQ(name,rawFastqFiles):\n",
    "    fastqName = ''\n",
    "    found = False\n",
    "    for fastqName in rawFastqFiles:\n",
    "        if name in fastqName: \n",
    "            found = True\n",
    "            break\n",
    "    if found:\n",
    "        rawFastqFiles.remove(fastqName)\n",
    "        return fastqName"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}