CeRNetwork.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import necessary python packages\n",
    "import math\n",
    "import pandas as pd\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CMN ceRNA Nodes: 25131\n",
      "CMN miRNA Nodes: 642\n",
      "CMN Edges: 1020679\n"
     ]
    }
   ],
   "source": [
    "# CREATE THE FIRST PART OF CeRNETWORK: Mapping ceRNA-miRNA Interactions\n",
    "\n",
    "# import the datasets for ceRNA:miRNA and delete extraneous columns\n",
    "# TWO COLUMNS: 'ceRNA' and 'miRNA'\n",
    "starBaseLnc = pd.read_csv('/databases/starBase_lncRNA.txt', delimiter = '\\t')\n",
    "starBaseSnc = pd.read_csv('/databases/starBase_sncRNA.txt', delimiter = '\\t')\n",
    "starBaseCirc = pd.read_csv('/databases/starBase_circRNA.txt', delimiter = '\\t')\n",
    "starBasePseudo = pd.read_csv('/databases/starBase_pseudogenes.txt', delimiter = '\\t')\n",
    "\n",
    "sbncIndexList = ['miRNAid', 'geneID', 'geneType', 'chromosome','start','end','strand','clipExpNum','degraExpNum','RBP','merClass','miRseq','align','targetSeq']\n",
    "\n",
    "for index in sbncIndexList:\n",
    "    del starBaseLnc[index]\n",
    "    del starBaseSnc[index]\n",
    "    del starBaseCirc[index]\n",
    "    del starBasePseudo[index]\n",
    "del starBaseLnc['pancancerNum']\n",
    "del starBaseSnc['pancancerNum']\n",
    "del starBasePseudo['pancancerNum']\n",
    "\n",
    "# write the first part as a new csv file\n",
    "ceRNetwork1 = starBaseLnc.append(starBaseSnc.append(starBaseCirc.append(starBasePseudo)), ignore_index = True) \n",
    "ceRNACRNList1 = []\n",
    "miRNACRNList1 = []\n",
    "for x in range(len(ceRNetwork1)):\n",
    "    if ceRNetwork1['ceRNA'][x] not in ceRNACRNList1:\n",
    "        ceRNACRNList1.append(ceRNetwork1['ceRNA'][x])\n",
    "    if ceRNetwork1['miRNA'][x] not in miRNACRNList1:\n",
    "        miRNACRNList1.append(ceRNetwork1['miRNA'][x])\n",
    "\n",
    "print(\"CMN ceRNA Nodes:\", len(ceRNACRNList1))\n",
    "print(\"CMN miRNA Nodes:\", len(miRNACRNList1))\n",
    "\n",
    "ceRNetwork1Duplicates = ceRNetwork1.duplicated()\n",
    "lenBeforeDrop1 = len(ceRNetwork1)\n",
    "ceRNetwork1.drop_duplicates(subset = None, inplace = True)\n",
    "\n",
    "print(\"CMN Edges:\", len(ceRNetwork1))\n",
    "ceRNetwork1.to_csv('/networks/ceRNetwork1.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/davidtoomer/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py:6201: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=True'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass sort=False\n",
      "\n",
      "  sort=sort)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MGN miRNA Nodes: 3024\n",
      "MGN Gene Nodes: 24590\n",
      "MGN Edges: 3457049\n"
     ]
    }
   ],
   "source": [
    "# CREATE THE SECOND PART OF CeRNETWORK: Mapping the miRNA-Gene Interactions\n",
    "\n",
    "# import datasets for miRNA:mRNA and delete extraneous columns\n",
    "# TWO COLUMNS: 'miRNA' and 'Gene'\n",
    "miRTarBase = pd.read_csv('/databases/miRTarBase.tsv', delimiter = '\\t')\n",
    "mtbIndexList = ['miRTarBase ID', 'Species (miRNA)', 'Target Gene (Entrez Gene ID)','Species (Target Gene)','Experiments','Support Type','References (PMID)']\n",
    "for index in mtbIndexList:\n",
    "    del miRTarBase[index]\n",
    "\n",
    "miRandaHighConserv = pd.read_csv('/databases/miRanda_HighScore_Conserved.txt', delimiter = '\\t')\n",
    "miRandaHighNonconserv = pd.read_csv('/databases/miRanda_HighScore_Nonconserved.txt', delimiter = '\\t')\n",
    "mrdIndexList = ['#mirbase_acc','gene_id','transcript_id','genBank_access','mirna_alignment','alignment','gene_alignment','mirna_start','mirna_end','gene_start','gene_end','genome_coordinates','conservation','align_score','seed_cat','energy','mirsvr_score']\n",
    "for index in mrdIndexList:\n",
    "    del miRandaHighConserv[index]\n",
    "    del miRandaHighNonconserv[index]\n",
    "\n",
    "miRecords = pd.read_csv('/databases/miRecords.tsv', delimiter = '\\t')\n",
    "mrcIndexList = ['pubmed_id','target_gene_species','genBank_access','miRNA_species']\n",
    "for index in mrcIndexList:\n",
    "    del miRecords[index]\n",
    "\n",
    "tarBase = pd.read_csv('/databases/tarBase.csv', delimiter = '\\t')\n",
    "trbIndexList = ['geneId','species','cell_line','tissue','category','method','positive_negative','direct_indirect','up_down','condition']\n",
    "nonHumanTargets = []\n",
    "for x in range(0, len(tarBase)):\n",
    "    if tarBase['species'][x] != \"Homo sapiens\":\n",
    "        nonHumanTargets.append(x)\n",
    "tarBase.drop(index = nonHumanTargets, inplace = True)\n",
    "for index in trbIndexList:\n",
    "    del tarBase[index]\n",
    "    \n",
    "# write the second part as a new csv file\n",
    "ceRNetwork2 = miRTarBase.append(miRandaHighConserv.append(miRandaHighNonconserv.append(miRecords.append(tarBase))), ignore_index = True)\n",
    "miRNACRNList2 = []\n",
    "geneCRNList2 = []\n",
    "for x in range(len(ceRNetwork2)):\n",
    "    if ceRNetwork2['miRNA'][x] not in miRNACRNList2:\n",
    "        miRNACRNList2.append(ceRNetwork2['miRNA'][x])\n",
    "    if ceRNetwork2['Gene'][x] not in geneCRNList2:\n",
    "        geneCRNList2.append(ceRNetwork2['Gene'][x])\n",
    "print(\"MGN miRNA Nodes:\", len(miRNACRNList2))\n",
    "print(\"MGN Gene Nodes:\", len(geneCRNList2))\n",
    "ceRNetwork2Duplicates = ceRNetwork2.duplicated()\n",
    "lenBeforeDrop2 = len(ceRNetwork2)\n",
    "ceRNetwork2.drop_duplicates(subset = None, inplace = True)\n",
    "print(\"MGN Edges:\", len(ceRNetwork2))\n",
    "ceRNetwork2.to_csv('/networks/ceRNetwork2.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GDN Gene Nodes: 17545\n",
      "GDN Disease Nodes: 24166\n",
      "GDN Edges: 628685\n"
     ]
    }
   ],
   "source": [
    "# CREATE THE THIRD PART OF CeRNETWORK: Mapping Gene-Disease Interactions\n",
    "\n",
    "# import dataset for gene:disease networks\n",
    "# TWO COLUMNS: 'Gene' and 'Disease'\n",
    "disGeNet = pd.read_csv('/databases/disGeNet.tsv', delimiter = '\\t')\n",
    "dgnIndexList = ['geneId','diseaseName','score','NofPmids','NofSnps','source']\n",
    "for index in dgnIndexList:\n",
    "    del disGeNet[index]\n",
    "    \n",
    "# write the third part as a new csv file\n",
    "ceRNetwork3 = disGeNet\n",
    "ceRNetwork3.to_csv('/networks/ceRNetwork3.csv', index = False)\n",
    "\n",
    "geneCRNList3 = []\n",
    "diseaseCRNList3 = []\n",
    "for x in range(0, len(ceRNetwork3)):\n",
    "    if ceRNetwork3['Gene'][x] not in geneCRNList3:\n",
    "        geneCRNList3.append(ceRNetwork3['Gene'][x])\n",
    "    if ceRNetwork3['Disease'][x] not in diseaseCRNList3:\n",
    "        diseaseCRNList3.append(ceRNetwork3['Disease'][x])\n",
    "print(\"GDN Gene Nodes:\", len(geneCRNList3))\n",
    "print(\"GDN Disease Nodes:\", len(diseaseCRNList3))\n",
    "print(\"GDN Edges:\", len(ceRNetwork3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing CGN1: Initializing miRNA Set\n",
      "10% complete... 534664\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "miRNA Set 100% Initialized.\n",
      "Processing CGN1: Creating EdgeList 1\n",
      "10% complete... 140294\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "CGN1 EdgeList 100% Complete.\n",
      "Processing CGN2: Initializing miRNA Set\n",
      "miRNA Set 100% Initialized.\n",
      "Processing CGN2: Creating EdgeList 2\n",
      "10% complete... 534664\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "CGN2 EdgeList 100% Complete.\n",
      "CMGN ceRNA Nodes: 25131\n",
      "CMGN miRNA Nodes: 642\n",
      "CMGN Gene Nodes: 24116\n",
      "CMGN Edges: 1802096\n",
      "Initializing Hypergeometric Test...\n",
      "CGN Mean: 7.787830050370143\n",
      "CGN Standard Deviation: 2.612857477918844\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-65-81e49eb2907a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    166\u001b[0m         \u001b[0mmirsetCGN2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0medgeListCGN2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 168\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mgeneCGN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mz\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0medgeListCGN2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    169\u001b[0m                 \u001b[0mmirsetCGN2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0medgeListCGN2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m         \u001b[0mcount\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmirsetCGN1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmirsetCGN2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# CONSTRUCTING BIPARTITE NETWORKS: ceRNA-Gene Network (CGN)\n",
    "\n",
    "print(\"Processing CGN1: Initializing miRNA Set\")\n",
    "cgnMiRList1 = []\n",
    "for x in tqdm(range(lenBeforeDrop2)):\n",
    "    if not ceRNetwork2Duplicates[x]:\n",
    "        cgnMiRList1.append(ceRNetwork2['miRNA'][x])\n",
    "cgnMiRSet1 = set(cgnMiRList1)\n",
    "print(\"miRNA Set 100% Initialized.\")\n",
    "\n",
    "print(\"Processing CGN1: Creating EdgeList 1\")\n",
    "ceRNACGN = []\n",
    "miRNACGN = []\n",
    "edgeListCGN1 = []\n",
    "for x in tqdm(range(lenBeforeDrop1)):\n",
    "    if not ceRNetwork1Duplicates[x]:\n",
    "        if ceRNetwork1['miRNA'][x] in cgnMiRSet1:\n",
    "            ceRNACGN.append(ceRNetwork1['ceRNA'][x])\n",
    "            miRNACGN.append(ceRNetwork1['miRNA'][x])\n",
    "            edgeListCGN1.append([ceRNetwork1['ceRNA'][x], ceRNetwork1['miRNA'][x]])\n",
    "print(\"CGN1 EdgeList 100% Complete.\")\n",
    "\n",
    "print(\"Processing CGN2: Initializing miRNA Set\")\n",
    "cgnMiRList2 = []\n",
    "for x in tqdm(range(lenBeforeDrop1)):\n",
    "    if not ceRNetwork1Duplicates[x]:\n",
    "        cgnMiRList2.append(ceRNetwork1['miRNA'][x])\n",
    "cgnMiRSet2 = set(cgnMiRList2)\n",
    "print(\"miRNA Set 100% Initialized.\")\n",
    "\n",
    "print(\"Processing CGN2: Creating EdgeList 2\")\n",
    "geneCGN = []\n",
    "edgeListCGN2 = []\n",
    "for x in tqdm(range(0, lenBeforeDrop2)):\n",
    "    if not ceRNetwork2Duplicates[x]:\n",
    "        if ceRNetwork2['miRNA'][x] in cgnMiRSet2:\n",
    "            geneCGN.append(ceRNetwork2['Gene'][x])\n",
    "            edgeListCGN2.append([ceRNetwork2['miRNA'][x], ceRNetwork2['Gene'][x]])\n",
    "print(\"CGN2 EdgeList 100% Complete.\")\n",
    "\n",
    "# method for dropping duplicate items in a list\n",
    "def drop_duplicates(my_list):\n",
    "    return list(set(my_list))\n",
    "\n",
    "# find number of nodes and edges\n",
    "ceRListCGN = drop_duplicates(ceRNACGN)\n",
    "lenCeRListCGN = len(ceRListCGN)\n",
    "print(\"CMGN ceRNA Nodes:\", lenCeRListCGN)\n",
    "\n",
    "miRListCGN = drop_duplicates(miRNACGN)\n",
    "lenMiRListCGN = len(miRListCGN)\n",
    "print(\"CMGN miRNA Nodes:\", lenMiRListCGN)\n",
    "\n",
    "geneListCGN = drop_duplicates(geneCGN)\n",
    "lenGeneListCGN = len(geneListCGN)\n",
    "print(\"CMGN Gene Nodes:\", lenGeneListCGN)\n",
    "\n",
    "numEdgesCGN1 = len(edgeListCGN1)\n",
    "numEdgesCGN2 = len(edgeListCGN2)\n",
    "numEdgesCGN = numEdgesCGN1 + numEdgesCGN2\n",
    "print(\"CMGN Edges:\", numEdgesCGN)\n",
    "\n",
    "print(\"Initializing Hypergeometric Test...\")\n",
    "# find average degrees of the bipartite networks\n",
    "avgDegCGN1 = (2*numEdgesCGN1)/(lenCeRListCGN + lenMiRListCGN)\n",
    "avgDegCGN2 = (2*numEdgesCGN2)/(lenMiRListCGN + lenGeneListCGN)\n",
    "\n",
    "# hypergeometric test: statistical features of a random network \n",
    "meanCGN = (avgDegCGN1*avgDegCGN2)/lenMiRListCGN\n",
    "sigmaCGN = math.sqrt(meanCGN*(1-(avgDegCGN1/lenMiRListCGN)))\n",
    "print(\"CGN Mean:\", meanCGN)\n",
    "print(\"CGN Standard Deviation:\", sigmaCGN)\n",
    "\n",
    "# returns the intersection of two lists\n",
    "def intersection(list1, list2):\n",
    "    return [item for item in list1 if item in list2]\n",
    "\n",
    "# create master lists of miRNAs associated with each ceRNA or gene node\n",
    "print(\"Processing: Creating the First Master miRNA List\")\n",
    "superMiRListCGN1 = []\n",
    "for x in tqdm(range(len(ceRListCGN))):\n",
    "    mirsetCGN1 = []\n",
    "    for y in range(len(edgeListCGN1)):\n",
    "        if ceRListCGN[x] == edgeListCGN1[y][0]:\n",
    "            mirsetCGN1.append(edgeListCGN1[y][1])\n",
    "    superMiRListCGN1.append(mirsetCGN1)\n",
    "print(\"First miRNA list complete.\")\n",
    "\n",
    "print(\"Processing: Creating the Second Master miRNA List\")\n",
    "superMiRListCGN2 = []\n",
    "for x in tqdm(range(len(geneListCGN))):\n",
    "    mirsetCGN2 = []\n",
    "    for y in range(len(edgeListCGN2)):\n",
    "        if geneListCGN[x] == edgeListCGN2[y][1]:\n",
    "            mirsetCGN2.append(edgeListCGN2[y][0])\n",
    "    superMiRListCGN2.append(mirsetCGN2)\n",
    "print(\"Second miRNA list complete.\")\n",
    "\n",
    "print(\"Finding intersections of the miRNA lists:\")\n",
    "CGN = []\n",
    "for x in tqdm(range(len(ceRListCGN))):\n",
    "    list1 = superMiRListCGN1[x]\n",
    "    for y in range(len(geneListCGN)):\n",
    "        list2 = superMiRListCGN2[y]\n",
    "        count = len(intersection(list1, list2))\n",
    "        zScore = (count - meanCGN)/sigmaCGN\n",
    "        if zScore > 3:\n",
    "            CGN.append([ceRListCGN[x], geneListCGN[y], zScore])\n",
    "print(\"CGN 100% Complete.\")\n",
    "print(\"Creating node and edge lists...\")\n",
    "\n",
    "# create node and edge lists for the CGN after hypergeometric reduction\n",
    "ceRNACGNList = []\n",
    "geneCGNList = []\n",
    "for x in range(len(CGN)):\n",
    "    if CGN[x][0] not in ceRNACGNList:\n",
    "        ceRNACGNList.append(CGN[x][0])\n",
    "    if CGN[x][1] not in geneCGNList:\n",
    "        geneCGNList.append(CGN[x][1])\n",
    "print(\"CGN ceRNA Nodes:\", len(ceRNACGNList))\n",
    "print(\"CGN Gene Nodes:\", len(geneCGNList))\n",
    "print(\"CGN Edges:\", len(CGN))\n",
    "\n",
    "# write the list as a pandas dataframe and export as a csv file\n",
    "columnsCGN = ['ceRNA','Gene','Score']\n",
    "dfCGN = pd.DataFrame(CGN, columns = columnsCGN)\n",
    "dfCGN.to_csv('/networks/ceRNetwork_CGN.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing MDN1: Initializing Gene Set 1\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "Gene Set 1 100% Initialized.\n",
      "Processing MDN1: Creating EdgeList 1\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "MDN1 EdgeList 100% Complete.\n",
      "Processing MDN2: Initializing Gene Set 2\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "Gene Set 2 100% Initialized.\n",
      "Processing MDN2: Creating EdgeList 2\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "MDN2 100% Complete.\n",
      "MGDN miRNA Nodes: 3021\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'lenGeneListMDN' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-112-8a76276ff80d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    117\u001b[0m \u001b[0mgeneListMDN\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdrop_duplicates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgeneMDN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m \u001b[0mlenGeneListMDN1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgeneListMDN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"MGDN Gene Nodes:\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlenGeneListMDN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m \u001b[0mdiseaseListMDN\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdrop_duplicates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdiseaseMDN\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'lenGeneListMDN' is not defined"
     ]
    }
   ],
   "source": [
    "# CONSTRUCTING BIPARTITE NETWORKS: miRNA-Disease Network (MDN)\n",
    "\n",
    "# find the number of miRNAs, genes, and diseases in the network (combination of CRN2 and CRN3)\n",
    "print(\"Processing MDN1: Initializing Gene Set 1\")\n",
    "mdnGeneList1 = []\n",
    "for x in tqdm(range(len(ceRNetwork3))):\n",
    "    mdnGeneList1.append(ceRNetwork3['Gene'][x])\n",
    "mdnGeneSet1 = set(mdnGeneList1)\n",
    "print(\"Gene Set 1 100% Initialized.\")\n",
    "\n",
    "print(\"Processing MDN1: Creating EdgeList 1\")\n",
    "miRNAMDN = []\n",
    "geneMDN = []\n",
    "edgeListMDN1 = []\n",
    "for x in tqdm(range(lenBeforeDrop2)):\n",
    "    if not ceRNetwork2Duplicates[x]:\n",
    "        if ceRNetwork2['Gene'][x] in mdnGeneSet1:\n",
    "            miRNAMDN.append(ceRNetwork2['miRNA'][x])\n",
    "            geneMDN.append(ceRNetwork2['Gene'][x])\n",
    "            edgeListMDN1.append([ceRNetwork2['miRNA'][x], ceRNetwork2['Gene'][x]])\n",
    "print(\"MDN1 EdgeList 100% Complete.\")\n",
    "\n",
    "print(\"Processing MDN2: Initializing Gene Set 2\")\n",
    "mdnGeneList2 = []\n",
    "for x in tqdm(range(lenBeforeDrop2)):\n",
    "    if not ceRNetwork2Duplicates[x]:\n",
    "        mdnGeneList2.append(ceRNetwork2['Gene'][x])\n",
    "mdnGeneSet2 = set(mdnGeneList2)\n",
    "print(\"Gene Set 2 100% Initialized.\")\n",
    "\n",
    "print(\"Processing MDN2: Creating EdgeList 2\")\n",
    "diseaseMDN = []\n",
    "edgeListMDN2 = []\n",
    "for x in tqdm(range(len(ceRNetwork3))):\n",
    "    if ceRNetwork3['Gene'][x] in mdnGeneSet2:\n",
    "        diseaseMDN.append(ceRNetwork3['Disease'][x])\n",
    "        edgeListMDN2.append([ceRNetwork3['Gene'][x], ceRNetwork3['Disease'][x]])\n",
    "print(\"MDN2 100% Complete.\")\n",
    "\n",
    "# find number of nodes and edges\n",
    "miRListMDN = drop_duplicates(miRNAMDN)\n",
    "lenMiRListMDN = len(miRListMDN)\n",
    "print(\"MGDN miRNA Nodes:\", lenMiRListMDN)\n",
    "\n",
    "geneListMDN = drop_duplicates(geneMDN)\n",
    "lenGeneListMDN1 = len(geneListMDN)\n",
    "print(\"MGDN Gene Nodes:\", lenGeneListMDN1)\n",
    "\n",
    "diseaseListMDN = drop_duplicates(diseaseMDN)\n",
    "lenDiseaseListMDN = len(diseaseListMDN)\n",
    "print(\"MGDN Disease Nodes:\", lenDiseaseListMDN)\n",
    "\n",
    "numEdgesMDN1 = len(edgeListMDN1)\n",
    "numEdgesMDN2 = len(edgeListMDN2)\n",
    "numEdgesMDN = numEdgesMDN1 + numEdgesMDN2\n",
    "print(\"MGDN Edges:\", numEdgesMDN)\n",
    "\n",
    "print(\"Initializing Hypergeometric Test\")\n",
    "# find average degrees of the bipartite networks\n",
    "avgDegMDN1 = (2*numEdgesMDN1)/(lenMiRListMDN + lenGeneListMDN1)\n",
    "avgDegMDN2 = (2*numEdgesMDN2)/(lenGeneListMDN1 + lenDiseaseListMDN)\n",
    "\n",
    "# hypergeometric test\n",
    "meanMDN = (avgDegMDN1*avgDegMDN2)/lenGeneListMDN1\n",
    "sigmaMDN = math.sqrt(meanMDN*(1-(avgDegMDN1/lenGeneListMDN1)))\n",
    "print(\"MDN Mean:\", meanMDN)\n",
    "print(\"MDN Standard Deviation:\", sigmaMDN)\n",
    "\n",
    "# create universal gene lists for each miRNA and disease node\n",
    "superGeneListMDN1 = []\n",
    "print(\"Processing: Creating the First Master Gene List\")\n",
    "for x in tqdm(range(len(miRListMDN))):\n",
    "    genesetMDN1 = []\n",
    "    for y in range(len(edgeListMDN1)):\n",
    "        if miRListMDN[x] == edgeListMDN1[y][0]:\n",
    "            genesetMDN1.append(edgeListMDN1[y][1])\n",
    "    superGeneListMDN1.append(genesetMDN1)\n",
    "print(\"First Master miRNA List 100% Complete.\")\n",
    "\n",
    "print(\"Processing: Creating the Second Master Gene List\")\n",
    "superGeneListMDN2 = []\n",
    "for x in tqdm(range(len(diseaseListMDN))):\n",
    "    genesetMDN2 = []\n",
    "    for y in range(len(edgeListMDN2)):\n",
    "        if diseaseListMDN[x] == edgeListMDN2[y][1]:\n",
    "            genesetMDN2.append(edgeListMDN2[y][0])\n",
    "    superGeneListMDN2.append(genesetMDN2)\n",
    "print(\"Second Master Gene List 100% Complete.\")\n",
    "\n",
    "print(\"Finding Intersection of the Master Gene Lists.\")\n",
    "MDN = []\n",
    "for x in tqdm(range(len(miRListMDN))):\n",
    "    list1 = superGeneListMDN1[x]\n",
    "    for y in range(len(diseaseListMDN)):\n",
    "        list2 = superGeneListMDN2[y]\n",
    "        count = len(intersection(list1, list2))\n",
    "        zScore = (count - meanMDN)/sigmaMDN\n",
    "        if zScore > 3:\n",
    "            MDN.append([miRListMDN[x], diseaseListMDN[y], zScore])\n",
    "print(\"MDN 100% Complete.\")\n",
    "\n",
    "# create node and edge lists for the CGN after hypergeometric reduction\n",
    "print(\"Creating node and edge lists...\")\n",
    "miRNAMDNList = []\n",
    "diseaseMDNList = []\n",
    "for x in range(0, len(MDN)):\n",
    "    if MDN[x][0] not in miRNAMDNList:\n",
    "        miRNAMDNList.append(MDN[x][0])\n",
    "    if MDN[x][1] not in diseaseMDNList:\n",
    "        diseaseMDNList.append(MDN[x][1])\n",
    "print(\"MDN miRNA Nodes:\", len(miRNAMDNList))\n",
    "print(\"MDN Disease Nodes:\", len(diseaseMDNList))\n",
    "print(\"MDN Edges:\", len(MDN))\n",
    "\n",
    "# write the list as a pandas dataframe and export as a csv file\n",
    "columnsMDN = ['miRNA', 'Disease','Score']\n",
    "dfMDN = pd.DataFrame(MDN, columns = columnsMDN)\n",
    "dfMDN.to_csv('/networks/ceRNetwork_MDN.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing CDN1: Initializing miRNA Set 1\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "miRNA Set 1 100% Initialized.\n",
      "Processing CDN1: Creating EdgeList 1\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "EdgeList 1 100% Complete.\n",
      "Processing CDN2: Initializing miRNA Set 2\n",
      "miRNA Set 2 100% Initialized.\n",
      "Processing CDN2: Creating EdgeList 2\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "EdgeList 2 100% Complete.\n",
      "CMDN ceRNA Nodes: 25122\n",
      "CMDN miRNA Nodes: 639\n",
      "CMDN Disease Nodes: 11522\n",
      "CMDN Edges: 2627523\n",
      "Initializing Hypergeometric Test...\n",
      "CDN Mean: 32.75782297701664\n",
      "CDN Standard Deviation: 5.357239326218578\n",
      "Processing: Creating the First Master miRNA List\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "First Master miRNA list complete.\n",
      "Processing: Creating the Second Master miRNA List\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "Second Master miRNA list complete.\n",
      "Creating CDN: Finding intersections of the miRNA lists\n",
      "10% complete...\n",
      "20% complete...\n",
      "30% complete...\n",
      "40% complete...\n",
      "50% complete...\n",
      "60% complete...\n",
      "70% complete...\n",
      "80% complete...\n",
      "90% complete...\n",
      "CDN 100% Complete.\n",
      "Creating node and edge lists...\n",
      "CDN ceRNA Nodes: 0\n",
      "CDN Disease Nodes: 0\n",
      "CDN Edges: 0\n"
     ]
    }
   ],
   "source": [
    "# CONSTRUCTING BIPARTITE NETWORKS: ceRNA-Disease Network (CDN)\n",
    "\n",
    "# find the number of ceRNAs, miRNAs, and diseases in the network (combination of CRN1 and MDN)\n",
    "print(\"Processing CDN1: Initializing miRNA Set 1\")\n",
    "cdnMiRList1 = []\n",
    "for x in tqdm(range(len(dfMDN))):\n",
    "    cdnMiRList1.append(dfMDN['miRNA'][x])\n",
    "cdnMiRSet1 = set(cdnMiRList1)\n",
    "print(\"miRNA Set 1 100% Initialized.\")\n",
    "\n",
    "print(\"Processing CDN1: Creating EdgeList 1\")\n",
    "ceRNACDN = []\n",
    "miRNACDN = []\n",
    "edgeListCDN1 = []\n",
    "for x in tqdm(range(lenBeforeDrop1)):\n",
    "    if not ceRNetwork1Duplicates[x]:\n",
    "        if ceRNetwork1['miRNA'][x] in cdnMiRSet1:\n",
    "            ceRNACDN.append(ceRNetwork1['ceRNA'][x])\n",
    "            miRNACDN.append(ceRNetwork1['miRNA'][x])\n",
    "            edgeListCDN1.append([ceRNetwork1['ceRNA'][x], ceRNetwork1['miRNA'][x]])\n",
    "print(\"EdgeList 1 100% Complete.\")\n",
    "\n",
    "print(\"Processing CDN2: Initializing miRNA Set 2\")\n",
    "cdnMiRList2 = []\n",
    "cdnMiRSet2 = cgnMiRSet2\n",
    "print(\"miRNA Set 2 100% Initialized.\")\n",
    "\n",
    "print(\"Processing CDN2: Creating EdgeList 2\")\n",
    "diseaseCDN = []\n",
    "edgeListCDN2 = []\n",
    "for x in tqdm(range(len(dfMDN))):\n",
    "    if dfMDN['miRNA'][x] in cdnMiRSet2:\n",
    "        diseaseCDN.append(dfMDN['Disease'][x])\n",
    "        edgeListCDN2.append([dfMDN['miRNA'][x], dfMDN['Disease'][x]])\n",
    "print(\"EdgeList 2 100% Complete.\")\n",
    "\n",
    "# find number of nodes and edges\n",
    "ceRListCDN = drop_duplicates(ceRNACDN)\n",
    "lenCeRListCDN = len(ceRListCDN)\n",
    "print(\"CMDN ceRNA Nodes:\", lenCeRListCDN)\n",
    "\n",
    "miRListCDN = drop_duplicates(miRNACDN)\n",
    "lenMiRListCDN = len(miRListCDN)\n",
    "print(\"CMDN miRNA Nodes:\", lenMiRListCDN)\n",
    "\n",
    "diseaseListCDN = drop_duplicates(diseaseCDN)\n",
    "lenDiseaseListCDN = len(diseaseListCDN)\n",
    "print(\"CMDN Disease Nodes:\", lenDiseaseListCDN)\n",
    "\n",
    "numEdgesCDN1 = len(edgeListCDN1)\n",
    "numEdgesCDN2 = len(edgeListCDN2)\n",
    "numEdgesCDN = numEdgesCDN1 + numEdgesCDN2\n",
    "print(\"CMDN Edges:\", numEdgesCDN)\n",
    "\n",
    "print(\"Initializing Hypergeometric Test...\")\n",
    "# find average degrees of the bipartite networks\n",
    "avgDegCDN1 = (2*numEdgesCDN1)/(lenCeRListCDN + lenMiRListCDN)\n",
    "avgDegCDN2 = (2*numEdgesCDN2)/(lenMiRListCDN + lenDiseaseListCDN)\n",
    "\n",
    "# hypergeometric test\n",
    "meanCDN = (avgDegCDN1*avgDegCDN2)/lenMiRListCDN\n",
    "sigmaCDN = math.sqrt(meanCDN*(1-(avgDegCDN1/lenMiRListCDN)))\n",
    "print(\"CDN Mean:\", meanCDN)\n",
    "print(\"CDN Standard Deviation:\", sigmaCDN)\n",
    "\n",
    "print(\"Processing: Creating the First Master miRNA List\")\n",
    "superMiRListCDN1 = []\n",
    "for x in tqdm(range(len(ceRListCDN))):\n",
    "    mirsetCDN1 = []\n",
    "    for y in range(len(edgeListCDN1)):\n",
    "        if ceRListCDN[x] == edgeListCDN1[y][0]:\n",
    "            mirsetCDN1.append(edgeListCDN1[y][1])\n",
    "    superMiRListCDN1.append(mirsetCDN1)\n",
    "print(\"First Master miRNA list complete.\")\n",
    "\n",
    "print(\"Processing: Creating the Second Master miRNA List\")\n",
    "superMiRListCDN2 = []\n",
    "for x in tqdm(range(len(diseaseListCDN))):\n",
    "    mirsetCDN2 = []\n",
    "    for y in range(len(edgeListCDN2)):\n",
    "        if diseaseListCDN[x] == edgeListCDN2[y][1]:\n",
    "            mirsetCDN2.append(edgeListCDN2[y][0])\n",
    "    superMiRListCDN2.append(mirsetCDN2)\n",
    "print(\"Second Master miRNA list complete.\")\n",
    "\n",
    "print(\"Creating CDN: Finding intersections of the miRNA lists\")\n",
    "CDN = []\n",
    "for x in tqdm(range(len(ceRListCDN))):\n",
    "    list1 = superMiRListCDN1[x]\n",
    "    for y in range(len(diseaseListCDN)):\n",
    "        list2 = superMiRListCDN2[y]\n",
    "        count = len(union(list1, list2))\n",
    "        zScore = (count - meanCDN)/sigmaCDN\n",
    "        if zScore > 3:\n",
    "            CDN.append([ceRListCDN[x], diseaseListCDN[y], zScore])\n",
    "print(\"CDN 100% Complete.\")\n",
    "\n",
    "# create node and edge lists for the CGN after hypergeometric reduction\n",
    "print(\"Creating node and edge lists...\")\n",
    "ceRNACDNList = []\n",
    "diseaseCDNList = []\n",
    "for x in range(0, len(CDN)):\n",
    "    if CDN[x][0] not in ceRNACDNList:\n",
    "        ceRNACDNList.append(CDN[x][0])\n",
    "    if CDN[x][1] not in diseaseCDNList:\n",
    "        diseaseCDNList.append(CDN[x][1])\n",
    "print(\"CDN ceRNA Nodes:\", len(ceRNACDNList))\n",
    "print(\"CDN Disease Nodes:\", len(diseaseCDNList))\n",
    "print(\"CDN Edges:\", len(CDN))\n",
    "\n",
    "# write the list as a pandas dataframe and export as a csv file\n",
    "columnsCDN = ['ceRNA','Disease','Score']\n",
    "dfCGN = pd.DataFrame(CDN, columns = columnsCDN)\n",
    "dfCGN.to_csv('/networks/ceRNetwork_CDN.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Master miRNA Lists for Each ceRNA Node\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-164-8708707eec2b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0mtempmirlist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlenBeforeDrop1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mceRNetwork1Duplicates\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mceRNACRNList1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mceRNetwork1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'ceRNA'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m                 \u001b[0mtempmirlist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mceRNetwork1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'miRNA'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/series.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    764\u001b[0m         \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply_if_callable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 766\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    767\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_scalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_value\u001b[0;34m(self, series, key)\u001b[0m\n\u001b[1;32m   3101\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3102\u001b[0m             return self._engine.get_value(s, k,\n\u001b[0;32m-> 3103\u001b[0;31m                                           tz=getattr(series.dtype, 'tz', None))\n\u001b[0m\u001b[1;32m   3104\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3105\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minferred_type\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'integer'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'boolean'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_value\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.index.Int64Engine._get_index_values\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m()\u001b[0m\n\u001b[1;32m   1921\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1922\u001b[0m         \u001b[0;31m# property, for now, slow to look up\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1923\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ndarray_values\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1924\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1925\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_validate_index_level\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/base.py\u001b[0m in \u001b[0;36m_ndarray_values\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    797\u001b[0m         \"\"\"\n\u001b[1;32m    798\u001b[0m         \u001b[0;31m# type: () -> np.ndarray\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 799\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mis_extension_array_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    800\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ndarray_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    801\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/dtypes/common.py\u001b[0m in \u001b[0;36mis_extension_array_dtype\u001b[0;34m(arr_or_dtype)\u001b[0m\n\u001b[1;32m   1710\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1711\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr_or_dtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mABCIndexClass\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mABCSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1712\u001b[0;31m         \u001b[0marr_or_dtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marr_or_dtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1713\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1714\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr_or_dtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mExtensionDtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mExtensionArray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36m_values\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    693\u001b[0m         \u001b[0m_ndarray_values\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    694\u001b[0m         \"\"\"\n\u001b[0;32m--> 695\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    696\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    697\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mvalues\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    659\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    660\u001b[0m         \u001b[0;34m\"\"\" return the underlying data as an ndarray \"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 661\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    662\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    663\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# MAKING THE MIRNA PROJECTION (CMP)\n",
    "\n",
    "print(\"Creating Master miRNA Lists for Each ceRNA Node\")\n",
    "superMiRListCMP = []\n",
    "for x in tqdm(range(len(ceRNACRNList1))):\n",
    "    tempmirlist = []\n",
    "    for y in range(lenBeforeDrop1):\n",
    "        if ceRNetwork1Duplicates[y] == False:\n",
    "            if ceRNACRNList1[x] == ceRNetwork1['ceRNA'][y]:\n",
    "                tempmirlist.append(ceRNetwork1['miRNA'][y])\n",
    "    superMiRListCMP.append(tempmirlist)\n",
    "print(\"Master miRNA List Complete.\")\n",
    "\n",
    "print(\"Finding Intersections of miRNA Lists for Each ceRNA-ceRNA Pair\")\n",
    "CMP = []\n",
    "for x in range(len(superMiRListCMP)):\n",
    "    list1 = superMiRListCMP[x]\n",
    "    for y in range(x + 1, len(superMiRListCMP)):\n",
    "        list2 = superMiRListCMP[y]\n",
    "        count = len(union(list1, list2))\n",
    "        if count != 0:\n",
    "            CMP.append([superMiRListCMP[x], superMiRListCMP[y], count])\n",
    "colsCMP = ['ceRNA1', 'ceRNA2', 'Score']\n",
    "dfCMP = pd.DataFrame(CMP, columns = colsCMP)\n",
    "dfCMP.to_csv('/networks/miRNA_projection.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CONSTRUCTING TRIPARTITE NETWORKS: CeRNetwork (CMGN)\n",
    "\n",
    "# create the edge lists\n",
    "CMGN1 = edgeListCGN1\n",
    "CMGN2 = []\n",
    "for x in range(0, len(edgeListCGN2)):\n",
    "    CMGN2.append([edgeListCGN2[x][1], edgeListCGN2[x][0]])\n",
    "for x in range(0, len(CMGN2)):\n",
    "    CMGN1.append(CMGN2[x])\n",
    "\n",
    "columnsCMGN = ['ceRNA_mRNA', 'miRNA']\n",
    "dfCMGN = pd.DataFrame(CMGN1, columns = columnsCMGN)\n",
    "dfCMGN.to_csv('/networks/ceRNetwork_CMGN.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CONSTRUCTING TRIPARTITE NETWORKS: CeRDiseaseNetwork (CMDN)\n",
    "\n",
    "# create the edge lists\n",
    "CMDN1 = edgeListCDN1\n",
    "CMDN2 = []\n",
    "for x in range(0, len(edgeListCDN2)):\n",
    "    CMDN2.append([edgeListCDN2[x][1], edgeListCDN2[x][0]])\n",
    "for x in range(0, len(CMDN2)):\n",
    "    CMDN1.append(CMDN2[x])\n",
    "\n",
    "columnsCMDN = ['miRNA_Disease', 'Gene']\n",
    "dfCMDN = pd.DataFrame(CMDN1, columns = columnsCMDN)\n",
    "dfCMDN.to_csv('/networks/ceRNetwork_CMDN.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CONSTRUCTING TRIPARTITE NETWORKS: MGDN\n",
    "\n",
    "# create the edge lists\n",
    "MGDN1 = edgeListMDN1\n",
    "MGDN2 = []\n",
    "for x in range(0, len(edgeListMDN2)):\n",
    "    MGDN2.append([edgeListMDN2[x][1], edgeListCDN2[x][0]])\n",
    "for x in range(0, len(MGDN2)):\n",
    "    MGDN1.append(MGDN2[x])\n",
    "\n",
    "columnsMGDN = ['miRNA_Disease', 'Gene']\n",
    "dfMGDN = pd.DataFrame(MGDN1, columns = columnsMGDN)\n",
    "dfMGDN.to_csv('/networks/ceRNetwork_MGDN.csv', index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}