Notebook_CORD-19_3_network_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CORD-19 citation network analysis\n",
    "\n",
    "In this notebook, we build a bibliographic coupling citation network of CORD-19 publications, using Dimensions data, and perform clustering using the Leiden algorithm."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# magics and warnings\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import warnings; warnings.simplefilter('ignore')\n",
    "\n",
    "import os, random, codecs, time, pickle\n",
    "import ujson as json\n",
    "from tqdm.notebook import tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "seed = 99\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "\n",
    "import nltk, sklearn\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set(style=\"white\")\n",
    "sns.set_context(\"notebook\", font_scale=1.2, rc={\"lines.linewidth\": 2.5})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataset Dimensions: source of citation data\n",
    "\n",
    "json_sources = \"datasets_output/json_dimensions_cwts\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the list of DOIs to filter papers in CORD-19\n",
    "\n",
    "cord_19_dois = pickle.load(open(\"datasets_output/networks/cord-19-dois.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Could not convert data.\n"
     ]
    }
   ],
   "source": [
    "# build networks and node metadata\n",
    "from collections import OrderedDict\n",
    "import sys\n",
    "\n",
    "metadata = list()\n",
    "metadata_dict = dict()\n",
    "citations = OrderedDict()\n",
    "total_publications = 0\n",
    "\n",
    "for root, dirs, files in os.walk(json_sources):\n",
    "    for f in files:\n",
    "        all_data = codecs.open(os.path.join(root,f),encoding='latin1').read()\n",
    "        for line in all_data.split(\"\\n\"):\n",
    "            total_publications += 1\n",
    "            if line:\n",
    "                try:\n",
    "                    data = json.loads(line)\n",
    "                    if \"doi\" in data.keys() and data[\"doi\"] in cord_19_dois: # filter by DOIs in CORD-19\n",
    "                        if \"references\" in data.keys() and len(data[\"references\"])>0: # filter out papers without outbound citations\n",
    "                            title = \"\"\n",
    "                            year = \"\"\n",
    "                            doi = \"\"\n",
    "                            times_cited = \"\"\n",
    "                            journal = \"\"\n",
    "                            if \"title\" in data.keys() and data[\"title\"]:\n",
    "                                title = data[\"title\"].replace('\"',\"\")\n",
    "                            if \"journal\" in data.keys() and \"title\" in data[\"journal\"].keys() and data[\"journal\"][\"title\"]:\n",
    "                                jorunal = data[\"journal\"][\"title\"]\n",
    "                            if \"year\" in data.keys() and data[\"year\"]:\n",
    "                                year = str(data[\"year\"])\n",
    "                            if \"doi\" in data.keys() and data[\"doi\"]:\n",
    "                                doi = data[\"doi\"]\n",
    "                            if \"times_cited\" in data.keys():\n",
    "                                times_cited = str(data[\"times_cited\"])\n",
    "                            metadata.append((data[\"id\"],'\"'+title+'\"',year,'\"'+doi+'\"','\"'+journal+'\"',times_cited))\n",
    "                            metadata_dict[data[\"id\"]] = {\"title\":title,\"year\":year,\"doi\":doi,\"journal\":journal,\"times_cited\":times_cited}\n",
    "                            citations[data[\"id\"]] = data[\"references\"]\n",
    "                except ValueError:\n",
    "                    print(\"Could not convert data.\")\n",
    "                except:\n",
    "                    print(line)\n",
    "                    continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38754"
      ]
     },
     "execution_count": 205,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(citations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43956"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cord_19_dois)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50033"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_publications"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# therefore:\n",
    "print(\"Not in CORD-19 \"+str(total_publications-len(cord_19_dois)))\n",
    "print(\"Without citations \"+str(len(cord_19_dois)-len(citations)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create directed network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e89d1c5cc57544539bf13f731383b33c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# create bibc network\n",
    "# this is a super naive method, and takes a while\n",
    "\n",
    "bibc_edge_list = list()\n",
    "items = [(k,v) for k,v in citations.items()]\n",
    "\n",
    "for n,(k1,v1) in tqdm(enumerate(items)):\n",
    "    for k2,v2 in items[n+1:]:\n",
    "        edge_weight = len(set(v2).intersection(set(v1))) / len(set(v2).union(set(v1))) # fractional edge weight\n",
    "        if edge_weight > 0:\n",
    "            bibc_edge_list.append((k1,k2,str(edge_weight)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export csv\n",
    "\n",
    "with open(\"datasets_output/networks/bib_coupling_sources_cord19_doi_edgelist_norm.csv\",\"w\") as f:\n",
    "    f.write(\"Source,Target,Weight\\n\")\n",
    "    for edge in bibc_edge_list:\n",
    "        #if float(edge[2])>1:\n",
    "        f.write(\",\".join(edge)+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"datasets_output/networks/bib_coupling_sources_cord19_doi_nodelist.csv\",\"w\") as f:\n",
    "    f.write(\"Id,Title,Year,DOI,Journal,Times_cited\\n\")\n",
    "    for node in metadata:\n",
    "        f.write(\",\".join(node)+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11781927"
      ]
     },
     "execution_count": 211,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(bibc_edge_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "weight_list = [float(x[2]) for x in bibc_edge_list if float(x[2])>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0136986301369863"
      ]
     },
     "execution_count": 213,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.median(weight_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1ab1029690>"
      ]
     },
     "execution_count": 214,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD+CAYAAADF/ZVnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAeHklEQVR4nO3dfVBU59kG8Gt3YZdFlJiKxBbfRKRMjBhFKGqnxkTBdiKh1g/UmknroI06E22nzaTpNCHaVE07On0p2kmm1kzVET+paUxVmqZ1TOxrrEaDiZAKRtEqVIIg7Aecfd4/lv10YQ/sLsuz5/rNZJTDAvcJM5f33M9zntUJIQSIiCgm6KNdABERhQ9DnYgohjDUiYhiCEOdiCiGMNSJiGIIQ52IKIbEqXnRgQMHUFpaCqPR6L728ssvY86cOfjFL36BY8eOQa/XY9myZXj22WdV/WCr1Yrq6mqkpKTAYDD0r3oiIo1RFAVNTU3IyspCQkLCPZ9XFeqffPIJli1bhp/85Cc+1zdv3oz6+npUVVWhra0Ny5cvR2pqKubOnRv0e1ZXV2Pp0qUqb4OIiLzt3r0bubm591xXFeoXL17EM888c8/1yspKbNq0CcnJyUhOTkZJSQkqKipUhXpKSoq7sAceeEBNGUREmnfz5k0sXbrUnaH+goa6oiioqanB4cOHsXHjRpjNZixcuBCLFi1CU1MTMjIy3K8dM2YMamtrVRXmGrk88MADSEtLU/U1RETk1NPYOmioNzc3IysrC3PnzkV5eTkuX76M1atXw263A4DPTMdsNsNqtYapZCIi6qugoZ6SkoJdu3a5Px43bhyefvppnDhxAgBgs9ncn7NYLEhMTIxAmUREpEbQLY2fffYZysrKfK51dnbCZDIhJSUFdXV17uv19fU+4xgiIhpYQUN92LBh2LFjB/bt2weHw4Hq6mrs3LkT8+bNQ1FREbZu3Yrm5mY0NDRg+/btKCoqGoi6iYgogKDjl9TUVGzbtg2//vWvsXHjRgwfPhyrV6/Gt771LTzxxBPYtGkTCgsL4XA4sGjRIixZsmQg6iYiogB00TpPvaGhAbNmzcK7774b8u4XIQR0Ol2YKiMiGryCZaf0xwSc+fQWlr78F+ytqol2KUREUSd9qJ8414C2jk4cP3012qUQEUWd9KHepTinR/ZOJcqVEBFFn/Sh7nA4Q72ToU5EJH+oKw4HAMDe5YhyJURE0RcDod7dqXc5EKWNPEREg0bMhDrAbp2ISPpQdyieUOdcnYi0Tv5QF+zUiYhcpA91n/ELO3Ui0jj5Q13xdOed7NSJSOPkD3V26kREbjEV6uzUiUjrpA91h8+WRnbqRKRt0oe67/iFnToRaZv0oe5weC+UslMnIm2TPtTZqRMRecRYqLNTJyJtkz7UHTz7hYjITfpQ993SyE6diLRN/lBXOFMnInKRPtS9d79wnzoRaZ30oe4zfmGnTkQaJ32o84lSIiIP6UOdZ78QEXlIHepCCO5TJyLyInWoO/zeZ5r71IlI6+QOdYdviHOhlIi0TupQV/xadS6UEpHWSR3qDv9Q50ydiDRO6lC/t1Pn+IWItE3uUFd8Q72TnToRaZzcoe63UMpOnYi0TnWot7a24vHHH8ehQ4fcH69ZswY5OTmYPn06Dh48GLEie+KX6ezUiUjz4tS+sLS0FLdu3XJ//Morr0Cv1+PkyZO4cuUKSkpKMHr0aOTl5UWk0EDYqRMR+VLVqVdWVuLu3bvIzMwEAFgsFhw7dgxr166F2WzGuHHjsHDhQuzduzeixfrz3/3C89SJSOuChvq1a9dQXl6ODRs2uK9duXIFOp0ODz30kPtaeno6amtrI1JkT+7Z/cKHj4hI43oNdUVR8Pzzz+OFF15ASkqK+3p7ezuMRiN0Op37mtlshtVqjVylgerzC3XFIaAoDHYi0q5eQ33btm0YM2YMZs+e7XM9MTERdrsdQnhC1WKxIDExMTJV9sB//ALwpEYi0rZeF0qPHDmCxsZGVFVVAXB26OvWrcOTTz4JIQQaGhowevRoAEBdXR0yMjIiX7EX/4VSwLlYmmAa0DKIiAaNXkP96NGjPh9/+9vfxve+9z3MmzcPHR0d2Lx5M375y1/i2rVr2L9/PzZu3BjRYv35j18ALpYSkbb1++Gj9evXQ6/XY+bMmVixYgVWrlyJGTNmhLO2oPyfKAUAG/eqE5GGqd6nDgCHDx92/z05ORlbtmwJe0F9EXCmzh0wRKRhUh8TECjUefwuEWmZ1KEeaKbOvepEpGWSh/q9Ac6FUiLSMslDPdD4hZ06EWlXzIU6F0qJSMukDnUulBIR+ZI61LlQSkTkS+pQd3ChlIjIh9ShHuiJUnbqRKRlcoc6z34hIvIhdag7BM9+ISLyJnWoBxq/8Dx1ItIyuUPda/xi0DvfhcnOTp2INEzqUPfe/ZJgch44yU6diLRM6lD37tTNRgMA7n4hIm2TOtS9nyg1GZ2dOp8oJSItkzrUvTv1BJOzU+f4hYi0LCZCXa/XwRjnGr+wUyci7ZI71BVnV27Q62CMd94KO3Ui0jKpQ91qd3blep0OrueQOqydaGzuQFuHPYqVERFFh9Sh7hq1CCHQbukEALR22HG2phEWa1c0SyMiigqpQ901U9fpdTAYnLcS6ClTIiKtkDrUXVsa9Tod4gzOJ0q7FM7UiUi7pA519+4XneeYAHbqRKRlUoe6I9D4JcAbZxARaYXcoS68xi961/hFQAQ4kpeISAvkDnVXp66Du1MHAp+zTkSkBVKHuuK1UGroXigFOFcnIu2SOtQdXscExOk9t8IdMESkVVKHumtRVOffqQd471IiIi2QOtRdG130fjN1jl+ISKvkDnXhWij17H4BOH4hIu2SOtR9F0r191wnItIaVaH+3nvv4amnnkJ2djby8/NRUVEBALDb7XjppZeQl5eHqVOn4vXXX49osf5c71Gq18N9TADgOZKXiEhr4oK9oLGxEWvWrEF5eTlmzJiBixcvYsmSJZgwYQKOHj2K+vp6VFVVoa2tDcuXL0dqairmzp07ELV77VPXwcDdL0REwTv1kSNH4tSpU5gxYwYcDgdaWlpgMBgwZMgQVFZWYuXKlUhOTkZaWhpKSkrcXfxA8CyU6nw69S6OX4hIo4J26gCQlJQEi8WC3NxcdHV1YcWKFbj//vvR1NSEjIwM9+vGjBmD2traiBXrz7Ol0X/3Czt1ItImVaEOACaTCefOnUNNTQ1+8IMfICEhAQDcfwKA2WyG1WoNf5U98H74yKDnE6VERKp3v+j1ehiNRkyYMAHFxcWorq4GANhsNvdrLBYLEhMTw19lDxTvA728OnWOX4hIq4KG+unTpzFv3jyfa3a7HcOGDUNKSgrq6urc1+vr633GMZHms1DK3S9ERMFDfdy4cbh16xZ27NgBRVFw9uxZHDx4EAsWLEBRURG2bt2K5uZmNDQ0YPv27SgqKhqIugH4ndKo5xOlRERBZ+pDhw7FG2+8gVdffRXl5eUYNWoUXn31VeTl5WHixInYtGkTCgsL4XA4sGjRIixZsmQg6gbgN1P32f3CTp2ItEnVQun48eOxZ8+ee66bTCaUlpaitLQ07IWp4f1Eqes/hxDs1IlIs6Q+JsB7/ALA3a1zpk5EWiV1qLt3v3RvZ3TtgOHuFyLSKqlD3Xv3C8BOnYgoJkLd9dyR692PujhTJyKNipFQ9+vUufuFiDRK6lBX/Mcv3Z06d78QkVZJHere+9QBz5nqPHqXiLRK2lB3OARc/fg9Wxq5+4WINEreUBee4Nb7jV/YqRORVkkb6t7duCvU49ipE5HGyRvqXt24rvsuXG+UwX3qRKRV0oa6I0Cn7nqjDO5TJyKtkjbUvUcsOp3vMQHs1IlIq6QNdd9O3flnfJzzdjoVB4Rgt05E2iNtqPsslHanuivUheAOGCLSppgIddf4xRXqAGC1KwNeExFRtEkc6p5OXO8OdYP7mo2hTkQaJG2oO3w6deef7NSJSOukDfWAM3WDd6h3DXhNRETRJm2oB9qn7t2pc/xCRFokbah7H6/rHr/Ec6ZORNomb6gHXCj1Gr90cvxCRNojbah7v7lRoC2N7NSJSIukDXWfTr37LhjqRKR1Eod6gIVSA7c0EpG2xUSou8YvOp3OfaiXjTN1ItIgaUPdEWCfOuAZwXD8QkRaFBOhrvNkujvUOX4hIi2SNtQDbWkE2KkTkbZJHOr3ztQBwOjq1DsZ6kSkPfKGuuI9U/dcd53UaOPZL0SkQfKGeoAtjQDHL0SkbdKGuqOH8QsXSolIy1SF+vvvv4958+Zh8uTJKCgoQEVFBQCgtbUVa9asQU5ODqZPn46DBw9GtFhvSoD3KAW8Q53jFyLSnrhgL/jPf/6D5557Dq+99hpmzZqF6upqLF++HF/5yldQWVkJvV6PkydP4sqVKygpKcHo0aORl5cX8cIdXrtfdD771F0zdXbqRKQ9QTv169evo7CwEAUFBdDr9Xj00UeRl5eHU6dO4dixY1i7di3MZjPGjRuHhQsXYu/evQNRd9CZuuIQ6Ozim08TkbYE7dRzc3ORm5vr/rilpQVnzpxBbm4udDodHnroIffn0tPT8be//S0ihfoLFuqAcwQTH2cckHqIiAaDPi2UtrW1YdWqVZg4cSLGjx8Po9Hos0hpNpthtVrDXmQgwZ4oBQCLjXN1ItIW1aFeX1+P4uJijBgxAmVlZRgyZAjsdjuE8ISrxWJBYmJiRAr1F+g9SgG/Tp2hTkQaoyrUP/zwQxQXFyM/Px9lZWUwmUx48MEHIYRAQ0OD+3V1dXXIyMiIWLHevI8J8N3S6HlLO25rJCKtCRrqV69exbPPPos1a9bgxz/+sTtAhwwZgvz8fGzevBnt7e24dOkS9u/fj6KioogXDQAOpfctjQDHL0SkPUEXSnfv3o329nZs2bIFW7ZscV//7ne/i/Xr12PdunWYOXMmjEYjVq5ciRkzZkS0YBfX+EWHwA8fARy/EJH2BA31F198ES+++GKPn/cO+oHk6J7lewc64Nepc/xCRBoj7TEBrgO99H534DNTZ6dORBojb6g7AnfqRr996kREWiJxqDt3v+j9Qj3OZ6GU4xci0haJQ901fvENdb1OhziD8xrHL0SkNdKGusM9frn3c665uoXjFyLSGOlD3X/8Anh2wHCfOhFpjbSh3tNCKeB1pjpDnYg0Rt5Q72FLIwDEG1yhzoVSItIWeUO9h90vgNf4hTN1ItIYaUO9pydKAc9CKccvRKQ10oa6a/wSePeLq1Pn+IWItEXeUO9hnzrAhVIi0i5pQ13NlkaGOhFpjbSh7lkovfdzrpm6vcsBReGbTxORdkgb6q43PuptnzrAdz8iIm2RNtRdnXrwUOcIhoi0Q+JQ7+XhI76lHRFplPyhHqxT51OlRKQh0oa6Qwn+8BHAp0qJSFvkDXWhrlPn+IWItETaUHcvlAaZqXOvOhFpibyhrqjt1DlTJyLtkDfU1S6UcqZORBoifahz/EJE5CFtqPd29otBr3e/+TQXSolIS6QP9UBbGgEgwRgHgMcEEJG2SBvqvR3oBQAmo3OvOjt1ItISiUO95/EL4Al1LpQSkZZIH+q6Hlr1hPju8Qu3NBKRhkgb6p6F0sCf5/iFiLRI2lBXgiyUcvxCRFokZagLIXrd0ggACa5Q5/iFiDSkT6F+4cIFTJs2zf2x3W7HSy+9hLy8PEydOhWvv/562AsMxBXoQOA3ngYAU3z3+IWdOhFpSJyaFwkhcODAAbz22ms+13/729+ivr4eVVVVaGtrw/Lly5Gamoq5c+dGpFgXxSvUe2jUYXLtU+dMnYg0RFWnXlZWhj179mDVqlU+1ysrK7Fy5UokJycjLS0NJSUlqKioiEih3hw+oR5k/GJX+ObTRKQZqkJ98eLFOHToELKystzXWltb0dTUhIyMDPe1MWPGoLa2NvxV+vHu1Huaqd831OT+e1OLJeI1ERENBqpCPTU19Z5rHR0dAICEhAT3NbPZDKvVGqbSeuYT6j3cQer9ie6/32hqj3RJRESDQr93v5jNZgCAzWZzX7NYLEhMTOzpS8LGdUQA0HOn7h3q15vuRrwmIqLBoN+hnpycjJSUFNTV1bmv1dfX+4xjIkXNTD3JHO/eAXO5oQWNzR1obO5AW4c94vUREUVLSPvUi4qKsHXrVjQ3N6OhoQHbt29HUVFRuGrrkZrdL/YuB5IS4wEAn11rwdmaRpytaYTFyt0wRBS7Qgr1tWvX4qtf/SoKCwuxYMECfPOb38SSJUvCVVuP1OxTBzyLpS13bT2+hogolqjap+4yZcoUnDlzxv2xyWRCaWkpSktLw15Yb9TsfgGA+5Kcod7WboficMDQ06oqEVGMkDLlvPed9zRTB4Dk7lAXAFrbOUsnotgnZ6ir2NIIeDp1AGhp4wiGiGKflKGuZvcL4PsA0p277NSJKPZJGepqZ+oJRgOM8c5b5GIpEWmBlKHus/ul50yHTqdzj2A4fiEiLZAy1BWV4xfAs1h6h506EWmApKHudUxAb606PIuldy2d6OJpjUQU46QMdYeKJ0pdfBdL2a0TUWyTMtTVLpQCnvELwMVSIop9MR/q3nvV77RxWyMRxTY5Q13xGr8EmambjAaYTc7TENipE1GskzLU1W5pdElOMgJgqBNR7JM+1INtaQQ8IxgulBJRrJMy1NW885E312Jph7ULdy2dEauLiCjaJA11dQd6uXi/td2Fz5oiURIR0aAgfairGb98OSUJJqPzre3OfHorYnUREUWb9KGuZvxi0OuQ/uVkAMDF+tto5wiGiGKUlKHelydKXTLSnKHepQic/uRmJMoiIoo6KUO9L2e/uHxl5FD3COb98zciUhcRUbRJGeqOPo5fAN8RzNmaRnRYOYIhotgjZaj7PFGqdv4CYGz3CKazy4HTFzmCIaLYI2eo93FLo0vayKFITHAeGXCSIxgiikFShrpD9K9TN+h1yM4cCcA5gmlt5wFfRBRbpAx1n/FLH792atYDAJwjmDffvhjGqoiIok/OUO/e/WLQ6/rUqQPAV0ffh6z0LwEAqk5fxcmPrqOxuQNtHezaiUh+Uoa6a/eL2u2M3uxdDkzKTIGh+2vfOPwxPvz0JizWrrDWSEQUDVKGuhJCqAPOA76+9kgqAOCLVhvO1fA8GCKKDVKGurtT7+PoxdukzBTcPywBgPM8mGu32sJSGxFRNEkZ6q5O3dDPTt35tXo8PjnN/f3K9n2EL1qtYamPiChapA71/o5fXEaNGIJpE0YBAJpbrfjljtOwdyoh10dEFC1yhrri2f0SquzMFDz80HAAQM3VL/C/FecY7EQkrbhoF9Af4erUAefDS49PToOiCHx2rQUnPrqO6rrb+PZj6Zg2YRSSEo0YmmgM+ecQEQ0EKTt11xOloSyUejPo9VhW+Ai+lOxcOG1utWLH25/ghfKTePf0VXQpjiDfgYhocAg51C9duoRFixZh0qRJeOqpp3DhwoVw1NUrhxK+Tt0lKdGI4lmZeCInDUPM8QCAL9ps2P7ni1ix4a/Y+9caVP3f5zhxrgHnP2tCZxeDnogGn5DGL3a7HatXr8YzzzyDXbt24fjx4ygpKcF7772HpKSkcNV4j3DsfglEr9fhkTFfQub/DMfHl/+Lj2qb0GHtwn9bLNj1l0s+rx2aGI8nckdjRnYahg9NgDFeD5PRgASjlBMtIooRISXQ6dOn0dnZie9///sAgDlz5mDXrl145513UFxcHI76AnIdExDOTt1bnEGP7MyRmDB2BO7cteEvp67gzl3fYwTaOjrx1ok6vHWizuf6EHM8Uu4zY8R9Ztw/LAHDh5qQnGSCTuf8x0hRBIzxephNcUgwxWFIQhySzEYMMcdDr9dBURzucY+++xiE//y3HZc+b0bt519AABj30P14ZMz9ePCBYe7/B0IAQgj3aMoYb4Ap3gCdToe7HXa0dthhsysYYo7H0EQjkrp/nuh+fV+PW/AmhICtU4HV5lxgNifEwRinD+l7ElH/hBTq//73vzF27Fifa+np6aitrQ36tYriDICbN/t+rvmdLxrR2dEM2107bjcNCfiamyYLbjfdCfl6eloyvpVzH2x2Z9gqigO371hx83Y7Ll+/9/UtHUDLbeCzPt+Veh9+FJnvq9ProNfpoAMggr7aeZiaTqdDl6JA+H2BTq9DvEHvue7/AsD9XoQ9ZX9f/knorV6136e//wYFurVwfv/+6K2mcNQR7J7573lw48eOwMrvTOhz8+PKTFeG+gsp1Ds6OpCQkOBzzWw2w2KxBP3apibno/lLly7t98+vB/D+vn5/ORFR1HwK4MDW/n99U1MTHnzwwXuuhxTqiYmJsNlsPtcsFgsSExODfm1WVhZ2796NlJQUGAyGUMogItIMRVHQ1NSErKysgJ8PKdTHjh2LN9980+daXV0d5s6dG/RrExISkJubG8qPJyLSpEAduktIWxqnTJkCIQTefPNNdHZ24siRI6ipqUFBQUEo35aIiPpJJ4TaZZ7AamtrUVpaikuXLiEtLQ0/+9nPMG3atHDVR0REfRByqBMR0eAh5TEBREQUGEOdiCiGMNSJiGIIQ52IKIYM+lBXewpkNE6LjAS19/H+++9j3rx5mDx5MgoKClBRUTHAlYZPX393ra2tePzxx3Ho0KEBqjC81N5vY2MjVq9ejZycHHz961/Hb37zmwGuNHzU3vNHH32EBQsWICcnBwUFBdi/f/8AVxpeFy5c6HU34I0bN7Bs2TJkZ2cjPz8f//jHP0L/oWIQs9ls4oknnhA7duwQdrtdvP322yI3N1e0tbX163WDndr7uHHjhsjOzhbHjx8XiqKI8+fPi6997WvixIkTUaq8//rzu/vhD38oHn74YXHw4MEBrDQ8+nK/8+fPFy+//LKwWq3i6tWrYsaMGeKtt96KQtWhUXvPiqKIadOmiT/96U9CCCHOnz8vsrKyxKeffhqNskPicDjEvn37RE5OjsjJyenxdYsWLRIbN24UNptNfPDBByI7O1tcvXo1pJ89qDt171Mg4+PjMWfOHGRkZOCdd97p1+sGO7X3cf36dRQWFqKgoAB6vR6PPvoo8vLycPbs2ShV3n99/d1VVlbi7t27yMzMHOBKw0Pt/Z4/fx7Xrl3Dz3/+c5hMJowePRo7d+7ElClTolR5/6m95zt37uD27dsQQkAIAZ1Oh7i4OMTHx0ep8v4rKyvDnj17sGrVqh5fU19fj+rqaqxZswZGoxHTpk3DzJkzceDAgZB+9qAOdbWnQIZyWuRgovY+cnNzsX79evfHLS0tOHPmDB555JEBqTOc+vK7u3btGsrLy7Fhw4aBKi/s1N5vdXU1MjMzUV5ejunTpyM/Px9VVVUYOXLkQJYbFmrvefjw4Xj66afx05/+FOPHj8eCBQvwox/96J6vlcHixYtx6NChHs9nAYDLly9j1KhRPmdlpaeno6amJqSfPahDXe0pkKGcFjmY9Oc+2trasGrVKkycOBGzZs2KdIlhp/aeFUXB888/jxdeeAEpKSkDWWJYqb3fO3fu4F//+hfi4uLw7rvvory8HH/4wx/w5z//eSDLDQu19+xwOGA0GrF582acP38eO3fuxNatW3Hy5MmBLDcsUlNTg76mvb094P8Xq9Ua0s8e1KGu9hTIUE6LHEz6eh/19fUoLi7GiBEjUFZWBr1+UP86A1J7z9u2bcOYMWMwe/bsgSwv7NTer9FoRFJSEp577jkYjUY8/PDDWLBgAaqqqgay3LBQe8/Hjx/HuXPnMGfOHMTHxyMvLw/z58/H3r17B7LcAROp3BrUKTB27FjU19f7XKurq0NGRka/XjfY9eU+PvzwQxQXFyM/Px9lZWUwmUwDVWZYqb3nI0eO4NixY8jNzUVubi5qa2uxbt06vPLKKwNYbejU3m96ejosFgvsds87bvX0pgiDndp7vnnzps/9AkBcXBzi4mLzLSLHjh2LGzdu+HTmYcmtkJZZI8xms4nHHnvMZ9U8Oztb3L59u1+vG+zU3sfnn38usrOzxR//+McoVRo+/f3dFRUVSbv7Rc39Wq1W8dhjj4n169cLm80mLl26JKZOnSqOHj0apcr7T+0919bWiqysLFFRUSEcDof4+OOPxZQpU0RVVVWUKg/dP//5z153v8yfP19s2LBB2Gw2cerUKTFp0iRRU1MT0s8c1KEuhBA1NTVi8eLFYtKkSaKwsFB88MEHQgghfve734knn3wy6Otko+Z+N2zYIDIzM8WkSZN8/vvVr34VzdL7Te3v2JusoS6E+vu9evWqWLFihcjLyxPf+MY3xO9///tolRwytff897//XXznO98RkydPFrNnzxb79u2LVslh4R/qhw8fFpMmTXJ/fOPGDVFSUiImT54sZs2aJY4cORLyz+QpjUREMWRQz9SJiKhvGOpERDGEoU5EFEMY6kREMYShTkQUQxjqREQxhKFORBRDGOpERDGEoU5EFEP+H777gQUb/TJIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(weight_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [],
   "source": [
    "import igraph as ig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = ig.Graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "edges = [(int(k[0].replace(\"pub.\",\"\")),int(k[1].replace(\"pub.\",\"\"))) for k in bibc_edge_list]\n",
    "nodes = list(set([k[0] for k in edges]).union(set([k[1] for k in edges])))\n",
    "weights = [float(k[2]) for k in bibc_edge_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mapping of nodes to identifiers for this stupud igraph to work\n",
    "\n",
    "node_map = {k:n for n,k in enumerate(nodes)}\n",
    "edges_int = [(node_map[k[0]],node_map[k[1]]) for k in edges]\n",
    "nodes_int = [node_map[k] for k in nodes]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38290"
      ]
     },
     "execution_count": 219,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(node_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'There are 464 disconnected publications (with references)'"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"There are \"+str(len(citations)-len(node_map))+\" disconnected publications (with references)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "inverse_node_map = {n:k for k,n in node_map.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "g.add_vertices(nodes_int)\n",
    "g.add_edges(edges_int)\n",
    "g.es[\"weight\"] = weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "import leidenalg as la"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 0.05 seems to find a relatively high modularity and reasonable sized clusters. Lowering it brings up one or two large clusters.\n",
    "\n",
    "partition = la.find_partition(g, la.CPMVertexPartition, resolution_parameter = 0.05)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11581408.800000027"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partition.quality()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.29706581071277316"
      ]
     },
     "execution_count": 226,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partition.q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1ace5ffad0>"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD9CAYAAABeOxsXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAewklEQVR4nO3dcVDUdf7H8SekC7vbyRyF1QxJCGhMViC0SiVWstFciKXkYTbXXdidkMo52tRcmUW/bLo5rNQadI7qDps0teU6c1LGqUlLzdKrrIDzWEK6Cn7yU4vdZQk+vz8cvyORggqu4Osx47T7eX+/7Oc96b74fr7f/W6YMcYgIiLntfBQT0BEREJPYSAiIgoDERFRGIiICAoDEREBhoR6AqcjEAiwb98+YmJiuOCCC0I9HRGRAaGjo4Pm5mbGjBlDZGRkl9qADIN9+/Yxc+bMUE9DRGRAevXVV0lPT+8yNiDDICYmBjja0KWXXhri2YiIDAzffvstM2fOtN5Djzcgw+DY0tCll15KbGxsiGcjIjKw/Nzyuk4gi4iIwkBERBQGIiKCwkBERFAYiIgICgMREUFhICIiDNDPGZyp731B/IEfAbBHDuEXDluIZyQiElrn5ZGBP/Aje2qa2FPTZIWCiMj57LwMAxER6UphICIiCgMREVEYiIgICgMREUFhICIiKAxERASFgYiI0MswaGpqoqioiLS0NK6//nqee+45AILBIIsWLcLlcjF+/HhWrlzZZb+KigoyMzNJTU1l4cKF+Hw+q7Zz504mT55MSkoK+fn5NDQ09GFbIiJyKnoVBkVFRcTExPDBBx+wdu1aKisr+ec//8ny5cvxer1UVVWxfv16PB4PlZWVAGzbto2ysjLKy8vZvn07wWCQJ598EoCWlhYeeOAB5syZw+7du8nKyqKgoIDOzs7+61RERE6oxzD45JNPOHDgAI8++igRERFcfvnlVFRUMG7cODweD7NnzyYqKorY2FgKCgpYs2YNAJWVlUybNo2kpCScTicLFixg48aNtLa2UlVVRVJSEtnZ2QwdOpRZs2YRDAbZsWNHvzcsIiLd9RgG+/btY9SoUaxYsYIJEyaQlZVFVVUVkZGRNDc3k5iYaG0bHx9PbW0tAPv37ychIcGqjRgxAmMM9fX13Wo/3VdERM6uHu9aevjwYT7++GNcLhdbt26lrq6OWbNmER0dDUBkZKS1rd1uJxAIAODz+bDb7VYtLCwMm82G3+/vVjv2c/x+f580JSIip6bHMLDZbFx44YXMnTsXgCuvvJK8vDw8Hg8AbW1t1rZ+vx+HwwF0DQYAYwzBYBCn09mtBhAIBKx9RUTk7OpxmWjkyJH4/X6CwaA11tHRQVRUFDExMdTV1VnjXq/XWjZKTEzE6/VatYaGBowxxMXFdasB1NXVdVlyEhGRs6fHMLjhhhuIjo7mmWeeIRgMUlNTw/r167n99tvJzc3lhRdeoKWlhcbGRsrLy8nNzQVgypQprFu3jurqalpbWyktLcXtduNwOHC73VRXV7Np0yba29spLy8nPDwcl8vV7w2LiEh3PYZBREQEq1ev5sCBA0yYMIFZs2Yxa9YssrOzKS4uJikpiZycHPLy8sjOzmbGjBkATJw4kcLCQoqKisjMzMQYQ0lJCQAXXXQRZWVlrFq1CpfLxebNmykrK8Nm0zeOiYiEQpgxxoR6EqeqsbGRSZMmsXXrVmJjY095/6YWH3tqmgAYO3o4w6N1rkJEBr+TvXfqdhQiIqIwEBERhYGIiKAwEBERFAYiIoLCQEREUBiIiAgKAxERQWEgIiIoDEREBIWBiIigMBARERQGIiKCwkBERFAYiIgICgMREUFhICIiKAxERASFgYiIoDAQEREUBiIigsJAREToZRisX7+eq666itTUVOuPx+MhGAyyaNEiXC4X48ePZ+XKlV32q6ioIDMzk9TUVBYuXIjP57NqO3fuZPLkyaSkpJCfn09DQ0PfdiYiIr3WqzD44osv+N3vfsfevXutP3feeSfLly/H6/VSVVXF+vXr8Xg8VFZWArBt2zbKysooLy9n+/btBINBnnzySQBaWlp44IEHmDNnDrt37yYrK4uCggI6Ozv7r1MRETmhXoXB559/TnJycrdxj8fD7NmziYqKIjY2loKCAtasWQNAZWUl06ZNIykpCafTyYIFC9i4cSOtra1UVVWRlJREdnY2Q4cOZdasWQSDQXbs2NG33YmISK/0GAYdHR3U1NTwj3/8gxtvvBG3282qVas4fPgwzc3NJCYmWtvGx8dTW1sLwP79+0lISLBqI0aMwBhDfX19t9pP9xURkbNrSE8btLS0MGbMGO644w5WrFjBf/7zH4qKiggGgwBERkZa29rtdgKBAAA+nw+73W7VwsLCsNls+P3+brVjP8fv9/dJUyIicmp6DIOYmBhWr15tPU9OTuaee+7hvffeA6Ctrc2q+f1+HA4H0DUYAIwxBINBnE5ntxpAIBCw9hURkbOrx2Wif//73yxbtqzLWHt7OxEREcTExFBXV2eNe71ea9koMTERr9dr1RoaGjDGEBcX160GUFdX12XJSUREzp4ew2DYsGG8/PLLvP7663R2drJv3z4qKiqYOnUqubm5vPDCC7S0tNDY2Eh5eTm5ubkATJkyhXXr1lFdXU1rayulpaW43W4cDgdut5vq6mo2bdpEe3s75eXlhIeH43K5+r1hERHprscwuOSSS3jxxRdZs2YNaWlpzJs3j6KiIm677TaKi4tJSkoiJyeHvLw8srOzmTFjBgATJ06ksLCQoqIiMjMzMcZQUlICwEUXXURZWRmrVq3C5XKxefNmysrKsNls/dutiIj8rDBjjAn1JE5VY2MjkyZNYuvWrcTGxp7y/k0tPvbUNAEwdvRwhkfrXIWIDH4ne+/U7ShERERhICIiCgMREUFhICIiKAxERASFgYiIoDAQEREUBiIigsJARERQGIiICAoDERFBYSAiIigMREQEhYGIiKAwEBERFAYiIoLCQEREUBiIiAgKAxERQWEgIiIoDEREBIWBiIhwCmFw5MgRbrrpJt544w3r+bx580hLS2PChAls2LDB2tYYw7PPPktGRgbp6eksWbKEH3/80apv2rQJt9tNSkoK999/PwcPHuzDlkRE5FT1OgwWL17Md999Zz1//PHHCQ8PZ/v27axatYrS0lI+/PBDANauXUtVVRUej4ctW7bw2WefUVZWBsD+/ft55JFHePrpp9m1axdxcXHMnz+/j9sSEZFT0asw8Hg8/PDDD4waNQoAv9/P5s2bKS4uxm63k5yczF133cXatWsBqKys5N577+XSSy8lOjqauXPnWrU333yTW265hfT0dCIiIliwYAF79uyhvr6+fzoUEZEe9RgGBw4cYMWKFSxZssQaq6+vJywsjCuuuMIaGzlyJLW1tcDR3/4TEhK61Jqamjh06FC3mt1u57LLLrP2FRGRs++kYdDR0cGDDz7IQw89RExMjDXe2tqKzWYjLCzMGrPb7QQCAQB8Ph92u92qRUZGAhAIBLrVjtX9fv+ZdyMiIqflpGHw4osvEh8fz6233tpl3OFwEAwGMcZYY36/H4fDAXQNBsB67HA4utWO1Z1O55l1IiIip23IyYpvvfUWTU1NVFVVAUePCJ544gl+9atfYYyhsbGRyy+/HIC6ujoSExMBSExMxOv1kpaWZtViYmIYNmyYVTvG7/fzzTffdFk6EhGRs+ukYfD22293eT5lyhTuvfdepk6dis/no7S0lKeeeooDBw6wbt06nn76aQByc3N56aWXyMjIwG63s3z5cqZMmQJATk4Od999N7t27SI1NZXS0lKSk5OJj4/vpxZFRKQnJw2DkykpKeGJJ57glltuwWazMXv2bCZOnAjAjBkzOHjwIPn5+QQCAW677TaKi4sBGD16NEuWLLEuVb322mt5/vnn+6YbERE5LWHm+IX/AaKxsZFJkyaxdetWYmNjT3n/phYfe2qaABg7ejjDox19PUURkXPOyd47dTsKERFRGIiIiMJARERQGIiICAoDERFBYSAiIigMREQEhYGIiKAwEBERFAYiIoLCQEREUBiIiAgKAxERQWEgIiIoDEREBIWBiIigMBARERQGIiKCwkBERFAYiIgICgMREUFhICIiKAxERIRehsE777zD5MmTSU1NJSsrizVr1gAQDAZZtGgRLpeL8ePHs3Llyi77VVRUkJmZSWpqKgsXLsTn81m1nTt3MnnyZFJSUsjPz6ehoaEP2xIRkVPRYxg0NTUxb948Fi5cyN69e3n++edZsmQJn3/+OcuXL8fr9VJVVcX69evxeDxUVlYCsG3bNsrKyigvL2f79u0Eg0GefPJJAFpaWnjggQeYM2cOu3fvJisri4KCAjo7O/u3WxER+Vk9hsHw4cPZsWMHEydOpLOzk0OHDnHBBRfgdDrxeDzMnj2bqKgoYmNjKSgosI4aKisrmTZtGklJSTidThYsWMDGjRtpbW2lqqqKpKQksrOzGTp0KLNmzSIYDLJjx45+b1hERLrr1TLRhRdeiN/v5+qrr+a+++5j5syZREdH09zcTGJiorVdfHw8tbW1AOzfv5+EhASrNmLECIwx1NfXd6v9dF8RETm7hvR2w4iICPbu3UtNTQ2///3viYyMBLD+C2C32wkEAgD4fD7sdrtVCwsLw2az4ff7u9WO/Ry/339GzYiIyOnp9dVE4eHh2Gw2rr76aqZPn86+ffsAaGtrs7bx+/04HA6gazAAGGMIBoM4nc5uNYBAIGDtKyIiZ1ePYfDhhx8yderULmPBYJBhw4YRExNDXV2dNe71eq1lo8TERLxer1VraGjAGENcXFy3GkBdXV2XJScRETl7egyD5ORkvvvuO15++WU6OjrYs2cPGzZsIC8vj9zcXF544QVaWlpobGykvLyc3NxcAKZMmcK6deuorq6mtbWV0tJS3G43DocDt9tNdXU1mzZtor29nfLycsLDw3G5XP3esIiIdNdjGPziF79g1apVbNmyBZfLxWOPPcb//M//4HK5KC4uJikpiZycHPLy8sjOzmbGjBkATJw4kcLCQoqKisjMzMQYQ0lJCQAXXXQRZWVlrFq1CpfLxebNmykrK8Nms/VvtyIi8rPCjDEm1JM4VY2NjUyaNImtW7cSGxt7yvs3tfjYU9MEwNjRwxkerXMVIjL4ney9U7ejEBERhYGIiCgMREQEhYGIiKAwEBERFAYiIoLCQEREUBiIiAgKAxERQWEgIiIoDEREBIWBiIigMBARERQGIiKCwkBERFAYiIgICgMREUFhICIiKAxERASFgYiIoDAQEREUBiIiQi/D4P3332fq1KmMHTsWt9vNmjVrADhy5Ajz5s0jLS2NCRMmsGHDBmsfYwzPPvssGRkZpKens2TJEn788UervmnTJtxuNykpKdx///0cPHiwj1sTEZHe6jEMvvnmG+bOnUthYSEfffQRpaWlLF26lG3btvH4448THh7O9u3bWbVqFaWlpXz44YcArF27lqqqKjweD1u2bOGzzz6jrKwMgP379/PII4/w9NNPs2vXLuLi4pg/f37/dioiIifUYxh8/fXX5OTk4Ha7CQ8P55prrsHlcrFjxw42b95McXExdrud5ORk7rrrLtauXQtAZWUl9957L5deeinR0dHMnTvXqr355pvccsstpKenExERwYIFC9izZw/19fX92qyIiPy8HsMgPT2dkpIS6/mhQ4f46KOPGD58OGFhYVxxxRVWbeTIkdTW1gJHf/tPSEjoUmtqauLQoUPdana7ncsuu8zaV0REzq4hp7Lx999/T2FhIddeey1XXXUVNpuNsLAwq2632wkEAgD4fD7sdrtVi4yMBCAQCHSrHav7/f7TbuR0/djRSVOLDwB75BB+4bCd9TmIiIRar68m8nq9TJ8+nYsvvphly5bhdDoJBoMYY6xt/H4/DocD6BoMgPXY4XB0qx2rO53OM2rmdLS1d7Cnpok9NU34Az/2vIOIyCDUqzDYvXs306dPJysri2XLlhEREUFcXBzGGBobG63t6urqSExMBCAxMRGv19ulFhMTw7Bhw7rV/H4/33zzTZelIxEROXt6DIOGhgb+8Ic/MG/ePBYsWGAtCzmdTrKysigtLaW1tZXq6mrWrVtHbm4uALm5ubz00kt8/fXXtLS0sHz5cqZMmQJATk4OW7duZdeuXQSDQUpLS0lOTiY+Pr4fWxURkRPp8ZzBq6++SmtrK0uXLmXp0qXW+N13301JSQlPPPEEt9xyCzabjdmzZzNx4kQAZsyYwcGDB8nPzycQCHDbbbdRXFwMwOjRo1myZAmLFy/mu+++49prr+X555/vpxZFRKQnYeb4Rf8BorGxkUmTJrF161ZiY2NPef+mFh97apoAGB33S2q++j8Axo4ezvBoR5/OVUTkXHGy907djkJERBQGIiKiMBARERQGIiKCwkBERFAYiIgICgMREUFhICIiKAxERASFgYiIoDAQERFO8cttBjt90Y2InK8UBsdpa+/g0/3/Cxy9aZ3CQETOF1omEhERhYGIiCgMREQEhYGIiKAwEBERFAYiIoLCQEREUBiIiAgKAxER4RTD4NNPPyUjI8N6HgwGWbRoES6Xi/Hjx7Ny5cou21dUVJCZmUlqaioLFy7E5/NZtZ07dzJ58mRSUlLIz8+noaHhDFvpW8duTdHU4uN7XzDU0xER6Ve9CgNjDOvWreO+++6jvb3dGl++fDler5eqqirWr1+Px+OhsrISgG3btlFWVkZ5eTnbt28nGAzy5JNPAtDS0sIDDzzAnDlz2L17N1lZWRQUFNDZ2dkPLZ6etvYO9tQ0saemCX/gx1BPR0SkX/UqDJYtW8Zrr71GYWFhl3GPx8Ps2bOJiooiNjaWgoIC1qxZA0BlZSXTpk0jKSkJp9PJggUL2LhxI62trVRVVZGUlER2djZDhw5l1qxZBINBduzY0fcdiohIj3oVBvn5+bzxxhuMGTPGGjty5AjNzc0kJiZaY/Hx8dTW1gKwf/9+EhISrNqIESMwxlBfX9+t9tN9RUTk7OrVXUsvueSSbmPH1v8jIyOtMbvdTiAQsOp2u92qhYWFYbPZ8Pv93WrHfo7f7z/1Ds4C3dpaRAa7076F9bE387a2NmvM7/fjcDis+rFggKPnHYLBIE6ns1sNIBAIWPuea46/tfU1iRdb5xAUDCIyWJz2paVRUVHExMRQV1dnjXm9XmvZKDExEa/Xa9UaGhowxhAXF9etBlBXV9dlyelcpRPLIjIYndHnDHJzc3nhhRdoaWmhsbGR8vJycnNzAZgyZQrr1q2jurqa1tZWSktLcbvdOBwO3G431dXVbNq0ifb2dsrLywkPD8flcvVJUyIicmrOKAyKi4tJSkoiJyeHvLw8srOzmTFjBgATJ06ksLCQoqIiMjMzMcZQUlICwEUXXURZWRmrVq3C5XKxefNmysrKsNkG1pLL8Z9F+N9DPn0uQUQGrFM6ZzBu3Dg++ugj63lERASLFy9m8eLFP7v9zJkzmTlz5s/WrrvuOuszCQPV8ecSRsf9kpqv/g/QV2aKyMCj70DuB7r6SEQGGoVBPzj+iEFHCSIyEOhGdSIioiOD/qYlIxEZCBQG/exEH1gDCA+HY/fmU1CISCgpDM6i44MBdAWSiJw7FAbnCC0niUgoKQzOEbr/kYiEksLgHKRLU0XkbNOlpSIioiODc53OJYjI2aAwOMdpyUhEzgaFwQCiowQR6S8KgwHkRFcc6cNrInKmFAYDlG6fLSJ9SWEwyBy/lKQjBhHpLYXBIHOiI4YTLSspMEQEFAbnjROFRG8CQyEhMvgpDMRyqkcVCgmRwUNhID1SSIgMfgoDOW06PyEyeCgMpM+dyfmJnz5XgIicHSENg+rqahYvXkxNTQ2XX345Tz31FNdcc00opyRnyYkC46fPT/WIo68eK3jkfBOyMAgGgxQVFfGb3/yG1atXs2XLFgoKCnjnnXe48MILQzUtOYf15oijrx73VfAoVGSgCFkYfPjhh7S3t/Pb3/4WgNtvv53Vq1ezadMmpk+fHqppiQB9Fzx9FSpnuv9gfXw6+yigf17IwmD//v0kJCR0GRs5ciS1tbU97tvR0QHAt99+e1qvffBwgIPNR/+hfxvh52Dz4bP+OJSvfS4/Plfm0VePv4rwU9d49PHI2KjTfnym+w/Wx6ezT/IV0VxwwdGvcgkPg07DgHocGXEBzsihnI5j75nH3kOPF7Iw8Pl8REZGdhmz2+34/f4e921ubgZg5syZ/TI3EZHBrLm5mbi4uC5jIQsDh8NBW1tblzG/34/D4ehx3zFjxvDqq68SExPDBRdc0F9TFBEZVDo6OmhubmbMmDHdaiELg4SEBF555ZUuY3V1ddxxxx097hsZGUl6eno/zUxEZPD66RHBMSH7DuRx48ZhjOGVV16hvb2dt956i5qaGtxud6imJCJy3gozxphQvXhtbS2LFy+murqa2NhY/vSnP5GRkRGq6YiInLdCGgYiInJuCNkykYiInDsUBiIiojAQERGFgYiIcJ6FQXV1Nb/+9a9JSUlh8uTJfPrpp6GeUr94//33mTp1KmPHjsXtdrNmzRoAjhw5wrx580hLS2PChAls2LAhxDPte0eOHOGmm27ijTfesJ4P5p6bmpooKioiLS2N66+/nueeew44eiPIRYsW4XK5GD9+PCtXrgzxTPvOv/71L/Ly8khLS8PtdrNu3Tpg8Pb86aefdrnKsqc+KyoqyMzMJDU1lYULF+Lz+Xr3QuY80dbWZm6++Wbz8ssvm2AwaDZu3GjS09PN999/H+qp9an//ve/JjU11WzZssV0dHSYTz75xFx33XXmvffeM/PnzzfFxcXG5/OZL774wmRkZJhdu3aFesp96o9//KO58sorzYYNG4wxZtD3PG3aNPPYY4+ZQCBgGhoazMSJE82bb75p/vKXv5iZM2eaQ4cOmQMHDpjs7Gzj8XhCPd0z1tHRYTIyMkxlZaUxxphPPvnEjBkzxnz55ZeDrufOzk7z+uuvm7S0NJOWlmaNn6zP9957z1x//fWmtrbW/PDDD2bu3Lnm4Ycf7tXrnTdHBsffJXXo0KHcfvvtJCYmsmnTplBPrU99/fXX5OTk4Ha7CQ8P55prrsHlcrFjxw42b95McXExdrud5ORk7rrrLtauXRvqKfcZj8fDDz/8wKhRo4CjtzcZzD1/8sknHDhwgEcffZSIiAguv/xyKioqGDduHB6Ph9mzZxMVFUVsbCwFBQXWEeJAdvjwYQ4ePIgxBmMMYWFhDBkyhKFDhw66npctW8Zrr71GYWFhl/GT9VlZWcm0adNISkrC6XSyYMECNm7cSGtra4+vd96EwZncJXUgSU9Pp6SkxHp+6NAhPvroI4YPH05YWBhXXHGFVRtM/R84cIAVK1awZMkSa6y+vn5Q97xv3z5GjRrFihUrmDBhAllZWVRVVREZGUlzczOJiYnWtvHx8YOi71/+8pfcc889PPzww1x11VXk5eUxf/58YmJiBl3P+fn5vPHGG13uI3TkyJGT9vnT97kRI0ZgjKG+vr7H1ztvvvbyTO6SOlB9//33FBYWcu2113LVVVdhs9kICwuz6na7nUAgEMIZ9o2Ojg4efPBBHnroIWJiYqzx1tbWQdszHP0t+eOPP8blcrF161bq6uqYNWsW0dHRAF3+vg+Wvjs7O7HZbJSWlnLrrbeyd+9e5s6dy7Bhw4DB1fMll1zSbezY+v+J+vT5fNjtdqsWFhaGzWbr1fvceXNkcCZ3SR2IvF4v06dP5+KLL2bZsmU4nU6CwSDmuA+cD5b+X3zxReLj47n11lu7jDscjkHbM4DNZuPCCy9k7ty52Gw2rrzySvLy8vB4PABd/r4Plr63bNnC3r17uf322xk6dCgul4tp06YN6p6Pd+yN/kR9/jQAjTEEg0GcTmePP/u8CYOEhAS8Xm+Xsbq6ui6HW4PF7t27mT59OllZWSxbtoyIiAji4uIwxtDY2GhtN1j6f+utt9i8eTPp6emkp6dTW1vLE088QUVFxaDtGY4uefn9foLBoDXW0dFBVFQUMTEx1NXVWeNer3dQ9P3tt9926RdgyJAhREdHD9qej9fT/9vExMQu73MNDQ0YY054p9Iu+vLs97msra3NZGZmdrmaKDU11Rw8eDDUU+tTX331lUlNTTV///vfu9XmzZtniouLzQ8//GC+/PJLk5GRYd59990QzLJ/5ebmWlcTDeaeA4GAyczMNCUlJaatrc1UV1eb8ePHm7fffts888wzZubMmebgwYPWFSevvvpqqKd8xmpra82YMWPMmjVrTGdnp/nss8/MuHHjTFVV1aDteefOnV2uJjpZn++++6654YYbzJdffmldTVRcXNyr1zlvwsAYY2pqakx+fr5JSUkxOTk55oMPPgj1lPrckiVLzKhRo0xKSkqXP3/+85/NoUOHzPz5843L5TI33nij+dvf/hbq6faL48NgsPfc0NBg7r//fqu/v/71r8aYo0Hx+OOPm4yMDDNu3DizdOlS09nZGeLZ9o13333X3HnnnWbs2LHm1ltvNa+//roxZvD2/NMw6KnP1atXm5tvvtmMHTvWzJkzxxw+fLhXr6O7loqIyPlzzkBERE5MYSAiIgoDERFRGIiICAoDERFBYSAiIigMREQEhYGIiKAwEBER4P8BMFIQFlQMTzYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot([x for x in partition.membership if x<100], norm_hist=False, kde=False, bins=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "c = Counter(partition.membership)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 229,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1601"
      ]
     },
     "execution_count": 229,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 6434),\n",
       " (1, 1995),\n",
       " (2, 1475),\n",
       " (3, 1127),\n",
       " (4, 999),\n",
       " (5, 987),\n",
       " (6, 879),\n",
       " (7, 877),\n",
       " (8, 807),\n",
       " (9, 714)]"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c.most_common(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [],
   "source": [
    "which_cluster = 0\n",
    "pubs_zero = [\"pub.\"+str(inverse_node_map[n]) for n,x in enumerate(partition.membership) if x==which_cluster]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1bab321d90>"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEBCAYAAAB7Wx7VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAYvUlEQVR4nO3de3BU5eH/8U8CLNmk3zBSgjgFERJQG6bmZiRcHUkEBJOWW6DIpE6w5SJJBVurhSpJiwoTQC4dHMXaAi0XbdKpOGjK9CLgBe9laog0oSECTUwMl+xuNiTP7w9/bIlAdonZJPC8XzPOkPOcsznPcXlnOXtyNsQYYwQAuOaFdvYOAAA6BsEHAEsQfACwBMEHAEsQfACwBMEHAEsEFPwPP/xQ06ZNU2JiotLS0rRr1y5Jktfr1bJly5ScnKzhw4fr2WefbbHdli1bNGbMGMXHx+vhhx+Wy+Vq/xkAAALiN/jNzc1asGCB5syZo/fee08FBQXKy8tTSUmJ1q9fr/LychUXF+ull15SYWGhioqKJElvvPGGNm3apM2bN2vfvn3yer3Kz88P+oQAAJfmN/inTp1STU2NjDEyxigkJETdu3dXjx49VFhYqHnz5qlXr17q37+/srOztX37dklSUVGRpk6dqiFDhigiIkJLlizRK6+8ovr6+qBPCgBwse7+Vrjuuut033336Wc/+5kee+wxNTU16ec//7mioqJUXV2tmJgY37qDBg1SaWmpJOnIkSMaM2aMb+zGG2+UMUZHjx5VbGys3x3zeDw6dOiQoqKi1K1bt7bMDQCs0tTUpOrqag0bNkxhYWEXjfsNfnNzsxwOhwoKCnT33Xfrgw8+0KJFixQZGSlJLR7U6XTK4/FIklwul5xOp28sJCREDodDbrc7oB0/dOiQZs+eHdC6AID/2bZtm5KSki5a7jf4r7/+uj744AM98sgjkqTk5GRNnTpVhYWFkqSGhgbfum63W+Hh4ZJaxl+SjDHyer2KiIgIaIejoqJ8O96vX7+AtgEAm508eVKzZ8/29fOr/Ab/5MmT8nq9LTfq3l29e/dWVFSUysrKdP3110uSysvLfad4YmJiVF5e7tumoqJCxhgNHDgwoB0/fxqnX79+6t+/f0DbAAB02dPgft+0HTlypD799FPt2LFDxhgdOnRIO3fu1KRJk5Senq6NGzeqtrZWlZWV2rx5s9LT0yVJGRkZ2rVrl0pKSlRfX6+CggKlpaX5/gUAAOhYfl/hDxkyRBs2bNAzzzyjlStXqk+fPlqyZIlSU1M1evRoPfXUU5o8ebKam5uVmZmpWbNmSZLGjh2r+fPna8GCBTp16pRGjBihvLy8oE8IAHBpIV31fviVlZUaN26c9u7dyykdAAiAv25yawUAsATBBwBLEHwAsATBBwBL+L1KBwDQfs64vHJ7zrW6jjOsu/4v3NHu35vgA0AHcnvO6f3DVa2uk3Bz36AEn1M6AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8Alggo+FVVVVqwYIESExM1YsQIrV27VpLk9Xq1bNkyJScna/jw4Xr22WdbbLdlyxaNGTNG8fHxevjhh+Vyudp/BgCAgAQU/AULFigqKkoHDhzQjh07VFRUpD//+c9av369ysvLVVxcrJdeekmFhYUqKiqSJL3xxhvatGmTNm/erH379snr9So/Pz+okwEAXJ7f4H/00Uc6duyYli5dqp49e2rAgAHasmWL7rjjDhUWFmrevHnq1auX+vfvr+zsbG3fvl2SVFRUpKlTp2rIkCGKiIjQkiVL9Morr6i+vj7okwIAXMxv8A8dOqShQ4dqw4YNGj16tFJTU1VcXKywsDBVV1crJibGt+6gQYNUWloqSTpy5Iiio6N9YzfeeKOMMTp69Gj7zwIA4Fd3fyucOnVK7733npKTk7V3716VlZVp7ty56t27tyQpLCzMt67T6ZTH45EkuVwuOZ1O31hISIgcDofcbnd7zwEAEAC/wXc4HPrGN76hRYsWSZJuueUWTZs2TYWFhZKkhoYG37put1vh4eGSWsZfkowx8nq9ioiIaNcJAAAC4/eUzuDBg+V2u+X1en3Lmpqa1KtXL0VFRamsrMy3vLy83HeKJyYmRuXl5b6xiooKGWM0cODA9tx/AECA/AZ/5MiR6t27t55++ml5vV4dPnxYL730kiZNmqT09HRt3LhRtbW1qqys1ObNm5Weni5JysjI0K5du1RSUqL6+noVFBQoLS3N9y8AAEDH8ntKp2fPntq6davy8/M1evRoORwOzZ07V+PHj9edd96pp556SpMnT1Zzc7MyMzM1a9YsSdLYsWM1f/58LViwQKdOndKIESOUl5cX9AkBAC4txBhjOnsnLqWyslLjxo3T3r171b9//87eHQBoF1W1Lr1/uKrVdRJu7qu+va/8bIi/bnJrBQCwBMEHAEsQfACwBMEHAEsQfACwBMEHAEsQfACwBMEHAEsQfACwBMEHAEsQfACwBMEHAEsQfACwBMEHAEsQfACwhN8PQAHQPs64vHJ7zrW6jjOsu/4v3NFBewTbEHygg7g95wL64AuCj2DhlA4AWILgA4AlCD4AWILgA4AlCD4AWILgA4AlCD4AWILgA4AlCD4AWILgA4AlCD4AWILgA4AlCD4AWILgA4AlCD4AWILgA4AlCD4AWILgA4Al+IhDoAs519SsqlrXZcf5zNtL4/OCA0PwgS6kobFJHx/5/LLjfObtpfF5wYHhlA4AWIJX+AAgO04LEXwAkB2nhTilAwCWIPgAYAmCDwCWIPgAYAnetAVgBX+/1NbQ2NSBe9M5Ag7+6dOnlZ6erpycHE2ZMkWnT5/W0qVLtX//foWHh+vHP/6xpk6dKkkyxmjt2rXauXOnGhsbNWXKFP30pz9V9+78fAHQOfz9UtvNA6/z+xj+fmhIXfvSzYAL/Pjjj+u///2v7+snnnhCoaGh2rdvn44ePars7GwNGDBAycnJ2rFjh4qLi1VYWCiHw6GFCxdq06ZNevDBB4MyCQDoCP5+aEhd+9LNgM7hFxYW6uzZsxo6dKgkye1267XXXlNubq6cTqduvfVWTZ8+XTt27JAkFRUVKSsrS/369VPv3r21aNEi3xgAoHP4Df6xY8e0YcMGrVixwrfs6NGjCgkJ0U033eRbNnjwYJWWlkqSjhw5oujo6BZjVVVVqqura8ddBwBciVaD39TUpJ/85Cd65JFHFBUV5VteX18vh8OhkJAQ3zKn0ymPxyNJcrlccjqdvrGwsDBJ8o0DADpeq8H/9a9/rUGDBunuu+9usTw8PFxer1fGGN8yt9ut8PBwSS3jL/0v9OfHAQAdr9U3bXfv3q2qqioVFxdL+vKV/fLly3XPPffIGKPKykoNGDBAklRWVqaYmBhJUkxMjMrLy5WYmOgbi4qKUmRkZDDnAgBoRavB37NnT4uvMzIylJWVpSlTpsjlcqmgoEC/+tWvdOzYMe3atUtPPvmkJCk9PV0vvPCCUlJS5HQ6tX79emVkZARvFgAAv9p8YXxeXp6WL1+uu+66Sw6HQ/PmzdPYsWMlSbNmzVJNTY1mzpwpj8ejCRMmKDc3t912GgBw5a4o+H/60598f+7Vq5dWr159yfVCQ0OVk5OjnJycr7d3AIB2w710AMASBB8ALEHwAcASBB8ALEHwAcAS3K8YQJd2xuWV23Ou1XW60r3su/J99wk+gC7N7Tmn9w9XtbpOIPey7yjtcd/9YOGUDgBYglf4wFXkav/EJXQugg9cRa72T1xC5+KUDgBYguADgCUIPgBYguADgCUIPgBYgqt0gGsMl27icgg+cI3h0k1cDqd0AMASBB8ALEHwAcASBB8ALMGbtoCF/F3Jw1U81yaCD1jI35U8XMVzbeKUDgBYguADgCUIPgBYguADgCUIPgBYguADgCUIPgBYguADgCX4xSsAF+Ge+tcmgg/gItxT/9rEKR0AsASv8AF0qjMur9yec5cdb2hs6sC9ubYRfACdyu05p/cPV112/OaB13Xg3lzbCD6AoPH36l3iFXxHIvgAgsbfq3eJV/AdieADaJNALt3k1XvXQvABtEkgl27y6r1r4bJMALAEr/CBdsCbk7gaEHygHfDmJK4GnNIBAEsEFPz9+/drypQpSkhIUFpamrZv3y5JOn36tHJycpSYmKjRo0fr5Zdf9m1jjNGaNWuUkpKipKQkrVixQufOtf5PXgBA8Pg9pXPixAktWrRITz/9tMaNG6dDhw5p7ty5+ta3vqXCwkKFhoZq3759Onr0qLKzszVgwAAlJydrx44dKi4uVmFhoRwOhxYuXKhNmzbpwQcf7Ih5AQC+wu8r/M8++0yTJ09WWlqaQkND9Z3vfEfJycl688039dprryk3N1dOp1O33nqrpk+frh07dkiSioqKlJWVpX79+ql3795atGiRbwwA0PH8Bj8pKUl5eXm+r+vq6vTuu++qb9++CgkJ0U033eQbGzx4sEpLSyVJR44cUXR0dIuxqqoq1dXVtePuAwACdUVv2p45c0bz58/XbbfdptjYWDkcDoWEhPjGnU6nPB6PJMnlcsnpdPrGwsLCJMk3DgDoWAEHv7y8XDNmzFCfPn20bt06RUREyOv1yhjjW8ftdis8PFxSy/hL/wv9+XEAQMcKKPgHDx7UjBkzlJqaqnXr1qlnz54aOHCgjDGqrKz0rVdWVqaYmBhJUkxMjMrLy1uMRUVFKTIysp2nAAAIhN/gV1RU6Ec/+pFycnK0ZMkS3ymciIgIpaamqqCgQPX19SopKdGuXbuUnp4uSUpPT9cLL7ygzz77TLW1tVq/fr0yMjKCOxsAwGX5vSxz27Ztqq+v1+rVq7V69Wrf8u9///vKy8vT8uXLddddd8nhcGjevHkaO3asJGnWrFmqqanRzJkz5fF4NGHCBOXm5gZvJgCAVvkN/qOPPqpHH330suMX/hC4UGhoqHJycpSTk9P2vQMAtBturQAAliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGAJgg8AliD4AGCJ7p29A8DV4IzLK7fn3GXHGxqbOnBvgLYh+EAA3J5zev9w1WXHbx54XQfuDdA2nNIBAEsQfACwBMEHAEsQfACwBMEHAEsQfACwRFCDX1JSoszMTMXFxenee+/Vxx9/HMxvBwBoRdCC7/V6tWDBAk2cOFEHDx7UvHnzlJ2drbNnzwbrWwIAWhG04L/zzjtqbGzUD37wA/Xo0UOTJk1STEyMXn311WB9SwBAK4L2m7ZHjhxRdHR0i2WDBw9WaWlpQNs3NX35q+onT55s933D1aHe0yhPQ+u3LAgNkZpN28cDXcfb2KSa6i8uO36yp1s11adafYz2WOda+z5daV+60vc5EXlOXldYq+tc8rH/fy/P9/OrghZ8l8ulsLCWO+x0OuV2uwPavrq6WpI0e/bsdt83ALiWVVdXa+DAgRctD1rww8PD1dDQ0GKZ2+1WeHh4QNsPGzZM27ZtU1RUlLp16xaMXQSAa0pTU5Oqq6s1bNiwS44HLfjR0dF68cUXWywrKyvTd7/73YC2DwsLU1JSUhD2DACuXZd6ZX9e0N60veOOO2SM0YsvvqjGxkbt3r1bhw8fVlpaWrC+JQCgFSHGGD9vV7VdaWmpHn/8cZWUlKh///567LHHlJKSEqxvBwBoRVCDDwDoOri1AgBYguADgCUIPgBYguADgCUIfhfz8ccft7iSqba2VkuWLNEdd9yhUaNGac2aNS1+bXrLli0aN26cEhMTNXfuXB07dsw3dvz4cd1///2Kj49Xamqq/v73v3foXDrD/v37NWXKFCUkJCgtLU3bt2+XJJ0+fVo5OTlKTEzU6NGj9fLLL/u2McZozZo1SklJUVJSklasWKFz5875xl999VWlpaUpLi5ODzzwgGpqajp8Xp2hLcfyvObmZi1cuPCi38V56623dO+99youLk4zZ85URUVFR0ylU7XlOLrdbv3iF7/QqFGjlJycrPnz5+v48eO+8TY/Jw26hObmZrNz506TmJhoEhMTfcvnzp1rsrKyzOeff26++OILc//995vVq1cbY4zZvXu3iY+PN2+//bZpbGw027ZtM+PGjTMej8cYY0xmZqZ58sknTUNDgzlw4ICJj483FRUVnTK/jnD8+HETHx9vXn/9ddPU1GQ++ugjc/vtt5t//OMf5qGHHjK5ubnG5XKZf/3rXyYlJcW8/fbbxhhj/vCHP5iJEyeaEydOmJqaGjNz5kyzfv16Y4wxn376qYmLizMHDx40Ho/H5Ofnmzlz5nTmNDtEW4+lMcYcO3bMPPDAA2bo0KHmN7/5jW95TU2NSUhIMHv27DFer9c899xzJjU11TQ1NXXCDDtGW49jfn6+ycrKMrW1tcbtdpulS5eazMxMY8zXe04S/C5i7dq15nvf+555/vnnfcF3uVzm5ptvNiUlJb71PvzwQzN8+HDT3NxscnNzzZNPPtnicSZMmGD++te/mrKyMhMbG2vq6+t9Y0uWLPH9sLgWHTx40CxbtqzFsoULF5qnn37afPvb3zZlZWW+5atXrzaLFy82xnz5g3H79u2+sf3795tRo0YZY4wpKCjwrWfMl/9PYmNjTXl5eRBn0vnaeizPnj1rkpOTTUFBgcnMzGwR/O3bt/uidd6YMWPMvn37gjeRTtbW47h06VJz8OBB39gnn3xibrnlFtPY2Pi1npOc0ukiZs6cqT/+8Y8t7oHR3NwsSS3uPxQaGqra2lqdPn1azc3NcjqdLR6nW7duOnr0qP7973/rhhtuaLHt4MGDdfjw4SDPpPMkJSUpLy/P93VdXZ3effdd9e3bVyEhIbrpppt8YxfeufWrd3YdPHiwqqqqVFdXd9GY0+nUDTfcEPBdX69WbT2WPXv21KuvvqrFixerR48eLR7zUnfQHTRo0DV9LNt6HPPz81vcWuYvf/mLhgwZou7du3+t5yTB7yKuv/76i5ZFRERo5MiRWrVqlerq6vTFF19o48aNkiSPx6Px48dr586d+uc//6nGxkbt3LlTZWVl8ng8qq+vv+TdSj0eT4fMp7OdOXNG8+fP12233abY2Fg5HA6FhIT4xi88Fi6Xq8UPzvPHzePxXDR2fjzQu75eC67kWHbv3l3f/OY3L/k4th/LKzmOF9q9e7c2b96sZcuWSfp6x5Hgd3GrVq1Sjx49dM8992jOnDlKTU2VJEVGRmrSpEn64Q9/qMWLF+vOO+9UaWmpRowYocjIyK99t9KrWXl5uWbMmKE+ffpo3bp1ioiIkNfrlbngl8ovPBZf/Yt2/s/h4eGX/Evo8XgUERHRATPpfFd6LFtzuWPJc/JLXz2Oxhht2LBBTzzxhDZu3Kjbb79d0uWPYyDPSYLfxVVVVWn58uU6cOCAXnnlFV1//fWKjo6W0+lUVVWV7rrrLhUXF2v//v169NFH9emnn2rYsGGKjo7W8ePHWzwxysrKFBMT04mzCb6DBw9qxowZSk1N1bp169SzZ08NHDhQxhhVVlb61rvwWMTExKi8vLzFWFRUlCIjIy8ac7vdOnHixEWnJq5FbTmWrfnqsbySba9mbTmOjY2NWrx4sYqKivT73/9eI0aM8K33tZ6TbX0zAsHx1ltvtbhKJysry/zyl780jY2NpqKiwqSnp5utW7caY4wpKioyaWlppqqqyrhcLrNq1Sozbdo009zcbIwxZurUqWbFihWmoaHBvPnmmyYuLs4cPny4U+bVEf7zn/+Y+Ph487vf/e6isZycHJObm2vOnj1rPvnkE5OSkmL+9re/GWOM2bp1q5k4caKprKz0XaWzcuVKY4wxJSUlJiEhwbz11lumoaHB5Ofnm2nTpnXovDpDW4/lhe67774Wb9p+/vnnJiEhwezevdt4vV7z/PPPm7Fjx5qGhoZgTqVTtfU4PvHEE2bChAmmpqbmou2+znOS4HcxXw1+RUWFycrKMvHx8Wbs2LHmueee8401NzeblStXmpSUFJOUlGQWLVpkqqurfePHjx832dnZJiEhwYwbN87s3r27Q+fS0VasWGGGDh1q4uLiWvy3cuVKU1dXZx566CGTnJxsRo0aZX7729/6tmtqajLPPPOMGTVqlElKSjJLly5tEaE9e/aY8ePHm7i4OJOVlWU+++yzzpheh2rrsbzQV4NvjDHvvPOOycjIMHFxcWb69Onmk08+6YDZdJ62HMdTp06ZW265xcTGxl603enTp40xbX9OcrdMALAE5/ABwBIEHwAsQfABwBIEHwAsQfABwBIEHwAsQfABwBIEHwAsQfABwBL/D05Kb8eFR0ntAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot([int(metadata_dict[p][\"year\"]) for p in pubs_zero], kde=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "clusters = dict()\n",
    "\n",
    "for m in range(len(c)):\n",
    "    clusters[m] = [metadata_dict[\"pub.\"+str(inverse_node_map[n])][\"doi\"] for n,x in enumerate(partition.membership) if x==m]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(clusters,open(\"datasets_output/networks/clusters_doi.pk\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create network of clusters\n",
    "from collections import defaultdict\n",
    "\n",
    "cluster_size_limit = 99\n",
    "\n",
    "node_to_cluster_map = {n:cl for n,cl in enumerate(partition.membership) if c[cl]>cluster_size_limit} #focus only on clusters with 100+ papers\n",
    "\n",
    "edges_clusters_int = defaultdict(float)\n",
    "edges_clusters_count = defaultdict(float) # this is used to average\n",
    "\n",
    "for e,w in zip(edges_int,weights):\n",
    "    if e[0] in node_to_cluster_map.keys() and e[1] in node_to_cluster_map.keys():\n",
    "        if node_to_cluster_map[e[0]] != node_to_cluster_map[e[1]]: #only consider weights across clusters\n",
    "            edges_clusters_int[tuple(sorted([node_to_cluster_map[e[0]],node_to_cluster_map[e[1]]]))] += w\n",
    "            edges_clusters_count[tuple(sorted([node_to_cluster_map[e[0]],node_to_cluster_map[e[1]]]))] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38290"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(node_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26510"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(node_to_cluster_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1072"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(edges_clusters_int.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export edges and nodes for the cluster network\n",
    "\n",
    "with open(\"datasets_output/networks/cluster_bibc_nodes.csv\",\"w\") as f:\n",
    "    f.write(\"Id,Size\\n\")\n",
    "    for nc,s in c.items():\n",
    "        if s>cluster_size_limit:\n",
    "            f.write(\",\".join([str(nc),str(s)])+\"\\n\")\n",
    "        \n",
    "with open(\"datasets_output/networks/cluster_bibc_edges.csv\",\"w\") as f:\n",
    "    f.write(\"Source,Target,Weight\\n\")\n",
    "    for nc,w in edges_clusters_int.items():\n",
    "        f.write(\",\".join([str(nc[0]),str(nc[1]),str(w/edges_clusters_count[nc])])+\"\\n\") # relative edge weights\n",
    "        #f.write(\",\".join([str(nc[0]),str(nc[1]),str(w)])+\"\\n\") # absolute edge weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}