4. BioPortal (distance-matrices-lcc).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "# !{sys.executable} -m pip install tldextract"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style='text-align:center;'>\n",
    "<h1>BioPortal</h1>\n",
    "<h2>distance (khops) matrices</h2>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "__author__ = \"Lisette Espin-Noboa\"\n",
    "__copyright__ = \"Copyright 2018, HopRank\"\n",
    "__credits__ = [\"Florian Lemmerich\", \"Markus Strohmaier\", \"Simon Walk\", \"Mark Musen\"]\n",
    "__license__ = \"GPL\"\n",
    "__version__ = \"1.0.3\"\n",
    "__maintainer__ = \"Lisette Espin-Noboa\"\n",
    "__email__ = \"Lisette.Espin@gesis.org\"\n",
    "__status__ = \"Developing\"\n",
    "\n",
    "########################################################################################\n",
    "# Warnings\n",
    "########################################################################################\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\", message=\"numpy.dtype size changed\")\n",
    "warnings.filterwarnings(\"ignore\", message=\"numpy.ufunc size changed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Dependencies</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################################################################################\n",
    "# System\n",
    "########################################################################################\n",
    "import os\n",
    "import importlib\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import networkx as nx\n",
    "from collections import Counter\n",
    "from joblib import Parallel, delayed\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt \n",
    "from collections import defaultdict\n",
    "\n",
    "########################################################################################\n",
    "# Local Dependencies\n",
    "########################################################################################\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "from org.gesis.libs.bioportal.ontology import Ontology\n",
    "from org.gesis.libs.bioportal.transition import Transition\n",
    "from org.gesis.libs.bioportal.clickstream import NAVITYPES\n",
    "from org.gesis.libs.bioportal.clickstream import load_clickstream\n",
    "from org.gesis.libs.bioportal.clickstream import preprocess_clickstream\n",
    "from org.gesis.libs.bioportal.clickstream import preprocess_sessions\n",
    "from org.gesis.libs.bioportal.clickstream import clean_clickstream\n",
    "from org.gesis.libs.bioportal.submission import get_submissions\n",
    "from org.gesis.libs.utils import log\n",
    "from org.gesis.libs.utils import weighted_to_undirected\n",
    "from org.gesis.libs.utils import to_symmetric\n",
    "\n",
    "########################################################################################\n",
    "# Plot's style\n",
    "########################################################################################\n",
    "sns.set_context(context=\"paper\", font_scale=1.5)\n",
    "sns.set_style(\"whitegrid\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Constants</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "SM = 'submission'\n",
    "SUBMISSIONS_FN = '/bigdata/lespin/datasets/bioportal/submissions.json'\n",
    "\n",
    "ON = 'ontology'\n",
    "ONTOLOGY_ROOT = '/bigdata/lespin/datasets/bioportal/ontologies'\n",
    "ONTOLOGY_GRAPH_OUTPUT = '/bigdata/lespin/bioportal/ontologies/graph'\n",
    "ONTOLOGY_ADJACENCY_OUTPUT = '/bigdata/lespin/bioportal/ontologies/matrix'\n",
    "ONTOLOGY_NODES_OUTPUT = '/bigdata/lespin/bioportal/ontologies/nodes'\n",
    "ONTOLOGY_HOPS_OUTPUT = '/bigdata/lespin/bioportal/ontologies/hops'\n",
    "ONTOLOGY_DISTANCE_OUTPUT = '/bigdata/lespin/bioportal/ontologies/distances'\n",
    "\n",
    "CS = 'clickstream'\n",
    "CS_FN = 'BP_webpage_requests_<YEAR><POSTFIX>.csv.bz2'\n",
    "CLICKSTREAM_ROOT = '/bigdata/lespin/datasets/bioportal/clickstream/'\n",
    "CLICKSTREAM_DF = '/bigdata/lespin/bioportal/clickstreams'\n",
    "\n",
    "TR = 'transitions'\n",
    "TRANS_GRAPH_OUTPUT = '/bigdata/lespin/bioportal/transitions/graph'\n",
    "TRANS_ADJACENCY_OUTPUT = '/bigdata/lespin/bioportal/transitions/matrix'\n",
    "TRANS_NODES_OUTPUT = '/bigdata/lespin/bioportal/transitions/nodes'\n",
    "LCC_TRANS_GRAPH_OUTPUT = '/bigdata/lespin/bioportal/transitions_in_lcc/graph'\n",
    "LCC_TRANS_ADJACENCY_OUTPUT = '/bigdata/lespin/bioportal/transitions_in_lcc/matrix'\n",
    "LCC_TRANS_NODES_OUTPUT = '/bigdata/lespin/bioportal/transitions_in_lcc/nodes'\n",
    "\n",
    "SU = 'summary'\n",
    "LOG_OUTPUT = '/bigdata/lespin/bioportal/logs'\n",
    "PLOTS_OUTPUT = '/bigdata/lespin/bioportal/plots'\n",
    "SUMMARY_OUTPUT = '/bigdata/lespin/bioportal/summary'\n",
    "YEAR = '2015'\n",
    "\n",
    "LAG_BREAK_IN_MINUTES = 60\n",
    "MIN_SESSION_LENGTH = 2\n",
    "MIN_REQ = 1000\n",
    "MIN_TRANSITIONS = 1000\n",
    "MAXKHOPS = 30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<div style='text-align:center;'>\n",
    "<h1>Distance matrices</h1>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading submissions...\n",
      "2019-02-15 19:47:43\t/bigdata/lespin/datasets/bioportal/submissions.json loaded!\n",
      "2019-02-15 19:47:43\t- 642 ontologies\n",
      "2019-02-15 19:47:43\t- 13 years\n",
      "loading ontologies...\n",
      "generating distance matrices for 12 ontologies...\n",
      "LCC_LOINC_2015_9HOP.npz\n",
      "LCC_LOINC_2015_1HOP.npz\n",
      "LCC_LOINC_2015_4HOP.npz\n",
      "LCC_LOINC_2015_5HOP.npz\n",
      "LCC_LOINC_2015_2HOP.npz\n",
      "LCC_LOINC_2015_7HOP.npz\n",
      "LCC_LOINC_2015_3HOP.npz\n",
      "LCC_LOINC_2015_8HOP.npz\n",
      "LCC_LOINC_2015_12HOP.npz\n",
      "LCC_LOINC_2015_10HOP.npz\n",
      "LCC_LOINC_2015_11HOP.npz\n",
      "LCC_LOINC_2015_6HOP.npz\n",
      "LCC_LOINC_2015_13HOP.npz\n",
      "file:LCC_LOINC_2015_9HOP.npz, k:9\n",
      "2019-02-15 19:47:44\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_9HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_1HOP.npz, k:1\n",
      "2019-02-15 19:47:45\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_1HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_4HOP.npz, k:4\n",
      "2019-02-15 19:49:46\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_4HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_5HOP.npz, k:5\n",
      "2019-02-15 19:51:53\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_5HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_2HOP.npz, k:2\n",
      "2019-02-15 19:53:42\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_2HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_7HOP.npz, k:7\n",
      "2019-02-15 19:56:32\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_7HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_3HOP.npz, k:3\n",
      "2019-02-15 19:59:38\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_3HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_8HOP.npz, k:8\n",
      "2019-02-15 20:03:05\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_8HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_12HOP.npz, k:12\n",
      "2019-02-15 20:06:05\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_12HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_10HOP.npz, k:10\n",
      "2019-02-15 20:09:03\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_10HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_11HOP.npz, k:11\n",
      "2019-02-15 20:11:42\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_11HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_6HOP.npz, k:6\n",
      "2019-02-15 20:17:42\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_6HOP.npz loaded!\n",
      "file:LCC_LOINC_2015_13HOP.npz, k:13\n",
      "2019-02-15 20:26:11\t/bigdata/lespin/bioportal/ontologies/hops/LCC_LOINC_2015_13HOP.npz loaded!\n",
      "2019-02-15 22:56:05\t/bigdata/lespin/bioportal/ontologies/distances/LCC_LOINC_2015_HOPs.npz saved!\n",
      "max khop (max-diameter): 13\n"
     ]
    }
   ],
   "source": [
    "print('loading submissions...')\n",
    "submissions_dict = get_submissions(SUBMISSIONS_FN)\n",
    "\n",
    "print('loading ontologies...')\n",
    "fn = os.path.join(SUMMARY_OUTPUT,'lcc_summary_ontologies_transitions_{}.{}'.format(YEAR,'csv'))\n",
    "df_onto = pd.read_csv(fn, index_col=False)\n",
    "tmp = df_onto.loc[df_onto[\"T'\"] >= MIN_TRANSITIONS].copy()\n",
    "del(df_onto)\n",
    "\n",
    "print('generating distance matrices for {} ontologies...'.format(tmp.ontology.nunique()))\n",
    "for index,row in tmp.sort_values(\"E'\", ascending=True).iterrows():\n",
    "    onto_name = row['ontology']\n",
    "    \n",
    "    if onto_name not in ['LOINC']:\n",
    "        continue\n",
    "\n",
    "    submission = submissions_dict[onto_name][YEAR]\n",
    "    onto = Ontology(name=onto_name, year=YEAR, submission_id=submission['submissionId'], root_folder=ONTOLOGY_ROOT)\n",
    "    m = onto.create_distance_matrix(path=ONTOLOGY_DISTANCE_OUTPUT, hopspath=ONTOLOGY_HOPS_OUTPUT, lcc=True)\n",
    "    print(\"max khop (max-diameter): {}\".format(m.max()))    \n",
    "    del(m)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "del(onto)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}