notebooks/dataprep/00b-PANGOLineage.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Downloads PANGO Lineages\n",
    "**[Work in progress]**\n",
    "\n",
    "This notebook downloads the current PANGO lineages and build a tree structure of the lineages.\n",
    "\n",
    "Data sources: [PANGO Lineage Designations](https://github.com/cov-lineages/pango-designation)\n",
    "\n",
    "Reference:\n",
    "Rambaut A, et al., A dynamic nomenclature proposal for SARS-CoV-2 lineages to assist genomic epidemiology(2020) Nature Microbiology [doi:10.1038/s41564-020-0770-5](https://doi.org/10.1038/s41564-020-0770-5).\n",
    "\n",
    "Author: Peter Rose (pwrose@ucsd.edu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import io\n",
    "import dateutil\n",
    "import re\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.options.display.max_rows = None  # display all rows\n",
    "pd.options.display.max_columns = None  # display all columsns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/peter/Library/Application Support/com.Neo4j.Relate/data/dbmss/dbms-8bf637fc-0d20-4d9f-9c6f-f7e72e92a4da/import\n"
     ]
    }
   ],
   "source": [
    "NEO4J_IMPORT = Path(os.getenv('NEO4J_IMPORT'))\n",
    "print(NEO4J_IMPORT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get PANGO lineages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "pango_url = 'https://raw.githubusercontent.com/cov-lineages/pango-designation/master/lineage_notes.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pango = pd.read_csv(pango_url, sep='\\t', skiprows=1, dtype=str, names=['lineage', 'description'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove spaces in lineage column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pango['lineage'] = pango['lineage'].str.strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove withdrawn lineages (start with a \"*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pango = pango[~pango['lineage'].str.startswith('*')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lineage</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>B.1.1.343</td>\n",
       "      <td>Canadian lineage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>439</th>\n",
       "      <td>B.1.12</td>\n",
       "      <td>Luxembourg lineage (also England, Belgium etc.)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>B.1.1.402</td>\n",
       "      <td>USA (Texas), split from B.1.1.177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>962</th>\n",
       "      <td>B.1.506</td>\n",
       "      <td>USA (NY, split from B.1.3)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>B.1.1.37</td>\n",
       "      <td>UK lineage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>418</th>\n",
       "      <td>B.1.1.483</td>\n",
       "      <td>Belgian lineage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>B.1.1.294</td>\n",
       "      <td>Russian and UK lineage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1210</th>\n",
       "      <td>M.3</td>\n",
       "      <td>Alias of B.1.1.294.3, English Lineage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>961</th>\n",
       "      <td>B.1.505</td>\n",
       "      <td>Israel and england (was B.1.3.4)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>867</th>\n",
       "      <td>B.1.406</td>\n",
       "      <td>Germany/ UK lineage</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        lineage                                      description\n",
       "295   B.1.1.343                                 Canadian lineage\n",
       "439      B.1.12  Luxembourg lineage (also England, Belgium etc.)\n",
       "353   B.1.1.402                USA (Texas), split from B.1.1.177\n",
       "962     B.1.506                       USA (NY, split from B.1.3)\n",
       "56     B.1.1.37                                       UK lineage\n",
       "418   B.1.1.483                                  Belgian lineage\n",
       "250   B.1.1.294                           Russian and UK lineage\n",
       "1210        M.3            Alias of B.1.1.294.3, English Lineage\n",
       "961     B.1.505                 Israel and england (was B.1.3.4)\n",
       "867     B.1.406                              Germany/ UK lineage"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pango.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract alias from description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern = re.compile('Alias of ([\\S]*?),', re.IGNORECASE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_alias(row):\n",
    "    match = pattern.findall(str(row.description))\n",
    "    if len(match) > 0:\n",
    "        return match[0]\n",
    "    else:\n",
    "        return ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pango['alias'] = pango.apply(get_alias, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "pango['predecessor'] = pango['alias'].str.rsplit('.', 1, expand=True)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lineage</th>\n",
       "      <th>description</th>\n",
       "      <th>alias</th>\n",
       "      <th>predecessor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1275</th>\n",
       "      <td>AQ.1</td>\n",
       "      <td>Alias of B.1.1.39.1, Finland lineage</td>\n",
       "      <td>B.1.1.39.1</td>\n",
       "      <td>B.1.1.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1276</th>\n",
       "      <td>AQ.2</td>\n",
       "      <td>Alias of B.1.1.39.2, Denmark lineage</td>\n",
       "      <td>B.1.1.39.2</td>\n",
       "      <td>B.1.1.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1277</th>\n",
       "      <td>AS.1</td>\n",
       "      <td>Alias of B.1.1.317.1, UK lineage</td>\n",
       "      <td>B.1.1.317.1</td>\n",
       "      <td>B.1.1.317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1278</th>\n",
       "      <td>AS.2</td>\n",
       "      <td>Alias of B.1.1.317.2, UK lineage</td>\n",
       "      <td>B.1.1.317.2</td>\n",
       "      <td>B.1.1.317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1279</th>\n",
       "      <td>AT.1</td>\n",
       "      <td>Alias of B.1.1.370.1, Russia and Finland linea...</td>\n",
       "      <td>B.1.1.370.1</td>\n",
       "      <td>B.1.1.370</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     lineage                                        description        alias  \\\n",
       "1275    AQ.1               Alias of B.1.1.39.1, Finland lineage   B.1.1.39.1   \n",
       "1276    AQ.2               Alias of B.1.1.39.2, Denmark lineage   B.1.1.39.2   \n",
       "1277    AS.1                   Alias of B.1.1.317.1, UK lineage  B.1.1.317.1   \n",
       "1278    AS.2                   Alias of B.1.1.317.2, UK lineage  B.1.1.317.2   \n",
       "1279    AT.1  Alias of B.1.1.370.1, Russia and Finland linea...  B.1.1.370.1   \n",
       "\n",
       "     predecessor  \n",
       "1275    B.1.1.39  \n",
       "1276    B.1.1.39  \n",
       "1277   B.1.1.317  \n",
       "1278   B.1.1.317  \n",
       "1279   B.1.1.370  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pango.tail(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Split into sublineages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_lineage(row):\n",
    "    lineage = row['lineage']\n",
    "    lineages =  np.empty(4, dtype=object)\n",
    "\n",
    "    for i in range(lineages.size):\n",
    "        lineages[i] = lineage\n",
    "        lineage = lineage.rpartition('.')[0]\n",
    "\n",
    "    return lineages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "pango[['l0', 'l1', 'l2', 'l3']] = pango.apply(split_lineage, axis=1, result_type='expand')\n",
    "pango['levels'] = pango['lineage'].str.count('\\.') + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lineage</th>\n",
       "      <th>description</th>\n",
       "      <th>alias</th>\n",
       "      <th>predecessor</th>\n",
       "      <th>l0</th>\n",
       "      <th>l1</th>\n",
       "      <th>l2</th>\n",
       "      <th>l3</th>\n",
       "      <th>levels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>B.1.143</td>\n",
       "      <td>Indian lineage</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>B.1.143</td>\n",
       "      <td>B.1</td>\n",
       "      <td>B</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1029</th>\n",
       "      <td>B.1.572</td>\n",
       "      <td>USA, split from B.1.325</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>B.1.572</td>\n",
       "      <td>B.1</td>\n",
       "      <td>B</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>B.1.195</td>\n",
       "      <td>UAE</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>B.1.195</td>\n",
       "      <td>B.1</td>\n",
       "      <td>B</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>727</th>\n",
       "      <td>B.1.254</td>\n",
       "      <td>Scotland</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>B.1.254</td>\n",
       "      <td>B.1</td>\n",
       "      <td>B</td>\n",
       "      <td></td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>229</th>\n",
       "      <td>B.1.1.267</td>\n",
       "      <td>Irish lineage</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>B.1.1.267</td>\n",
       "      <td>B.1.1</td>\n",
       "      <td>B.1</td>\n",
       "      <td>B</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        lineage              description alias predecessor         l0     l1  \\\n",
       "523     B.1.143           Indian lineage                      B.1.143    B.1   \n",
       "1029    B.1.572  USA, split from B.1.325                      B.1.572    B.1   \n",
       "671     B.1.195                      UAE                      B.1.195    B.1   \n",
       "727     B.1.254                 Scotland                      B.1.254    B.1   \n",
       "229   B.1.1.267            Irish lineage                    B.1.1.267  B.1.1   \n",
       "\n",
       "       l2 l3  levels  \n",
       "523     B          3  \n",
       "1029    B          3  \n",
       "671     B          3  \n",
       "727     B          3  \n",
       "229   B.1  B       4  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pango.sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "pango.to_csv(NEO4J_IMPORT / \"00b-PANGOLineage.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}