1.download.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Download biorxiv preprint table from the PrePubMed repository"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import logging\n",
    "\n",
    "import pandas\n",
    "import requests\n",
    "\n",
    "import utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Configure logging to write to file\n",
    "logging.basicConfig(level=logging.INFO, filename=os.path.join('logs/donwload.log'), filemode='w')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get `OmnesRes/prepub` version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sha': '6e5c180b3cf81cff9c58653aebd6b35556470260',\n",
       " 'type': 'commit',\n",
       " 'url': 'https://api.github.com/repos/OmnesRes/prepub/git/commits/6e5c180b3cf81cff9c58653aebd6b35556470260'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://api.github.com/repos/OmnesRes/prepub/git/refs/heads/master'\n",
    "response = requests.get(url)\n",
    "response = response.json()\n",
    "response['object']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load bioRxiv data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url = 'https://github.com/OmnesRes/prepub/raw/master/biorxiv/biorxiv_licenses.tsv'\n",
    "biorxiv_df = pandas.read_table(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Limit to preprints through November 2016\n",
    "biorxiv_df = biorxiv_df.query(\"Date <= '2017-06-07'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOI</th>\n",
       "      <th>Date</th>\n",
       "      <th>Subjects</th>\n",
       "      <th>License</th>\n",
       "      <th>Title</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Affiliations</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>http://dx.doi.org/10.1101/049031</td>\n",
       "      <td>2016-04-16</td>\n",
       "      <td>Microbiology</td>\n",
       "      <td>CC BY-NC</td>\n",
       "      <td>Alternative Growth Behavior of Mycobacterium A...</td>\n",
       "      <td>Peilin Zhang|Lawrence M Minardi|J. Todd Kuenst...</td>\n",
       "      <td>PZM Diagnostics, LLC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>http://dx.doi.org/10.1101/049049</td>\n",
       "      <td>2016-04-16</td>\n",
       "      <td>Genomics</td>\n",
       "      <td>CC BY-NC-ND</td>\n",
       "      <td>Lateral genetic transfers between eukaryotes a...</td>\n",
       "      <td>Sarah R Bordenstein|Seth R Bordenstein</td>\n",
       "      <td>Vanderbilt University</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                DOI        Date      Subjects      License  \\\n",
       "0  http://dx.doi.org/10.1101/049031  2016-04-16  Microbiology     CC BY-NC   \n",
       "1  http://dx.doi.org/10.1101/049049  2016-04-16      Genomics  CC BY-NC-ND   \n",
       "\n",
       "                                               Title  \\\n",
       "0  Alternative Growth Behavior of Mycobacterium A...   \n",
       "1  Lateral genetic transfers between eukaryotes a...   \n",
       "\n",
       "                                             Authors           Affiliations  \n",
       "0  Peilin Zhang|Lawrence M Minardi|J. Todd Kuenst...   PZM Diagnostics, LLC  \n",
       "1             Sarah R Bordenstein|Seth R Bordenstein  Vanderbilt University  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "biorxiv_df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove URL from DOIs\n",
    "biorxiv_df.DOI = biorxiv_df.DOI.str.extract(r'(10\\.[0-9]+/[0-9]+)', expand=False)\n",
    "biorxiv_df.License = biorxiv_df.License.str.replace('CC-BY', 'CC BY')\n",
    "biorxiv_df.License = biorxiv_df.License.fillna('None')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "author_df = (biorxiv_df\n",
    "    .pipe(utilities.tidy_split, column='Authors')\n",
    "    .rename(columns={'Authors': 'Author'})\n",
    "    [['DOI', 'Author']]\n",
    "    .sort_values(['DOI', 'Author'])\n",
    "    .drop_duplicates()\n",
    "    .reset_index(drop=True)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Standardize author names\n",
    "author_df['Standard_Author'] = author_df.Author.map(utilities.get_standard_author)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOI</th>\n",
       "      <th>Author</th>\n",
       "      <th>Standard_Author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>70102</th>\n",
       "      <td>10.1101/146837</td>\n",
       "      <td>Roxana E. Georgescu</td>\n",
       "      <td>Roxana Georgescu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70103</th>\n",
       "      <td>10.1101/146837</td>\n",
       "      <td>Ryan Mayle</td>\n",
       "      <td>Ryan Mayle</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  DOI               Author   Standard_Author\n",
       "70102  10.1101/146837  Roxana E. Georgescu  Roxana Georgescu\n",
       "70103  10.1101/146837           Ryan Mayle        Ryan Mayle"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "author_df.tail(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Mark Daly         43\n",
       "Benjamin Neale    34\n",
       "Name: Standard_Author, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Authors with the most preprints\n",
    "author_df.Standard_Author.value_counts().head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Subjects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOI</th>\n",
       "      <th>Subject</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10040</th>\n",
       "      <td>10.1101/146712</td>\n",
       "      <td>Developmental Biology</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10041</th>\n",
       "      <td>10.1101/146779</td>\n",
       "      <td>Neuroscience</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  DOI                Subject\n",
       "10040  10.1101/146712  Developmental Biology\n",
       "10041  10.1101/146779           Neuroscience"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a TSV with a row per preprint-subject pair\n",
    "subject_df = (biorxiv_df\n",
    "    .pipe(utilities.tidy_split, column='Subjects')\n",
    "    .rename(columns={'Subjects': 'Subject'})\n",
    "    [['DOI', 'Subject']]\n",
    "    .sort_values(['DOI', 'Subject'])\n",
    "    .reset_index(drop=True)\n",
    ")\n",
    "subject_df.tail(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    9985\n",
       "2      22\n",
       "5       2\n",
       "3       1\n",
       "Name: DOI, dtype: int64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of subjects per preprint\n",
    "subject_df.DOI.value_counts().value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Bioinformatics                            1486\n",
       "Neuroscience                              1346\n",
       "Evolutionary Biology                      1251\n",
       "Genomics                                  1099\n",
       "Genetics                                   799\n",
       "Microbiology                               522\n",
       "Ecology                                    483\n",
       "Systems Biology                            387\n",
       "Biophysics                                 346\n",
       "Cell Biology                               317\n",
       "Developmental Biology                      243\n",
       "Plant Biology                              235\n",
       "Cancer Biology                             232\n",
       "Molecular Biology                          195\n",
       "Biochemistry                               188\n",
       "Epidemiology                               168\n",
       "Animal Behavior and Cognition              146\n",
       "Synthetic Biology                          128\n",
       "Immunology                                 110\n",
       "Bioengineering                              99\n",
       "Physiology                                  57\n",
       "Zoology                                     50\n",
       "Scientific Communication and Education      44\n",
       "Pharmacology and Toxicology                 28\n",
       "Scientific Communication                    25\n",
       "Pathology                                   25\n",
       "Pharmacology                                18\n",
       "Paleontology                                 9\n",
       "Clinical Trials                              6\n",
       "Name: Subject, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of preprints by subject\n",
    "subject_df.Subject.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DOI</th>\n",
       "      <th>Date</th>\n",
       "      <th>License</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11350</th>\n",
       "      <td>10.1101/146779</td>\n",
       "      <td>2017-06-06</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11351</th>\n",
       "      <td>10.1101/146787</td>\n",
       "      <td>2017-06-06</td>\n",
       "      <td>CC BY</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11352</th>\n",
       "      <td>10.1101/146811</td>\n",
       "      <td>2017-06-06</td>\n",
       "      <td>CC BY-ND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11353</th>\n",
       "      <td>10.1101/146837</td>\n",
       "      <td>2017-06-06</td>\n",
       "      <td>CC BY</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  DOI        Date   License\n",
       "11350  10.1101/146779  2017-06-06      None\n",
       "11351  10.1101/146787  2017-06-06     CC BY\n",
       "11352  10.1101/146811  2017-06-06  CC BY-ND\n",
       "11353  10.1101/146837  2017-06-06     CC BY"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprint_df = (biorxiv_df\n",
    "    [['DOI', 'Date', 'License']]\n",
    "    .sort_values('DOI')\n",
    "    .reset_index(drop=True)\n",
    ")\n",
    "preprint_df.tail(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11354"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(preprint_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>License</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CC BY-NC-ND</td>\n",
       "      <td>0.349304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>None</td>\n",
       "      <td>0.292848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CC BY</td>\n",
       "      <td>0.196230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CC BY-NC</td>\n",
       "      <td>0.093976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CC BY-ND</td>\n",
       "      <td>0.067641</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         index   License\n",
       "0  CC BY-NC-ND  0.349304\n",
       "1         None  0.292848\n",
       "2        CC BY  0.196230\n",
       "3     CC BY-NC  0.093976\n",
       "4     CC BY-ND  0.067641"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Preprints by license\n",
    "preprint_df.License.value_counts(normalize=True).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2016    4901\n",
       "2017    3864\n",
       "2015    1706\n",
       "2014     806\n",
       "2013      77\n",
       "Name: Date, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Preprints by year\n",
    "preprint_df.Date.map(lambda date: date.split('-')[0]).value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save as a TSVs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = os.path.join('data', 'preprints.tsv')\n",
    "preprint_df.to_csv(path, sep='\\t', index=False)\n",
    "\n",
    "path = os.path.join('data', 'subjects.tsv')\n",
    "subject_df.to_csv(path, sep='\\t', index=False)\n",
    "\n",
    "path = os.path.join('data', 'authors.tsv')\n",
    "author_df.to_csv(path, sep='\\t', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:biorxiv-licenses]",
   "language": "python",
   "name": "conda-env-biorxiv-licenses-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}