Notebook_2_API_queries.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting Dimensions and Altmetrics data\n",
    "\n",
    "In this notebook, starting from a list of COVID19 publications with a DOI or PMID, we query the Dimensions and Altmetrics APIs. We then export the results in JSON, according to the format and structure we need at CWTS. Yours might vary, but adapting these scripts should be straightforward.\n",
    "\n",
    "Dimensions API reference: https://docs.dimensions.ai/dsl/index.html\n",
    "Altmetrics API reference: http://api.altmetric.com\n",
    "\n",
    "*Please note you will need your own access keys from each of the two APIs to use this code.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# magics, warnings and imports\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import warnings; warnings.simplefilter('ignore')\n",
    "\n",
    "import os, random, codecs, json, time\n",
    "from tqdm.notebook import tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "seed = 99\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the pub dataframe (see Notebook_1 for this)\n",
    "\n",
    "df_pub = pd.read_csv(\"datasets_output/df_pub.csv\", compression=\"gzip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(160656, 15)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pub.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_id</th>\n",
       "      <th>title</th>\n",
       "      <th>abstract</th>\n",
       "      <th>publication_year</th>\n",
       "      <th>publication_month</th>\n",
       "      <th>journal</th>\n",
       "      <th>volume</th>\n",
       "      <th>issue</th>\n",
       "      <th>pages</th>\n",
       "      <th>doi</th>\n",
       "      <th>pmid</th>\n",
       "      <th>pmcid</th>\n",
       "      <th>dimensions_id</th>\n",
       "      <th>arxiv_id</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Coronavirus disease 2019 (COVID-19) and simult...</td>\n",
       "      <td>Coronavirus disease 2019 (COVID-19) is a recen...</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>CardioVascular Journal of Africa</td>\n",
       "      <td>31</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1-4</td>\n",
       "      <td>10-5830-cvja-2020-016</td>\n",
       "      <td>32490506.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pub.1128171889</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-07-02 10:54:27.347501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>[Ten hot issues of breast cancer under the nov...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Chinese medical journal</td>\n",
       "      <td>100</td>\n",
       "      <td>0</td>\n",
       "      <td>e002</td>\n",
       "      <td>10.0376/cma.j.issn.0376-2491.2020.0002</td>\n",
       "      <td>32036640.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pub.1124777091</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-07-02 10:54:27.347501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Cohesion in Distancing.</td>\n",
       "      <td>In isolation, we are physically apart; in soli...</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>The AMA Journal of Ethic</td>\n",
       "      <td>22</td>\n",
       "      <td>4</td>\n",
       "      <td>e344-345</td>\n",
       "      <td>10.1001/amajethics.2020.344</td>\n",
       "      <td>32345430.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pub.1127182341</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-07-02 10:54:27.347501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Salvation in a Time of Plague.</td>\n",
       "      <td>Health workers offer their skills and care to ...</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>The AMA Journal of Ethic</td>\n",
       "      <td>22</td>\n",
       "      <td>5</td>\n",
       "      <td>e441-445</td>\n",
       "      <td>10.1001/amajethics.2020.441</td>\n",
       "      <td>32449663.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>pub.1127905371</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-07-02 10:54:27.347501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>What Should Health Science Journalists Do in E...</td>\n",
       "      <td>Journalists have long covered outbreaks of inf...</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>AMA journal of ethics</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.1001/amajethics.2020.55</td>\n",
       "      <td>31958392.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2020-07-02 10:54:27.347501</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pub_id                                              title  \\\n",
       "0       0  Coronavirus disease 2019 (COVID-19) and simult...   \n",
       "1       1  [Ten hot issues of breast cancer under the nov...   \n",
       "2       2                            Cohesion in Distancing.   \n",
       "3       3                     Salvation in a Time of Plague.   \n",
       "4       4  What Should Health Science Journalists Do in E...   \n",
       "\n",
       "                                            abstract  publication_year  \\\n",
       "0  Coronavirus disease 2019 (COVID-19) is a recen...            2020.0   \n",
       "1                                                NaN            2020.0   \n",
       "2  In isolation, we are physically apart; in soli...            2020.0   \n",
       "3  Health workers offer their skills and care to ...            2020.0   \n",
       "4  Journalists have long covered outbreaks of inf...            2020.0   \n",
       "\n",
       "   publication_month                           journal volume issue     pages  \\\n",
       "0                5.0  CardioVascular Journal of Africa     31   NaN       1-4   \n",
       "1                2.0           Chinese medical journal    100     0      e002   \n",
       "2                4.0          The AMA Journal of Ethic     22     4  e344-345   \n",
       "3                5.0          The AMA Journal of Ethic     22     5  e441-445   \n",
       "4                1.0             AMA journal of ethics    NaN   NaN       NaN   \n",
       "\n",
       "                                      doi        pmid pmcid   dimensions_id  \\\n",
       "0                   10-5830-cvja-2020-016  32490506.0   NaN  pub.1128171889   \n",
       "1  10.0376/cma.j.issn.0376-2491.2020.0002  32036640.0   NaN  pub.1124777091   \n",
       "2             10.1001/amajethics.2020.344  32345430.0   NaN  pub.1127182341   \n",
       "3             10.1001/amajethics.2020.441  32449663.0   NaN  pub.1127905371   \n",
       "4              10.1001/amajethics.2020.55  31958392.0   NaN             NaN   \n",
       "\n",
       "  arxiv_id                   timestamp  \n",
       "0      NaN  2020-07-02 10:54:27.347501  \n",
       "1      NaN  2020-07-02 10:54:27.347501  \n",
       "2      NaN  2020-07-02 10:54:27.347501  \n",
       "3      NaN  2020-07-02 10:54:27.347501  \n",
       "4      NaN  2020-07-02 10:54:27.347501  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's get identifiers out\n",
    "\n",
    "dois = df_pub[pd.notna(df_pub.doi)].doi.values\n",
    "pmids = df_pub[(pd.isna(df_pub.doi)) & (pd.notna(df_pub.pmid))].pmid.values\n",
    "pmids = [str(int(i)) for i in pmids]\n",
    "pmcids = df_pub[(pd.isna(df_pub.doi)) & (pd.isna(df_pub.pmid)) & (pd.notna(df_pub.pmcid))].pmcid.values\n",
    "dids = df_pub[(pd.isna(df_pub.doi)) & (pd.isna(df_pub.pmid)) & (pd.isna(df_pub.pmcid)) & (pd.notna(df_pub.dimensions_id))].dimensions_id.values\n",
    "arxivids = df_pub[(pd.isna(df_pub.doi)) & (pd.isna(df_pub.pmid)) & (pd.isna(df_pub.pmcid)) & (pd.notna(df_pub.arxiv_id))].arxiv_id.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "147256\n",
      "11306\n",
      "68\n",
      "1332\n",
      "2026\n"
     ]
    }
   ],
   "source": [
    "print(len(dois))\n",
    "print(len(pmids))\n",
    "print(len(pmcids))\n",
    "print(len(dids))\n",
    "print(len(arxivids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# some DOIs will need cleaning\n",
    "def clean_doi(d):\n",
    "    if isinstance(d,str):\n",
    "        d = d.replace(\"https://doi.org/\",\"\")\n",
    "        d = d.replace(\"doi:\",\"\")\n",
    "        return d\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "dois = [clean_doi(d) for d in dois]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Schema to convert to CWTS-compatible JSON. Skip this if you prefer to have the Dimensions' schema.\n",
    "\n",
    "mapping_scheme = {\"id\": None,\n",
    "\"format\": None,\n",
    "\"status\": None,\n",
    "\"publication_type\": None,\n",
    "\"doi\": None,\n",
    "\"pmid\": None,\n",
    "\"pmcid\": None,\n",
    "\"title\": None,\n",
    "\"year\" : None,\n",
    "\"publication_date\" : None,\n",
    "\"volume\" : None,\n",
    "\"issue\" : None,\n",
    "\"pages\" : None,\n",
    "\"open_access_versions\": [],\n",
    "\"concepts\": {},\n",
    "\"journal\": {\"id\": None, \"title\": None, \"issn\": None, \"eissn\": None}, \n",
    "\"publisher\": {\"id\": None, \"name\": None},\n",
    "\"open_access_categories\": [],\n",
    "\"journal_lists\": [],\n",
    "\"author_affiliations\": [], #\"author_affiliations\": [{\"first_name\": \"Sunir\", \"last_name\": \"Gohil\", \"researcher_id\": \"ur.01154753576.20\", \"grid_ids\": []}, {\"first_name\": \"Sabine\", \"last_name\": \"Vuik\", \"researcher_id\": \"ur.015721262671.44\", \"grid_ids\": []}, {\"first_name\": \"Ara\", \"last_name\": \"Darzi\", \"researcher_id\": \"ur.01255016073.58\", \"grid_ids\": []}]\n",
    "\"funding\": [], #SKIP for now\n",
    "\"for\": [], # [{\"first_level\": {\"id\": \"11\", \"name\": \"Medical and Health Sciences\"}, \"second_level\": {\"id\": \"1117\", \"name\": \"Public Health and Health Services\"}}]\n",
    "\"language\": None,\n",
    "\"references\": [],\n",
    "\"clinical_trials\": [], #SKIP for now\n",
    "\"created_in_dimensions\" : None,\n",
    "\"version_of_record\" : None,\n",
    "\"times_cited\" : None, #NEW\n",
    "\"relative_citation_ratio\" : None, #NEW\n",
    "\"resulting_publication_doi\": None, #NEW\n",
    "\"funding\": [], #NEW\n",
    "\"mesh_headings\": [], #NEW\n",
    "\"altmetric_id\": None #NEW\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://app.dimensions.ai/browse/categories/publication/for\n",
    "for_highest_level = {\"01\":\"Mathematical Sciences\",\n",
    "\"02\":\"Physical Sciences\",\n",
    "\"03\": \"Chemical Sciences\",\n",
    "\"04\": \"Earth Sciences\",\n",
    "\"05\": \"Environmental Sciences\",\n",
    "\"06\": \"Biological Sciences\",\n",
    "\"07\": \"Agricultural and Veterinary Sciences\",\n",
    "\"08\": \"Information and Computing Sciences\",\n",
    "\"09\": \"Engineering\",\n",
    "\"10\": \"Technology\",\n",
    "\"11\": \"Medical and Health Sciences\",\n",
    "\"12\": \"Built Environment and Design\",\n",
    "\"13\": \"Education\",\n",
    "\"14\": \"Economics\",\n",
    "\"15\": \"Commerce, Management, Tourism and Services\",\n",
    "\"16\": \"Studies in Human Society\",\n",
    "\"17\": \"Psychology and Cognitive Sciences\",\n",
    "\"18\": \"Law and Legal Studies\",\n",
    "\"19\": \"Studies in Creative Arts and Writing\",\n",
    "\"20\": \"Language, Communication and Culture\",\n",
    "\"21\": \"History and Archaeology\",\n",
    "\"22\": \"Philosophy and Religious Studies\"}\n",
    "\n",
    "publishers_dict = {None:None}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is ugly and could be improved\n",
    "import copy\n",
    "\n",
    "def convert_json(input_from_api, publishers_dict=publishers_dict, mapping_scheme=mapping_scheme):\n",
    "    new_json = copy.deepcopy(mapping_scheme)\n",
    "    # direct fields\n",
    "    new_json[\"title\"] = input_from_api[\"title\"]\n",
    "    new_json[\"id\"] = input_from_api[\"id\"]\n",
    "    if \"doi\" in input_from_api.keys():\n",
    "        new_json[\"doi\"] = input_from_api[\"doi\"]\n",
    "    if \"altmetric_id\" in input_from_api.keys():\n",
    "        new_json[\"altmetric_id\"] = input_from_api[\"altmetric_id\"]\n",
    "    new_json[\"publication_type\"] = input_from_api[\"type\"]\n",
    "    if \"year\" in input_from_api.keys():\n",
    "        new_json[\"year\"] = input_from_api[\"year\"]\n",
    "    if \"date\" in input_from_api.keys():\n",
    "        new_json[\"publication_date\"] = input_from_api[\"date\"]\n",
    "    new_json[\"times_cited\"] = input_from_api[\"times_cited\"]\n",
    "    if \"references\" in input_from_api.keys():\n",
    "        new_json[\"references\"] = input_from_api[\"references\"]\n",
    "    if \"relative_citation_ratio\" in input_from_api.keys():\n",
    "        new_json[\"relative_citation_ratio\"] = input_from_api[\"relative_citation_ratio\"]\n",
    "    if \"volume\" in input_from_api.keys():\n",
    "        new_json[\"volume\"] = input_from_api[\"volume\"]\n",
    "    if \"issue\" in input_from_api.keys():\n",
    "        new_json[\"issue\"] = input_from_api[\"issue\"]\n",
    "    if \"pages\" in input_from_api.keys():\n",
    "        new_json[\"pages\"] = input_from_api[\"pages\"]\n",
    "    if \"pmid\" in input_from_api.keys():\n",
    "        new_json[\"pmid\"] = input_from_api[\"pmid\"]\n",
    "    if \"pmcid\" in input_from_api.keys():\n",
    "        new_json[\"pmcid\"] = input_from_api[\"pmcid\"]\n",
    "    if \"resulting_publication_doi\" in input_from_api.keys():\n",
    "        new_json[\"resulting_publication_doi\"] = input_from_api[\"resulting_publication_doi\"]\n",
    "    if \"concepts\" in input_from_api.keys():\n",
    "        for c in input_from_api[\"concepts\"]:\n",
    "            new_json[\"concepts\"].update({c:1.0})\n",
    "    if \"supporting_grant_ids\" in input_from_api.keys():\n",
    "        for c in input_from_api[\"supporting_grant_ids\"]:\n",
    "            new_json[\"funding\"].append({\"grid_id\": None, \"grant_id\": c})\n",
    "    if \"journal\" in input_from_api.keys():\n",
    "        new_json[\"journal\"][\"id\"] = input_from_api[\"journal\"][\"id\"]\n",
    "        new_json[\"journal\"][\"title\"] = input_from_api[\"journal\"][\"title\"]\n",
    "        if \"issn\" in input_from_api.keys():\n",
    "            new_json[\"journal\"][\"issn\"] = input_from_api[\"issn\"][0]\n",
    "            if len(input_from_api[\"issn\"])>1:\n",
    "                new_json[\"journal\"][\"eissn\"] = input_from_api[\"issn\"][1]\n",
    "    if \"publisher\" in input_from_api.keys():\n",
    "        new_json[\"publisher\"][\"name\"] = input_from_api[\"publisher\"]\n",
    "        if input_from_api[\"publisher\"] in publishers_dict.keys():\n",
    "            new_json[\"publisher\"][\"id\"] = publishers_dict[input_from_api[\"publisher\"]]\n",
    "    if \"open_access\" in input_from_api.keys():\n",
    "        new_json[\"open_access_categories\"] = input_from_api[\"open_access\"]\n",
    "    if \"journal_lists\" in input_from_api.keys():\n",
    "        new_json[\"journal_lists\"] = input_from_api[\"journal_lists\"]\n",
    "    if \"author_affiliations\" in input_from_api.keys():\n",
    "        for affiliation in input_from_api[\"author_affiliations\"]:\n",
    "            for researcher in affiliation:\n",
    "                new_researcher = {\"first_name\": researcher[\"first_name\"], \"last_name\": researcher[\"last_name\"], \"researcher_id\": researcher[\"researcher_id\"], \"grid_ids\": []}\n",
    "                if \"affiliations\" in researcher.keys():\n",
    "                    new_researcher[\"grid_ids\"] = [x[\"id\"] for x in researcher[\"affiliations\"] if \"id\" in x.keys()]\n",
    "                new_json[\"author_affiliations\"].append(new_researcher)\n",
    "    if \"mesh_terms\" in input_from_api.keys():\n",
    "        new_json[\"mesh_headings\"] = input_from_api[\"mesh_terms\"]\n",
    "    if \"FOR\" in input_from_api.keys():\n",
    "        for item in input_from_api[\"FOR\"]:\n",
    "            item_id = item[\"name\"][:4]\n",
    "            upper_item_id = item_id[:2]\n",
    "            item_name = item[\"name\"][5:]\n",
    "            new_json[\"for\"].append({\"first_level\":{\"id\": upper_item_id,\"name\": for_highest_level[upper_item_id]},\"second_level\":{\"id\": item_id,\"name\": item_name}})\n",
    "    return new_json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chunks(lst, n):\n",
    "    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n",
    "    for i in range(0, len(lst), n):\n",
    "        yield lst[i:i + n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get credentials key\n",
    "# USE YOURS HERE\n",
    "\n",
    "import configparser\n",
    "config = configparser.ConfigParser()\n",
    "config.read(\"credentials/conf.ini\")\n",
    "dimensions_username = config[\"DIMENSIONS\"][\"username\"]\n",
    "dimensions_password = config[\"DIMENSIONS\"][\"password\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "\n",
    "#   The credentials to be used\n",
    "login = {\n",
    "    'username': dimensions_username,\n",
    "    'password': dimensions_password\n",
    "}\n",
    "\n",
    "#   Send credentials to login url to retrieve token. Raise\n",
    "#   an error, if the return code indicates a problem.\n",
    "#   Please use the URL of the system you'd like to access the API\n",
    "#   in the example below.\n",
    "resp = requests.post('https://app.dimensions.ai/api/auth.json', json=login)\n",
    "resp.raise_for_status()\n",
    "\n",
    "#   Create http header using the generated token.\n",
    "headers = {\n",
    "    'Authorization': \"JWT \" + resp.json()['token']\n",
    "}\n",
    "\n",
    "queried_publishers = list()\n",
    "already_done = list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_results = list()\n",
    "payloads = {\"id\":dids,\"pmcid\":pmcids,\"pmid\":pmids,\"doi\":dois}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8dfa3024032c4b5d87743bdd942dac24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pmcid\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e227c396be014f16982e7012cd8eb792",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "pmid\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ae0bd699baa48259f523bf92fea6f51",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "doi\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65f2ee0334784bd696c95d3fbca88313",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# get and save all results for DOIs - CWTS format\n",
    "query_template_1 = 'search publications where %s in [\"'\n",
    "query_template_2 = '\"] return publications[basics+extras+pmcid+publisher+journal_lists+concepts+issn+altmetric_id+resulting_publication_doi+mesh_terms+supporting_grant_ids] limit 300'\n",
    "query_template_orgs_1 = 'search organizations where name = \"'\n",
    "query_template_orgs_2 = '\" return organizations[basics]'\n",
    "limit = 300\n",
    "current_payload = list()\n",
    "\n",
    "for key,payload in payloads.items():\n",
    "    print(key)\n",
    "    for n,i in tqdm(enumerate(payload)):\n",
    "        current_payload.append(i)\n",
    "        if (n > 0 and n % limit == 0) or n >= (len(payload)-1): # query Dimensions, limit reached\n",
    "            #print((query_template_1+'\",\"'.join(current_payload)+query_template_2))\n",
    "            resp = requests.post(\n",
    "                'https://app.dimensions.ai/api/dsl.json',\n",
    "                data=(query_template_1%key+'\",\"'.join(current_payload)+query_template_2).encode(),\n",
    "                headers=headers)\n",
    "            #print(resp.json())\n",
    "\n",
    "            #   Display raw result\n",
    "            #print(resp)\n",
    "            #print(resp.url)\n",
    "            r = resp.json()\n",
    "            #print(r[\"_stats\"][\"total_count\"])\n",
    "            #print(len(r[\"publications\"]))\n",
    "            \n",
    "            if not \"publications\" in r.keys():\n",
    "                continue\n",
    "            \n",
    "            new_publishers = list(set([x[\"publisher\"] for x in r[\"publications\"] if \"publisher\" in x.keys()]).difference(set([p for p in publishers_dict.keys()])))\n",
    "            new_publishers = [p for p in new_publishers if not p in queried_publishers]\n",
    "            queried_publishers.extend(new_publishers)\n",
    "            #print(len(new_publishers))\n",
    "            if len(new_publishers)>0:\n",
    "                for np in new_publishers:\n",
    "                    presp = requests.post(\n",
    "                    'https://app.dimensions.ai/api/dsl.json',\n",
    "                    data=(query_template_orgs_1+np+query_template_orgs_2).encode(),\n",
    "                    headers=headers)\n",
    "                    try:\n",
    "                        if 'organizations' in presp.json().keys() and len(presp.json()['organizations'])>0 and 'types' in presp.json()['organizations'][0].keys() and 'Company' in presp.json()['organizations'][0]['types']:\n",
    "                            publishers_dict[np] = presp.json()['organizations'][0]['id']\n",
    "                    except:\n",
    "                        continue\n",
    "            \n",
    "            converted_json = [convert_json(result) for result in r[\"publications\"]]\n",
    "            all_results.extend(converted_json)\n",
    "            already_done.extend(current_payload)\n",
    "            current_payload = list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{None: None,\n",
       " 'Springer Nature': 'grid.420067.7',\n",
       " 'Wiley': 'grid.462413.6',\n",
       " 'Mary Ann Liebert': 'grid.431455.2',\n",
       " 'Thieme': 'grid.466788.3',\n",
       " 'Karger Publishers': 'grid.431201.5',\n",
       " 'De Gruyter': 'grid.431130.4',\n",
       " 'Decker': 'grid.467251.3',\n",
       " 'Bentham Science Publishers': 'grid.431355.5',\n",
       " 'Springer Publishing Company': 'grid.467062.1',\n",
       " 'SLACK': 'grid.467574.3',\n",
       " 'Begell House': 'grid.431346.7',\n",
       " 'Bioscientifica': 'grid.431370.2',\n",
       " 'Ubiquity Press': 'grid.431280.c',\n",
       " 'Intellect': 'grid.498173.0',\n",
       " 'IRBIS': 'grid.485652.8',\n",
       " 'Wolfram Research': 'grid.486191.3',\n",
       " 'BioAxis DNA Research Centre': 'grid.473487.b'}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "publishers_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150749\n"
     ]
    }
   ],
   "source": [
    "print(len(all_results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store data\n",
    "out_folder = \"datasets_output/json_dimensions_cwts\"\n",
    "\n",
    "for n,chunk in enumerate(chunks(all_results,10000)):\n",
    "    with codecs.open(os.path.join(out_folder,\"chunk_%d\"%n)+\".json\",\"w\") as f:\n",
    "        for r in chunk:\n",
    "            json.dump(r, f)\n",
    "            f.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get DOIs from Dimensions out\n",
    "dois_dimensions = [clean_doi(d[\"doi\"]) for d in all_results]\n",
    "pmids_dimensions = [d[\"pmid\"] for d in all_results if not d[\"doi\"]]\n",
    "altids_dimensions = list(set([d[\"altmetric_id\"] for d in all_results if not (d['doi'] or d[\"pmid\"])]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "150749\n",
      "4824\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "print(len(dois_dimensions))\n",
    "print(len(pmids_dimensions))\n",
    "print(len(altids_dimensions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_dois = list(set(dois).union(set(dois_dimensions)))\n",
    "all_pmids = list(set(pmids).union(set(pmids_dimensions)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "154892\n",
      "11307\n",
      "147256\n",
      "11306\n",
      "68\n",
      "2026\n"
     ]
    }
   ],
   "source": [
    "print(len(all_dois))\n",
    "print(len(all_pmids))\n",
    "print(len(dois))\n",
    "print(len(pmids))\n",
    "print(len(pmcids))\n",
    "print(len(arxivids))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Altmetrics\n",
    "\n",
    "Note the Altmetrics API cannot be queried by PMCID, so we try to use all available DOIs and PMIDs from Dimensions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests, time\n",
    "from requests.adapters import HTTPAdapter\n",
    "from requests.packages.urllib3.util.retry import Retry\n",
    "\n",
    "doi_base_url = \"http://api.altmetric.com/v1/fetch/doi/\"\n",
    "pmid_base_url = \"http://api.altmetric.com/v1/fetch/pmid/\"\n",
    "altid_base_url = \"http://api.altmetric.com/v1/fetch/id/\"\n",
    "arxivid_base_url = \"https://api.altmetric.com/version/fetch/arxiv_id/\"\n",
    "\n",
    "out_folder = \"datasets_output/json_altmetrics_cwts\"\n",
    "all_tweet_ids = list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get API key\n",
    "# USE YOURS HERE\n",
    "\n",
    "import configparser\n",
    "config = configparser.ConfigParser()\n",
    "config.read(\"credentials/conf.ini\")\n",
    "#api_key = config[\"ALTMETRICS\"][\"keyh\"]\n",
    "api_key = \"\"\n",
    "payload = {'key': api_key}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "553db381e099436d8115c84f62f0d522",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "### Query by DOI\n",
    "\n",
    "# this scaffolding is needed to avoid the request per second limitation of Altmetrics, which is variably enforced \n",
    "session = requests.Session()\n",
    "retry = Retry(connect=5, backoff_factor=0.5)\n",
    "adapter = HTTPAdapter(max_retries=retry)\n",
    "session.mount('http://', adapter)\n",
    "session.mount('https://', adapter)\n",
    "\n",
    "for n,doi in tqdm(enumerate(all_dois[100800:-22600])):\n",
    "    if not doi:\n",
    "        continue\n",
    "    #if n>0 and (n)%500 == 0:\n",
    "        #print(\"Sleeping\")\n",
    "        #time.sleep(60) # avoid being banned\n",
    "    r = session.get(doi_base_url+doi, params=payload)\n",
    "    if not r.status_code == 200:\n",
    "        #print(r.headers)\n",
    "        if r.status_code == 429: # means API limitations, we need to backoff\n",
    "            all_dois.append(doi)\n",
    "            time.sleep(60)\n",
    "        continue\n",
    "        \n",
    "    f_name = doi.replace(\".\",\"_\")\n",
    "    f_name = f_name.replace(\"/\",\":\")\n",
    "    with codecs.open(os.path.join(out_folder,f_name)+\".json\",\"w\") as f:\n",
    "        json.dump(r.json(), f)\n",
    "    if \"posts\" in r.json().keys():\n",
    "        if isinstance(r.json()[\"posts\"],dict) and \"twitter\" in r.json()[\"posts\"].keys():\n",
    "            for tweet in r.json()[\"posts\"][\"twitter\"]:\n",
    "                if \"tweeter_id\" in tweet[\"author\"].keys():\n",
    "                    all_tweet_ids.append((doi,\"\",tweet[\"tweet_id\"],str(tweet[\"author\"][\"tweeter_id\"])))\n",
    "                else:\n",
    "                    all_tweet_ids.append((doi,\"\",tweet[\"tweet_id\"],\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b62bb2b6b4a747ed8ca6a0244e17021b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=11307.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Query by PMID\n",
    "\n",
    "# this scaffolding is needed to avoid the request per second limitation of Altmetrics, which is variably enforced \n",
    "session = requests.Session()\n",
    "retry = Retry(connect=5, backoff_factor=0.5)\n",
    "adapter = HTTPAdapter(max_retries=retry)\n",
    "session.mount('http://', adapter)\n",
    "session.mount('https://', adapter)\n",
    "\n",
    "for pmid in tqdm(all_pmids):\n",
    "    if not pmid:\n",
    "        continue\n",
    "    r = session.get(pmid_base_url+str(int(pmid)), params=payload)\n",
    "    if not r.status_code == 200:\n",
    "        if r.status_code == 429: # means API limitations, we need to backoff\n",
    "            all_dois.append(doi)\n",
    "            time.sleep(60)\n",
    "        continue\n",
    "    f_name = str(int(pmid))\n",
    "    with codecs.open(os.path.join(out_folder,f_name)+\".json\",\"w\") as f:\n",
    "        json.dump(r.json(), f)\n",
    "    if \"posts\" in r.json().keys():\n",
    "        if isinstance(r.json()[\"posts\"],dict) and \"twitter\" in r.json()[\"posts\"].keys():\n",
    "            for tweet in r.json()[\"posts\"][\"twitter\"]:\n",
    "                if \"tweeter_id\" in tweet[\"author\"].keys():\n",
    "                    all_tweet_ids.append((\"\",str(int(pmid)),tweet[\"tweet_id\"],str(tweet[\"author\"][\"tweeter_id\"])))\n",
    "                else:\n",
    "                    all_tweet_ids.append((\"\",str(int(pmid)),tweet[\"tweet_id\"],\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dddb85bd50814be29aaa11cc21cefb00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=2026.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Query by arXiv ID\n",
    "\n",
    "# this scaffolding is needed to avoid the request per second limitation of Altmetrics, which is variably enforced \n",
    "session = requests.Session()\n",
    "retry = Retry(connect=5, backoff_factor=0.5)\n",
    "adapter = HTTPAdapter(max_retries=retry)\n",
    "session.mount('http://', adapter)\n",
    "session.mount('https://', adapter)\n",
    "\n",
    "for aid in tqdm(arxivids):\n",
    "    if not aid:\n",
    "        continue\n",
    "    r = session.get(arxivid_base_url+str(aid), params=payload)\n",
    "    if not r.status_code == 200:\n",
    "        if r.status_code == 429: # means API limitations, we need to backoff\n",
    "            all_dois.append(doi)\n",
    "            time.sleep(60)\n",
    "        continue\n",
    "    f_name = str(int(aid))\n",
    "    with codecs.open(os.path.join(out_folder,f_name)+\".json\",\"w\") as f:\n",
    "        json.dump(r.json(), f)\n",
    "    if \"posts\" in r.json().keys():\n",
    "        if isinstance(r.json()[\"posts\"],dict) and \"twitter\" in r.json()[\"posts\"].keys():\n",
    "            for tweet in r.json()[\"posts\"][\"twitter\"]:\n",
    "                if \"tweeter_id\" in tweet[\"author\"].keys():\n",
    "                    all_tweet_ids.append((\"\",\"\",tweet[\"tweet_id\"],str(tweet[\"author\"][\"tweeter_id\"])))\n",
    "                else:\n",
    "                    all_tweet_ids.append((\"\",\"\",tweet[\"tweet_id\"],\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65aa5f2e5b8e45f59f4300ede67f5eac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Query by Altmetric ID\n",
    "\n",
    "# this scaffolding is needed to avoid the request per second limitation of Altmetrics, which is variably enforced \n",
    "session = requests.Session()\n",
    "retry = Retry(connect=5, backoff_factor=0.5)\n",
    "adapter = HTTPAdapter(max_retries=retry)\n",
    "session.mount('http://', adapter)\n",
    "session.mount('https://', adapter)\n",
    "\n",
    "for aid in tqdm(altids_dimensions):\n",
    "    if not aid:\n",
    "        continue\n",
    "    r = session.get(altid_base_url+str(int(aid)), params=payload)\n",
    "    if not r.status_code == 200:\n",
    "        if r.status_code == 429: # means API limitations, we need to backoff\n",
    "            all_dois.append(doi)\n",
    "            time.sleep(60)\n",
    "        continue\n",
    "    f_name = str(int(aid))\n",
    "    with codecs.open(os.path.join(out_folder,f_name)+\".json\",\"w\") as f:\n",
    "        json.dump(r.json(), f)\n",
    "    if \"posts\" in r.json().keys():\n",
    "        if isinstance(r.json()[\"posts\"],dict) and \"twitter\" in r.json()[\"posts\"].keys():\n",
    "            for tweet in r.json()[\"posts\"][\"twitter\"]:\n",
    "                if \"tweeter_id\" in tweet[\"author\"].keys():\n",
    "                    all_tweet_ids.append((\"\",\"\",tweet[\"tweet_id\"],str(tweet[\"author\"][\"tweeter_id\"])))\n",
    "                else:\n",
    "                    all_tweet_ids.append((\"\",\"\",tweet[\"tweet_id\"],\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_tweet_ids = list(set(all_tweet_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6199602"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_tweet_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "with codecs.open(\"datasets_output/all_tweet_ids.csv\", \"w\") as f:\n",
    "    f.write(\"doi,pmid,tweet_id,user_id\\n\")\n",
    "    for tweet in all_tweet_ids:\n",
    "        f.write(\",\".join(tweet)+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "# separately export twitter IDs if necessary (to hydrate them)\n",
    "\n",
    "out_folder = \"datasets_output/json_altmetrics_cwts\"\n",
    "all_tweet_ids = list()\n",
    "\n",
    "for root, dirs, files in os.walk(out_folder):\n",
    "    for file in files:\n",
    "        if \".json\" in file:\n",
    "            data = json.loads(codecs.open(os.path.join(root,file)).read())\n",
    "            if \"posts\" in data.keys():\n",
    "                if isinstance(data[\"posts\"],dict) and \"twitter\" in data[\"posts\"].keys():\n",
    "                    for tweet in data[\"posts\"][\"twitter\"]:\n",
    "                        doi = \"\"\n",
    "                        if \"doi\" in data[\"citation\"]:\n",
    "                            doi = str(data[\"citation\"][\"doi\"])\n",
    "                        pmid = \"\"\n",
    "                        if \"pmid\" in data[\"citation\"]:\n",
    "                            pmid = str(data[\"citation\"][\"pmid\"])\n",
    "                        user_id = \"\"\n",
    "                        if \"tweeter_id\" in tweet[\"author\"].keys():\n",
    "                            user_id = str(tweet[\"author\"][\"tweeter_id\"])\n",
    "                        all_tweet_ids.append((doi,pmid,str(tweet[\"tweet_id\"]),user_id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "covid",
   "language": "python",
   "name": "covid"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}