Demonstration.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from robotsearch.parsers import ris"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "files = ['/home/iain/Downloads/Blue light for acne.txt', '/home/iain/Downloads/citation-export.ris']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defused a BOM\n",
      "other non numbered\n"
     ]
    }
   ],
   "source": [
    "\n",
    "with open(files[0], 'r') as f:\n",
    "    inp = ris.load(f)\n",
    "    \n",
    "# or try 'robotsearch/examples/pubmed.txt'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('TY', ['JOUR']),\n",
       "             ('AN', ['4223790']),\n",
       "             ('DA', ['Jun 24']),\n",
       "             ('DP', ['NLM']),\n",
       "             ('ET', ['1966/06/24']),\n",
       "             ('IS', ['13']),\n",
       "             ('J2', ['Drug and therapeutics bulletin']),\n",
       "             ('KW',\n",
       "              ['Acne Vulgaris/*drug therapy/radiotherapy',\n",
       "               'Humans',\n",
       "               'Ultraviolet Therapy']),\n",
       "             ('LA', ['eng']),\n",
       "             ('LB', ['PubMed 20180305']),\n",
       "             ('N1',\n",
       "              ['Journal Article',\n",
       "               'England',\n",
       "               'Drug Ther Bull. 1966 Jun 24;4(13):49-51.']),\n",
       "             ('PY', ['1966']),\n",
       "             ('SN', ['0012-6543 (Print)', '0012-6543']),\n",
       "             ('SP', ['49-51']),\n",
       "             ('ST', ['Management of acne vulgaris']),\n",
       "             ('T2', ['Drug Ther Bull']),\n",
       "             ('TI', ['Management of acne vulgaris']),\n",
       "             ('VL', ['4']),\n",
       "             ('ID', ['338']),\n",
       "             ('ER', [''])])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inp[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The inital search result has 4345 articles\n"
     ]
    }
   ],
   "source": [
    "print(\"The inital search result has {} articles\".format(len(inp)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/iain/anaconda3/envs/rr3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using Theano backend.\n",
      "/home/iain/anaconda3/envs/rr3/lib/python3.6/site-packages/keras/engine/topology.py:1242: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "  return cls(**config)\n",
      "/home/iain/anaconda3/envs/rr3/lib/python3.6/site-packages/keras/utils/conv_utils.py:87: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.\n",
      "  return np.copy(kernel[slices])\n",
      "/home/iain/anaconda3/envs/rr3/lib/python3.6/site-packages/keras/engine/topology.py:1242: UserWarning: The `Merge` layer is deprecated and will be removed after 08/2017. Use instead layers from `keras.layers.merge`, e.g. `add`, `concatenate`, etc.\n",
      "  return cls(**config)\n",
      "/home/iain/anaconda3/envs/rr3/lib/python3.6/site-packages/keras/utils/conv_utils.py:87: FutureWarning: Using a non-tuple sequence for multidimensional indexing is deprecated; use `arr[tuple(seq)]` instead of `arr[seq]`. In the future this will be interpreted as an array index, `arr[np.array(seq)]`, which will result either in an error or a different result.\n",
      "  return np.copy(kernel[slices])\n"
     ]
    }
   ],
   "source": [
    "from robotsearch.robots import rct_robot\n",
    "r = rct_robot.RCTRobot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defused a BOM\n",
      "other non numbered\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "OrderedDict([('TY', ['JOUR']),\n",
       "             ('AN', ['26977496']),\n",
       "             ('DA', ['Nov 19']),\n",
       "             ('DP', ['NLM']),\n",
       "             ('ET', ['2016/03/16']),\n",
       "             ('IS', ['20']),\n",
       "             ('J2', ['MMW Fortschritte der Medizin']),\n",
       "             ('KW',\n",
       "              ['Acne Vulgaris/physiopathology/*radiotherapy',\n",
       "               'Adolescent',\n",
       "               'Adult',\n",
       "               'Humans',\n",
       "               'Hyperthermia, Induced/*methods',\n",
       "               'Lasers, Semiconductor/*therapeutic use',\n",
       "               'Low-Level Light Therapy/*methods',\n",
       "               'Sebaceous Glands/physiopathology/*radiation effects']),\n",
       "             ('LA', ['ger']),\n",
       "             ('LB', ['PubMed 20180305']),\n",
       "             ('N1',\n",
       "              ['Comparative Study',\n",
       "               'News',\n",
       "               'Randomized Controlled Trial',\n",
       "               'Germany',\n",
       "               'MMW Fortschr Med. 2015 Nov 19;157(20):20.']),\n",
       "             ('OP', ['Photothermolyse soll Akne bekampfen.']),\n",
       "             ('PY', ['2015']),\n",
       "             ('SN', ['1438-3276 (Print)', '1438-3276']),\n",
       "             ('SP', ['20']),\n",
       "             ('ST', ['[Photothermolysis for acne control]']),\n",
       "             ('T2', ['MMW Fortschr Med']),\n",
       "             ('TI', ['[Photothermolysis for acne control]']),\n",
       "             ('VL', ['157']),\n",
       "             ('ID', ['34']),\n",
       "             ('ER', [''])])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from robotsearch.parsers import ris\n",
    "ris.loads(inp_r)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing RIS data\n",
      "ovid endnote format\n",
      "\t1 of 6\n",
      "\t2 of 6\n",
      "\t3 of 6\n",
      "\t4 of 6\n"
     ]
    }
   ],
   "source": [
    "rcts_sens = r.filter_articles(inp_r, filter_class='svm_cnn', filter_type='sensitive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('robotsearch/examples/ovid.txt', 'r') as f:\n",
    "    inp_r = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<1. >\n",
      "VN  - Ovid Technologies\n",
      "DB  - Ovid MEDLINE(R)\n",
      "AU  - Hirko KA\n",
      "AU  - Willett WC\n",
      "AU  - Hankinson SE\n",
      "AU  - Rosner BA\n",
      "AU  - Beck AH\n",
      "AU  - Tamimi RM\n",
      "AU  - Eliassen AH\n",
      "TI  - Healthy dietary patterns and risk of breast cancer by molecular subtype.\n",
      "AB  - We examined associations between dietary quality indices and breast cancer risk by molecular subtype among 100,643 women in the prospective Nurses' Health Study (NHS) cohort, followed from 1984 to 2006. Dietary quality scores for the Alternative Healthy Eating Index (AHEI), alternate Mediterranean diet (aMED), and Dietary Approaches to Stop Hypertension (DASH) dietary patterns were calculated from semi-quantitative food frequency questionnaires collected every 2-4 years. Breast cancer molecular subtypes were defined according to estrogen receptor (ER), progesterone receptor, human epidermal growth factor 2 (HER2), cytokeratin 5/6 (CK5/6), and epidermal growth factor receptor status from immunostained tumor microarrays in combination with histologic grade. Cox proportional hazards models, adjusted for age and breast cancer risk factors, were used to estimate hazard ratios (HRs) and 95 % confidence intervals (CIs). Competing risk analyses were used to assess heterogeneity by subtype. We did not observe any significant associations between the AHEI or aMED dietary patterns and risk of breast cancer by molecular subtype. However, a significantly reduced risk of HER2-type breast cancer was observed among women in 5th versus 1st quintile of the DASH dietary pattern [n = 134 cases, Q5 vs. Q1 HR (95 % CI) = 0.44 (0.25-0.77)], and the inverse trend across quintiles was significant (p trend = 0.02). We did not observe any heterogeneity in associations between AHEI (p het = 0.25), aMED (p het = 0.71), and DASH (p het = 0.12) dietary patterns and breast cancer by subtype. Adherence to the AHEI, aMED, and DASH dietary patterns was not strongly associated with breast cancer molecular subtypes.\n",
      "PT  - Journal Article\n",
      "PT  - Research Support, N.I.H., Extramural\n",
      "XL  - https://openathens.ovid.com/secure-ssl/home.oa?idpselect=https://kclidp.kcl.ac.uk/idp/shibboleth&entityID=https://kclidp.kcl.ac.uk/idp/shibboleth&?T=JS&CSC=Y&NEWS=N&PAGE=fulltext&D=medl&AN=26872903\n",
      "XL  - http://sfx.kcl.ac.uk/kings?sid=OVID:medline&id=pmid:&id=doi:10.1007%2Fs10549-016-3706-2&genre=article&atitle=Healthy+dietary+patterns+and+risk+of+breast+cancer+by+molecular+subtype.&title=Breast+Cancer+Research+%26+Treatment&issn=0167-6806&date=2016&volume=155&issue=3&spage=579&aulast=Hirko+KA&isbn=&__char_set=utf8\n",
      "\n",
      "<2. >\n",
      "VN  - Ovid Technologies\n",
      "DB  - Ovid MEDLINE(R)\n",
      "AU  - Bengtsson U\n",
      "AU  - Kjellgren K\n",
      "AU  - Hofer S\n",
      "AU  - Taft C\n",
      "AU  - Ring L\n",
      "TI  - Developing an interactive mobile phone self-report system for self-management of hypertension. Part 2: content validity and usability.\n",
      "AB  - Self-management support tools using technology may improve adherence to hypertension treatment. There is a need for user-friendly tools facilitating patients' understanding of the interconnections between blood pressure, wellbeing and lifestyle. This study aimed to examine comprehension, comprehensiveness and relevance of items, and further to evaluate the usability and reliability of an interactive hypertension-specific mobile phone self-report system. Areas important in supporting self-management and candidate items were derived from five focus group interviews with patients and healthcare professionals (n = 27), supplemented by a literature review. Items and response formats were drafted to meet specifications for mobile phone administration and were integrated into a mobile phone data-capture system. Content validity and usability were assessed iteratively in four rounds of cognitive interviews with patients (n = 21) and healthcare professionals (n = 4). Reliability was examined using a test-retest. Focus group analyses yielded six areas covered by 16 items. The cognitive interviews showed satisfactory item comprehension, relevance and coverage; however, one item was added. The mobile phone self-report system was reliable and perceived easy to use. The mobile phone self-report system appears efficiently to capture information relevant in patients' self-management of hypertension. Future studies need to evaluate the effectiveness of this tool in improving self-management of hypertension in clinical practice.\n",
      "PT  - Journal Article\n",
      "PT  - Research Support, Non-U.S. Gov't\n",
      "XL  - https://openathens.ovid.com/secure-ssl/home.oa?idpselect=https://kclidp.kcl.ac.uk/idp/shibboleth&entityID=https://kclidp.kcl.ac.uk/idp/shibboleth&?T=JS&CSC=Y&NEWS=N&PAGE=fulltext&D=medl&AN=24786778\n",
      "XL  - http://sfx.kcl.ac.uk/kings?sid=OVID:medline&id=pmid:&id=doi:10.3109%2F08037051.2014.901009&genre=article&atitle=Developing+an+interactive+mobile+phone+self-report+system+for+self-management+of+hypertension.+Part+2%3A+content+validity+and+usability.&title=Blood+Pressure&issn=0803-7051&date=2014&volume=23&issue=5&spage=296&aulast=Bengtsson+U&isbn=&__char_set=utf8\n"
     ]
    }
   ],
   "source": [
    "len(rct_sens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VN  - Ovid Technologies\n",
      "DB  - Ovid MEDLINE(R)\n",
      "AU  - Adelman EE\n",
      "AU  - Scott PA\n",
      "AU  - Skolarus LE\n",
      "AU  - Fox AK\n",
      "AU  - Frederiksen SM\n",
      "AU  - Meurer WJ\n",
      "TI  - Protocol Deviations before and after Treatment with Intravenous Tissue Plasminogen Activator in Community Hospitals.\n",
      "AB  - BACKGROUND: Protocol deviations before and after tissue plasminogen activator (tPA) treatment for ischemic stroke are common. It is unclear if patient or hospital factors predict protocol deviations. We examined predictors of protocol deviations and the effects of protocol violations on symptomatic intracerebral hemorrhage (sICH).\n",
      "AB  - METHODS: We used data from the Increasing Stroke Treatment through Interventional Behavior Change Tactics trial, a cluster-randomized, controlled trial evaluating the efficacy of a barrier assessment and educational intervention to increase appropriate tPA use in 24 Michigan community hospitals, to review tPA treatments between 2007 and 2010. Protocol violations were defined as deviations from the standard tPA protocol, both before and after treatment. Multilevel logistic regression models were fitted to determine if patient and hospital variables were associated with pretreatment or post-treatment protocol deviations.\n",
      "AB  - RESULTS: During the study, 557 patients (mean age 70, 52% male, median National Institutes of Health Stroke Scale score 12) were treated with tPA. Protocol deviations occurred in 233 (42%) patients: 16% had pretreatment deviations, 35% had post-treatment deviations, and 9% had both. The most common protocol deviations included elevated post-treatment blood pressure, antithrombotic agent use within 24 hours of treatment, and elevated pretreatment blood pressure. Protocol deviations were not associated with sICH, stroke severity, or hospital factors. Older age was associated with pretreatment protocol deviations (adjusted odds ratio [OR], .52; 95% confidence interval [CI], .30-.92). Pretreatment deviations were associated with post-treatment deviations (adjusted OR, 3.20; 95% CI, 1.91-5.35).\n",
      "AB  - CONCLUSIONS: Protocol deviations were not associated with sICH. Aside from age, patient and hospital factors were not associated with protocol deviations.Copyright © 2015 National Stroke Association. Published by Elsevier Inc. All rights reserved.\n",
      "PT  - Journal Article\n",
      "PT  - Research Support, N.I.H., Extramural\n",
      "XL  - https://openathens.ovid.com/secure-ssl/home.oa?idpselect=https://kclidp.kcl.ac.uk/idp/shibboleth&entityID=https://kclidp.kcl.ac.uk/idp/shibboleth&?T=JS&CSC=Y&NEWS=N&PAGE=fulltext&D=medl&AN=26419527\n",
      "XL  - http://sfx.kcl.ac.uk/kings?sid=OVID:medline&id=pmid:&id=doi:10.1016%2Fj.jstrokecerebrovasdis.2015.08.036&genre=article&atitle=Protocol+Deviations+before+and+after+Treatment+with+Intravenous+Tissue+Plasminogen+Activator+in+Community+Hospitals.&title=Journal+of+Stroke+%26+Cerebrovascular+Diseases&issn=1052-3057&date=2016&volume=25&issue=1&spage=67&aulast=Adelman+EE&isbn=&__char_set=utf8\n",
      "\n",
      "\n",
      "VN  - Ovid Technologies\n",
      "DB  - Ovid MEDLINE(R)\n",
      "AU  - Hooper LM\n",
      "AU  - Tomek S\n",
      "AU  - Roter D\n",
      "AU  - Carson KA\n",
      "AU  - Mugoya G\n",
      "AU  - Cooper LA\n",
      "TI  - Depression, patient characteristics, and attachment style: correlates and mediators of medication treatment adherence in a racially diverse primary care sample.\n",
      "AB  - BACKGROUND: The depth and breadth of problems related to depressive symptomatology and optimal treatment outcomes, including medication treatment adherence, have long been documented in the literature. Missing are clear explanations as to what factors and patient characteristics may account for lack of medication treatment adherence.\n",
      "AB  - OBJECTIVES: The two objectives of the current study were to examine the predictive strength of depression, patient characteristics, and patient attachment style regarding medication treatment adherence and to consider the extent to which attachment styles mediate the relation between depression and medication treatment adherence.\n",
      "AB  - METHOD: Participants in the present study were 237 racially diverse American primary care patients with a diagnosis of hypertension who were participants in a clinical trial. Depression, patient characteristics, attachment style, and medication treatment adherence were assessed.\n",
      "AB  - RESULTS: Partly consistent with our four hypotheses, the following results were found: (a) Black American, younger, never married, and poorer patients had lower medication treatment adherence (b) depression was significantly associated with lower self-reported medication adherence; (c) insecure-dismissing attachment style was related to lower medication adherence; and (d) insecure-dismissing attachment style mediates the relation between depression and medication treatment adherence by exacerbating the negative association.\n",
      "AB  - CONCLUSION: Physicians and other primary care providers should consider how depressive symptomatology, patient characteristics, and attachment style may inform the treatment plans they put forward and the extent to which \n"
     ]
    }
   ],
   "source": [
    "print(rcts_sens[:5000])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The best performing sensitive search has 3757949 articles\n"
     ]
    }
   ],
   "source": [
    "print(\"The best performing sensitive search has {} articles\".format(len(rcts_sens)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "rcts_prec = r.filter_articles(inp, filter_class='svm_cnn_ptyp', filter_type='precise')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The best performing precise search has 516 articles\n"
     ]
    }
   ],
   "source": [
    "print(\"The best performing precise search has {} articles\".format(len(rcts_prec)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Here is one...\n",
      "{'format': 'pubmed', 'title': 'Depression, patient characteristics, and attachment style: correlates and mediators of medication treatment adherence in a racially diverse primary care sample.', 'rct_ptyp': True, 'abstract': 'BACKGROUND: The depth and breadth of problems related to depressive symptomatology and optimal treatment outcomes, including medication treatment adherence, have long been documented in the literature. Missing are clear explanations as to what factors and patient characteristics may account for lack of medication treatment adherence. OBJECTIVES: The two objectives of the current study were to examine the predictive strength of depression, patient characteristics, and patient attachment style regarding medication treatment adherence and to consider the extent to which attachment styles mediate the relation between depression and medication treatment adherence. METHOD: Participants in the present study were 237 racially diverse American primary care patients with a diagnosis of hypertension who were participants in a clinical trial. Depression, patient characteristics, attachment style, and medication treatment adherence were assessed. RESULTS: Partly consistent with our four hypotheses, the following results were found: (a) Black American, younger, never married, and poorer patients had lower medication treatment adherence (b) depression was significantly associated with lower self-reported medication adherence; (c) insecure-dismissing attachment style was related to lower medication adherence; and (d) insecure-dismissing attachment style mediates the relation between depression and medication treatment adherence by exacerbating the negative association. CONCLUSION: Physicians and other primary care providers should consider how depressive symptomatology, patient characteristics, and attachment style may inform the treatment plans they put forward and the extent to which patients may adhere to those treatment plans.'}\n"
     ]
    }
   ],
   "source": [
    "print(\"Here is one...\")\n",
    "print(ris.simplify(rcts_prec[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}