W10-2 Civil War Entities.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Civil War Entities: Extracting information using pos tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikipediaapi\n",
    "import re\n",
    "from string import punctuation\n",
    "import nltk\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the civil war corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wikipediaapi\n",
    "pages = [\n",
    "    \"American Civil War\",\n",
    "    \"Abraham Lincoln\",\n",
    "    \"Slavery in the United States\",\n",
    "    \"Slave states and free states\",\n",
    "    \"Emancipation Proclamation\",\n",
    "    \"Robert E. Lee\",\n",
    "    \"Ulysses S. Grant\",\n",
    "    \"Conclusion of the American Civil War\",\n",
    "    \"Origins of the American Civil War\",\n",
    "    \"Issues of the American Civil War\"\n",
    "]\n",
    "import re\n",
    "\n",
    "def underscorize(pagename):\n",
    "    return re.sub(\" \", \"_\", pagename)\n",
    "\n",
    "wiki_wiki = wikipediaapi.Wikipedia(\n",
    "        language='en',\n",
    "        extract_format=wikipediaapi.ExtractFormat.WIKI\n",
    ")\n",
    "page_dict = {}\n",
    "for page in pages:\n",
    "    pagename = underscorize(page)\n",
    "    print(pagename)\n",
    "    p_wiki = wiki_wiki.page(pagename)\n",
    "    page_text = p_wiki.text.split(\"\\n\")\n",
    "    page_paras = [para for para in page_text if len(para) > 1]\n",
    "    page_dict[pagename] = page_paras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chunking: Finding noun phrases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tag all of the sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "def tag_paragraph_sentences(para):\n",
    "    sentences = nltk.sent_tokenize(para)\n",
    "    tagged_sentences = []\n",
    "    for sent in sentences:\n",
    "        tokenized_sentence = nltk.word_tokenize(sent)\n",
    "        tagged_sentence = nltk.pos_tag(tokenized_sentence)\n",
    "        tagged_sentences.append(tagged_sentence)\n",
    "    return tagged_sentences\n",
    "tagged_sentences = []\n",
    "for name, page in page_dict.items():\n",
    "    for para in page:\n",
    "        tagged_sentences += tag_paragraph_sentences(para)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tagged_sentences[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tagged_sentences)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using NLTK's regular expression parser"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Construct the parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grammar = \"NounPhrase: {<DT>?<J.*>*<N.*>+}\"\n",
    "cp = nltk.RegexpParser(grammar)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the parser to one sentence.\n",
    "\n",
    "It produces an NLTK `tree` object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = cp.parse(tagged_sentences[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tagged_sentences[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tree)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extract the found nounphrases from the trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "np_dist = nltk.FreqDist()\n",
    "for i, sent in enumerate(tagged_sentences):\n",
    "    if i % 100 == 0:\n",
    "        clear_output(wait=True)\n",
    "        print('Sentence {} of {}'.format(i, len(tagged_sentences)))\n",
    "    tree = cp.parse(sent)\n",
    "    for i in tree:\n",
    "        if type(i) == nltk.Tree and i.label() == \"NounPhrase\":\n",
    "            np = \" \".join([token.lower() for token, pos in i.leaves()])\n",
    "            np_dist[np] += 1\n",
    "clear_output(wait=True)\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np_dist.most_common(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chunking: Finding named entities using NLTK's named expression chunker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "tree = nltk.ne_chunk(tagged_sentences[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(tree)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chunk them all. \n",
    "This will take a while"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "chunked_sentences = []\n",
    "for n, sent in enumerate(tagged_sentences):\n",
    "    if n % 500 == 0:\n",
    "        clear_output(wait=True)\n",
    "        print('Sentence {} of {}'.format(n, len(tagged_sentences)))\n",
    "    chunked_sentences.append(nltk.ne_chunk(sent))\n",
    "clear_output(wait=True)\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count them up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def entity_to_tuple(t):\n",
    "    return tuple([t.label(), \" \".join([token for token, pos in t.leaves()])])\n",
    "\n",
    "def extract_entities(chunked_sentence):\n",
    "    entities = []\n",
    "    for i in chunked_sentence:\n",
    "        if type(i) == nltk.Tree:\n",
    "            entities.append(entity_to_tuple(i))\n",
    "    return entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_fdist = nltk.FreqDist()\n",
    "for n, chunked_sent in enumerate(chunked_sentences):\n",
    "    entities = extract_entities(chunked_sent)\n",
    "    entity_fdist.update(entities)\n",
    "entity_fdist.most_common(25)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find relations between named entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def matching_label(it, label):\n",
    "    if type(it) == nltk.Tree:\n",
    "        if it.label() == label:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "def matching_entity(it, entity):\n",
    "    if type(it) == nltk.Tree:\n",
    "        if entity == \"*\" or \" \".join(entity_to_tuple(it)) == entity:\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "def get_matching_chunks(the_tree, e1, e2):\n",
    "    in_match = False\n",
    "    matches = []\n",
    "    current_match = []\n",
    "    for k in the_tree:\n",
    "        if in_match:\n",
    "            if matching_entity(k, e2):\n",
    "                current_match.append(entity_to_tuple(k))\n",
    "                matches.append(current_match)\n",
    "                in_match = False\n",
    "            else:\n",
    "                current_match.append(k)\n",
    "        else:\n",
    "            if matching_entity(k, e1):\n",
    "                in_match = True\n",
    "                current_match = [entity_to_tuple(k)]\n",
    "    return matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "e1 = \"PERSON Grant\"\n",
    "e2 = \"PERSON Lincoln\"\n",
    "relations = []\n",
    "for n, chunked_sentence in enumerate(chunked_sentences):\n",
    "    if n % 25 == 0:\n",
    "        clear_output(wait=True)\n",
    "        print('Sentence {} of {}'.format(n, len(chunked_sentences)))\n",
    "    relations += (get_matching_chunks(chunked_sentence, e1, e2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(relations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "relations[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RelationList(list):\n",
    "    def get_one_relation(self, rel):\n",
    "        html = \"<b>{}</b>\".format(rel[0][1])\n",
    "        for row in rel[1:-1]:\n",
    "            html += \" \" + str(row[0])         \n",
    "        html += \" \" + \"<b>{}</b>\".format(rel[-1][1])\n",
    "        return html\n",
    "    \n",
    "    def _repr_html_(self):\n",
    "        html = \"\"\n",
    "        for rel in self:\n",
    "            html += \"<div>\" + self.get_one_relation(rel) + \"</div>\"\n",
    "        return html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RelationList(relations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_relation(rel):\n",
    "    outstring = rel[0][1]\n",
    "    for it in rel[1:-1]:\n",
    "        outstring += \" \" + str(it[0])\n",
    "    outstring += \" \" + rel[-1][1]\n",
    "    print(outstring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for relation in relations:\n",
    "    print_relation(relation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}