geograph_corpus-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e3a6e9b9",
   "metadata": {},
   "source": [
    "## Introduction\n",
    "\n",
    "This notebook illustrates how we can use NER to search for placenames in a corpus, and enhance a gazetteer. It uses two datesets to illustrate the concepts.\n",
    "\n",
    "1) [Geograph](https://geograph.org.uk) \n",
    "This site invites users to take pictures in the UK and add descriptions. It has almost 7 million pictures, and the data are licenced using a CC By-SA licence, making them available for research as long as we keep the names of the users, and allow others to have access to any data we might create.\n",
    "\n",
    "2) [Ordnance Survey](https://ordnancesurvey.co.uk/) 50k gazetteer\n",
    "This gazetteer was published under a UK Open Government licence and contains all place name found on 1:50k maps in the UK. It is a legacy product (i.e. not used or updated any more), but it is suitable for our purposes.\n",
    "\n",
    "We are going to look for names found in the Geograph data that don't exist in the gazetteer. Since we know that many names occur multiple times, we will do this locally, to increase the chances that we really find new names.\n",
    "\n",
    "**The first block of our code reads in data and builds a simple spatial index for the gazetteer. We only need to do this once.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "d5a2314b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd #To use pandas for elegant data handling\n",
    "import spacy #Our NLP tools\n",
    "import math\n",
    "\n",
    "class Postings:\n",
    "    \n",
    "    def __init__(self, firstMondayTerms):\n",
    "        #Load a language model to do NLP\n",
    "        self.nlp = spacy.load(\"en_core_web_md\")\n",
    "        #First we read in the geograph data\n",
    "        geograph = pd.read_csv('./data/geograph_mini_corpus.csv', encoding='latin-1')\n",
    "        \n",
    "        sample = geograph.sample(n = 100)\n",
    "        self.ndocs = len(sample)\n",
    "        \n",
    "        \n",
    "        # firstMonday works like an inverse stop list, and we only use words in these lists for our posting file\n",
    "        if firstMondayTerms:\n",
    "            list = {}\n",
    "            elements = set(pd.read_csv('./data/elements.txt', header=None)[0])\n",
    "            qualities = set(pd.read_csv('./data/qualities.txt', header=None)[0])\n",
    "            activities = set(pd.read_csv('./data/activities.txt', header=None)[0])\n",
    "\n",
    "            terms = elements.union(qualities).union(activities)\n",
    "            lemmas = ' '.join(str(e) for e in terms)\n",
    "\n",
    "            doc = self.nlp(lemmas)\n",
    "            terms = set()\n",
    "            for token in doc:\n",
    "                terms.add(token.lemma_)\n",
    "                \n",
    "            # Now we process our corpus and create a postings file\n",
    "            docs = self.nlp.pipe(sample.text,n_process=2, batch_size=100)\n",
    "\n",
    "            self.postings = dict()\n",
    "\n",
    "            for (idxRow, s1), (_, s2) in zip(sample.iterrows(), enumerate(docs)):\n",
    "                id = s1.id\n",
    "                for token in s2:\n",
    "                    lemma = token.lemma_\n",
    "                    if lemma in terms:\n",
    "\n",
    "                        if lemma in self.postings:\n",
    "                            tf = self.postings[lemma]\n",
    "                            if id in tf:\n",
    "                                tf[id] = tf[id] + 1\n",
    "                            else:\n",
    "                                tf[id] = 1\n",
    "                        else:\n",
    "                            tf = {id: 1}\n",
    "                        self.postings[lemma] = tf\n",
    "                        \n",
    "    def tfIdf(self, query):\n",
    "        results = {}\n",
    "        qdoc = self.nlp(query)\n",
    "        for token in qdoc:\n",
    "            qt = token.lemma_\n",
    "            if qt in self.postings:\n",
    "                dc = len(self.postings[qt])\n",
    "                idf = math.log10(self.ndocs/(dc + 1))\n",
    "                for doc in self.postings[qt]:\n",
    "                    tf = self.postings[qt][doc]\n",
    "                    tfidf = tf * idf\n",
    "                    if doc in results:\n",
    "                        score = results[doc]\n",
    "                        results[doc] = tfidf + score\n",
    "                    else:\n",
    "                        results[doc] = tfidf\n",
    "        results = dict(sorted(results.items(), key = lambda x: x[1], reverse=True))\n",
    "        \n",
    "        return results\n",
    "                        \n",
    "                        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "c78ff881",
   "metadata": {},
   "outputs": [],
   "source": [
    "postings = Postings(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "3eb3c819",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{2775351: 2.3098039199714866,\n",
       " 885215: 1.1549019599857433,\n",
       " 650430: 1.1549019599857433,\n",
       " 1171797: 1.1549019599857433,\n",
       " 3113207: 1.1549019599857433,\n",
       " 969246: 1.1549019599857433,\n",
       " 210766: 0.6777807052660807,\n",
       " 849212: 0.6777807052660807,\n",
       " 1771153: 0.6777807052660807,\n",
       " 755207: 0.6777807052660807,\n",
       " 2004630: 0.6777807052660807,\n",
       " 54600: 0.6777807052660807,\n",
       " 1050188: 0.6777807052660807,\n",
       " 2784778: 0.6777807052660807,\n",
       " 2932828: 0.6777807052660807,\n",
       " 1744155: 0.6777807052660807,\n",
       " 3310901: 0.6777807052660807,\n",
       " 1144802: 0.6777807052660807,\n",
       " 407290: 0.6777807052660807,\n",
       " 3108258: 0.6777807052660807,\n",
       " 151138: 0.6777807052660807,\n",
       " 2161064: 0.6777807052660807,\n",
       " 2883804: 0.6777807052660807,\n",
       " 39454: 0.6777807052660807,\n",
       " 2226055: 0.6777807052660807,\n",
       " 1409271: 0.6777807052660807}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "postings.tfIdf('hill mountain')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "3ddd1d5e",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'sample' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_4468\\1685629626.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0msample\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'sample' is not defined"
     ]
    }
   ],
   "source": [
    "sample.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd15bdbe",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}