EAS244-CLST344-INTG344-semanticNetworks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk import sent_tokenize, word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer as wnl\n",
    "import nltk, gensim, re, string, glob\n",
    "from itertools import islice, compress\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy\n",
    "import networkx as nx\n",
    "nltk.download(\"punkt\")\n",
    "nltk.download(\"wordnet\")\n",
    "\n",
    "model = \"./enwiki_5_ner.txt\"\n",
    "word_vectors = gensim.models.KeyedVectors.load_word2vec_format(model, binary=False)\n",
    "\n",
    "#################################################\n",
    "# Initialize, config & define helpful functions #\n",
    "#################################################\n",
    "\n",
    "translator = str.maketrans('', '', string.punctuation.replace('-', '')) #filters punctuation except dash\n",
    "lemmatizeCondition = 1\n",
    "lemmatizer = wnl()\n",
    "\n",
    "# Function for finding index of words of interest, like 'references'\n",
    "\n",
    "def find(target):\n",
    "    for i, word in enumerate(sents):\n",
    "        try:\n",
    "            j = word.index(target)\n",
    "        except ValueError:\n",
    "            continue\n",
    "        yield i\n",
    "\n",
    "# Function for handling the input for gensim word2vec\n",
    "\n",
    "class FileToSent(object):\n",
    "    def __init__(self, filename):\n",
    "        self.filename = filename\n",
    "\n",
    "    def __iter__(self):\n",
    "        for line in open(self.filename, 'r'):\n",
    "            ll = line.strip().split(\",\")\n",
    "            ll = [''.join(c for c in s if c not in string.punctuation) for s in ll]\n",
    "            ll = [num.strip() for num in ll]\n",
    "            yield ll\n",
    "\n",
    "# Function for looking for element x occurs at least n times in list\n",
    "\n",
    "def check_list(lst, x, n):\n",
    "    gen = (True for i in lst if i==x)\n",
    "    return next(islice(gen, n-1, None), False)\n",
    "\n",
    "nltk.download('stopwords')\n",
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "newStopWords = ['fig', 'figure', 'table', 'may', 'also', 'zero', 'one', 'two', 'three', 'four', 'five', 'six',\n",
    "                'seven', 'eight', 'nine', 'ten', 'across', 'among', 'beside', 'however', 'yet', 'within']\n",
    "stop_words.extend(newStopWords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################################################\n",
    "# Read in .txt file(s) from a specified directory #\n",
    "###################################################\n",
    "\n",
    "IDs = glob.glob('./text/*')\n",
    "IDs_subIDs = []\n",
    "for ID in IDs:\n",
    "    IDs_subIDs += glob.glob(ID + '/*.txt')\n",
    "print(len(IDs)) # Print number of files read\n",
    "\n",
    "####################\n",
    "# Clean, lemmatize #\n",
    "####################\n",
    "\n",
    "for ID in IDs: # loop through papers\n",
    "    print(ID)\n",
    "    with open(ID) as paper:\n",
    "        text = paper.read()\n",
    "\n",
    "        ############\n",
    "        # Cleaning #\n",
    "        ############\n",
    "\n",
    "        text = re.sub(\"\\u2013|\\u2014\", \"-\", text) # Replace em-dashes\n",
    "        sents = sent_tokenize(text) # Split into sentences\n",
    "        sents = [word_tokenize(s) for s in sents]\n",
    "        sents = [[w.translate(translator) for w in s] for s in sents] # filter punctuation\n",
    "        sents = [[re.sub(r'\\d+','numeric', w) for w in s] for s in sents] # replace all numerals with the holder \"number\"\n",
    "        sents = [[w for w in s if re.search('[^a-zA-Z-]+', w) is None] for s in sents] # trips everything but alphabetic\n",
    "        sents = [[w.lower() for w in s] for s in sents] # make lower case\n",
    "        sents = [s for s in sents if len(s) > 0] # remove empty lines\n",
    "        sents = [[w for w in s if not w in stop_words] for s in sents]  # filter stop words\n",
    "        sents = [[w for w in s if len(w) > 1] for s in sents]  # filters out variables, etc\n",
    "        sents = [[w for w in s if len(w) > 2] for s in sents]  # filters out variables, etc\n",
    "        sents = [[w for w in s if len(w) > 3] for s in sents]  # filters out variables and abbreviations\n",
    "        sents = [s for s in sents if len(s) > 0]  # remove empty lines\n",
    "        words = [[lemmatizer.lemmatize(w) for w in s if lemmatizeCondition == 1] for s in sents] # lemmatize\n",
    "        words = list(itertools.chain.from_iterable(words)) # join list of lists\n",
    "        \n",
    "        # Write cleaned text to file\n",
    "        with open('./cleanText/cleanedText.txt', 'w') as f:\n",
    "            for _list in words:\n",
    "                f.write(str(_list) + ' ')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################\n",
    "# Construct semantic networks #\n",
    "###############################\n",
    "\n",
    "\"\"\"\n",
    "Code to make a network out of the shortest N cosine-distances (or, equivalently, the strongest N associations)\n",
    "between a set of words in a gensim word2vec model.\n",
    "\"\"\"\n",
    "\n",
    "model = word_vectors # load\n",
    "\n",
    "# Specify words\n",
    "###############\n",
    "\n",
    "my_words = []\n",
    "\n",
    "text = open('./cleanText/cleanedText.txt').read()\n",
    "\n",
    "for word in word_tokenize(text):  # append unique words in the whole corpus\n",
    "    print(word)\n",
    "    if word in my_words:\n",
    "        continue\n",
    "    else:\n",
    "        my_words.append(word)\n",
    "\n",
    "# filter out words not in model\n",
    "my_words = [word for word in my_words if word in model]\n",
    "\n",
    "# The number of connections we want: either as a factor of the number of words or a set number\n",
    "num_top_conns = len(my_words) * 19\n",
    "\n",
    "# Make a list of all word-to-word distances [each as a tuple of (word1,word2,dist)]\n",
    "dists=[]\n",
    "\n",
    "# Find similarity distances between each word pair\n",
    "\n",
    "for i1,word1 in enumerate(my_words):\n",
    "    for i2,word2 in enumerate(my_words):\n",
    "        if i1>=i2: continue\n",
    "        cosine_similarity = model.similarity(word1,word2)\n",
    "        cosine_distance = 1 - cosine_similarity\n",
    "        dist = (word1, word2, cosine_distance)\n",
    "        dists.append(dist)\n",
    "\n",
    "# Sort the list by ascending distance\n",
    "dists.sort(key=lambda _tuple: _tuple[-1])\n",
    "\n",
    "# Get the top connections\n",
    "top_conns = dists[:num_top_conns]\n",
    "\n",
    "# Make a network\n",
    "g = nx.Graph()\n",
    "g.add_nodes_from(my_words)\n",
    "for word1,word2,dist in top_conns:\n",
    "    weight = 1 - dist # cosine similarity for weight\n",
    "    g.add_edge(word1, word2, weight=float(weight))\n",
    "\n",
    "# Write the network\n",
    "nx.write_graphml(g, \"./semanticNetwork/semanticNetwork.graphml\") # Readable by Gephi\n",
    "\n",
    "A = nx.adjacency_matrix(g, nodelist=my_words, weight=float(weight))\n",
    "adjmat = A.todense()\n",
    "\n",
    "numpy.savetxt(\"./semanticNetwork/semanticNetworkAdjmat.txt\", adjmat, delimiter = ' ')\n",
    "\n",
    "with open('./semanticNetwork/semanticNetworkNodeLabels.txt', 'w') as f:\n",
    "    print(g.nodes, file=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################################\n",
    "# Construct co-occurrence network #\n",
    "###################################\n",
    "\n",
    "for ID in IDs:  # loop through papers\n",
    "    print(ID)\n",
    "    with open(ID) as paper:\n",
    "        text = paper.read()\n",
    "\n",
    "        ############\n",
    "        # Cleaning #\n",
    "        ############\n",
    "\n",
    "        text = re.sub(\"\\u2013|\\u2014\", \"-\", text)  # Replace em-dashes\n",
    "        sents = sent_tokenize(text)  # Split into sentences\n",
    "        sents = [word_tokenize(s) for s in sents]\n",
    "        sents = [[w.translate(translator) for w in s] for s in sents]  # filter punctuation\n",
    "        sents = [[re.sub(r'\\d+', 'numeric', w) for w in s] for s in\n",
    "                 sents]  # replace all numerals with the holder \"number\"\n",
    "        sents = [[w for w in s if re.search('[^a-zA-Z-]+', w) is None] for s in\n",
    "                 sents]  # trips everything but alphabetic\n",
    "        sents = [[w.lower() for w in s] for s in sents]  # make lower case\n",
    "        sents = [s for s in sents if len(s) > 0]  # remove empty lines\n",
    "        sents = [[w for w in s if not w in stop_words] for s in sents]  # filter stop words\n",
    "        sents = [[w for w in s if len(w) > 1] for s in sents]  # filters out variables, etc\n",
    "        sents = [[w for w in s if len(w) > 2] for s in sents]  # filters out variables, etc\n",
    "        sents = [[w for w in s if len(w) > 3] for s in sents]  # filters out variables and abbreviations\n",
    "        sents = [s for s in sents if len(s) > 0]  # remove empty lines\n",
    "        words = [[w for w in s] for s in sents]  # lemmatize\n",
    "        words = list(itertools.chain.from_iterable(words))  # join list of lists\n",
    "\n",
    "from collections import OrderedDict\n",
    "\n",
    "windows = []\n",
    "\n",
    "# For each sentence, retrieve 5-gram windows\n",
    "for sent in sents:\n",
    "    print(sent)\n",
    "    for i in list(range(len(sent))):\n",
    "        if len(sent) <= 5: # if the sentence is less than 5 words, just return that sentence\n",
    "            window_slice = sent\n",
    "            windows.append(window_slice)\n",
    "            break\n",
    "        else:\n",
    "            window_slice = sent[i:i + 5] # otherwise, return as many 5-grams as possible\n",
    "            if len(window_slice) == 5:\n",
    "                windows.append(window_slice)\n",
    "\n",
    "my_words = []\n",
    "\n",
    "for word in words:  # append unique words\n",
    "    print(word)\n",
    "    if word in my_words:\n",
    "        continue\n",
    "    else:\n",
    "        my_words.append(word)\n",
    "\n",
    "# Create an ordered dictionary that counts the occurrence of words\n",
    "# in a 5-gram sliding window\n",
    "\n",
    "document = windows\n",
    "names = my_words\n",
    "\n",
    "occurrences = OrderedDict((name, OrderedDict((name, 0) for name in names)) for name in names)\n",
    "\n",
    "# Find the co-occurrences:\n",
    "for l in document:\n",
    "    for i in range(len(l)):\n",
    "        for item in l[:i] + l[i + 1:]:\n",
    "            occurrences[l[i]][item] += 1\n",
    "\n",
    "# Print the matrix:\n",
    "print(' ', ' '.join(occurrences.keys()))\n",
    "for name, values in occurrences.items():\n",
    "    print(' '.join(str(i) for i in values.values()))\n",
    "\n",
    "# Save the data\n",
    "with open(\"./semanticNetwork/coOccurrenceMatrix.txt\", \"w\") as text_file:\n",
    "    for name, values in occurrences.items():\n",
    "        print(', '.join(str(i) for i in values.values()), file = text_file)\n",
    "\n",
    "with open(\"./semanticNetwork/coOccurrenceNodeLabels.txt\", \"w\") as text_file:\n",
    "    print(my_words, file = text_file)\n",
    "    \n",
    "A = numpy.genfromtxt(\"./semanticNetwork/coOccurrenceMatrix.txt\", delimiter=\",\", dtype=str,  converters={0:lambda x:x.decode()})\n",
    "\n",
    "g = nx.from_numpy_matrix(A)\n",
    "mapping=dict(zip(g.nodes(),my_words))\n",
    "g = nx.relabel_nodes(g, mapping)\n",
    "\n",
    "nx.write_gexf(g,'./semanticNetwork/coOccurrenceNetwork.gexf')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}