advanced-nlp-prepoc-r2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re,pickle,nltk.data,os,progressbar,time,gensim\n",
    "import pandas as pd\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "from gensim.models import word2vec\n",
    "from nltk.corpus import wordnet\n",
    "from gensim.models import Phrases\n",
    "from gensim.models.phrases import Phraser\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### <b> 1. Load Data And Write Texts To Individual .txt Files </b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Read data from files \n",
    "# train = pd.read_csv(\"data/train.csv\", header=0)\n",
    "# test = pd.read_csv( \"data/test.csv\", header=0)\n",
    "# print(\"Read %d labeled train reviews and  %d unlabelled test reviews\" % (len(train),len(test)))\n",
    "\n",
    "# train_comments = train['comment_text'].fillna(\"_na_\").tolist()\n",
    "# test_comments = test['comment_text'].fillna(\"_na_\").tolist()\n",
    "# all_comments = train['comment_text'].fillna(\"_na_\").tolist() + test['comment_text'].fillna(\"_na_\").tolist() \n",
    "\n",
    "# def comments_to_txtfile(comment_list,output_file):\n",
    "#     with open(output_file, \"w+\") as output:\n",
    "#         for comment in comment_list:\n",
    "#             comment = \" \".join(comment.split()).replace('\"',\"\").replace('|',\"\").replace('{',\"\").replace('}',\"\")\n",
    "#             comment = \" \".join(comment.split()).replace('\"',\"\")\n",
    "#             comment = re.sub(\"[^a-zA-Z0-9 ]\",\"\",str(comment)) #Remove unneeded whitespace and punctuation and move to lowercase\n",
    "#             output.write(\"%s\\n\" % str(comment))\n",
    "#     return print(\"%s Written\" % output_file)\n",
    "\n",
    "# comments_to_txtfile(train_comments,'data/raw_comments/train_comments.csv')\n",
    "# comments_to_txtfile(test_comments,'data/raw_comments/test_comments.csv')\n",
    "# comments_to_txtfile(all_comments,'data/raw_comments/all_comments.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nlp_utility(input_file, \n",
    "                embeddings_file ='',\n",
    "                fasttext_file = '',\n",
    "                remove_stops = True,\n",
    "                lemmatize = True, #Must lemmatize to be able to remove proper nouns - can be fixed in future\n",
    "                remove_proper_nouns = True,\n",
    "                n_grams = 0, \n",
    "                fasttext_spellcheck = True,\n",
    "                filter_embeddings_file = False,\n",
    "                max_vocabulary = 0\n",
    "                ):\n",
    "    \n",
    "    import os,shutil,smart_open,gensim,sys,time,re\n",
    "    from gensim.models import Word2Vec\n",
    "    from sys import platform\n",
    "    from nltk.corpus import stopwords\n",
    "    from nltk.stem import WordNetLemmatizer\n",
    "    from nltk.corpus import wordnet\n",
    "    from nltk import pos_tag\n",
    "    from gensim.models import FastText\n",
    "    stops = set(stopwords.words(\"english\"))\n",
    "    \n",
    "    unfiltered_glove_path = 'w2v_models/unfiltered_models/' + embeddings_file\n",
    "    filtered_glove_path =  'w2v_models/filtered_models/' + 'vocab=' + str(max_vocabulary) + '.remove_pn=' + str(remove_proper_nouns) + '.remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + embeddings_file\n",
    "    gensim_filtered_glove_path = 'w2v_models/gensim_filtered_models/' + 'vocab=' +  str(max_vocabulary) + '.remove_pn=' + str(remove_proper_nouns) + '.remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + embeddings_file\n",
    "    output_file = 'data/tokenized_comments/' + 'vocab=' + str(max_vocabulary) + '.remove_pn=' + str(remove_proper_nouns) + '.remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + embeddings_file + '.' + input_file.split('/')[2]\n",
    "    print('Result File = %s' % output_file)\n",
    "    #Create a generator function to run through the input file one line at a time:\n",
    "    class FileToComments(object):    \n",
    "        def __init__(self, filename):\n",
    "            self.filename = filename\n",
    "        def __iter__(self):    \n",
    "            for line in open(self.filename, 'r'):\n",
    "                yield line   \n",
    "    #Function used in Step 2 to match treebank tags to pos tags \n",
    "    def get_wordnet_pos(treebank_tag):\n",
    "        if treebank_tag.startswith('J'):\n",
    "            return wordnet.ADJ\n",
    "        elif treebank_tag.startswith('V'):\n",
    "            return wordnet.VERB\n",
    "        elif treebank_tag.startswith('N'):\n",
    "            return wordnet.NOUN\n",
    "        elif treebank_tag.startswith('R'):\n",
    "            return wordnet.ADV\n",
    "        else:\n",
    "            return ''\n",
    "    def filter_vectors_gensim(unfiltered_glove_file,comment_file):\n",
    "        unfiltered_glove_path = 'w2v_models/unfiltered_models/' + unfiltered_glove_file\n",
    "        filtered_glove_path =  'w2v_models/filtered_models/' + 'vocab=' + str(max_vocabulary) + '.remove_pn=' + str(remove_proper_nouns) + '.remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + unfiltered_glove_file\n",
    "        gensim_filtered_glove_path = 'w2v_models/gensim_filtered_models/' +'vocab=' + str(max_vocabulary) + '.remove_pn=' + str(remove_proper_nouns) + '.remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + unfiltered_glove_file\n",
    "        comment_streamer = FileToComments(comment_file)\n",
    "\n",
    "        #Create a Vocabulary From Our Existing Words\n",
    "        from collections import defaultdict\n",
    "        import codecs, gensim\n",
    "        vocab = defaultdict(int)\n",
    "        for sentence in comment_streamer:\n",
    "            sentence = sentence.split()\n",
    "            for word in sentence:\n",
    "                vocab[word] += 1\n",
    "        vocab = set(vocab)\n",
    "        print('Vocabulary set, starting to filter word embeddings')\n",
    "        nread = 0\n",
    "        nwrote = 0\n",
    "        with codecs.open(unfiltered_glove_path, encoding='utf-8') as f:\n",
    "            with codecs.open(filtered_glove_path, 'w', encoding='utf-8') as out:\n",
    "                for line in f:\n",
    "                    nread += 1\n",
    "                    line = line.strip()\n",
    "                    if not line: continue\n",
    "                    if line.split(u' ', 1)[0] in vocab:\n",
    "                        out.write(line + '\\n')\n",
    "                        nwrote += 1\n",
    "                    if (nwrote >= max_vocabulary) & (max_vocabulary > 0):\n",
    "                        sys.stdout.write(\"\\r\\n\")\n",
    "                        print('read %s lines, wrote %s' % (nread, nwrote))\n",
    "                        print('Prepending Gensim W2V Line Count and Embedding Size')\n",
    "                        gensim.scripts.glove2word2vec.glove2word2vec(filtered_glove_path,gensim_filtered_glove_path)\n",
    "                        return gensim_filtered_glove_path\n",
    "                    else:\n",
    "                        sys.stdout.write(\"Read: %d lines. Wrote %d To Word Embedding File  \\r\" % (nread,nwrote))\n",
    "                        sys.stdout.flush()\n",
    "        sys.stdout.write(\"\\r\\n\")\n",
    "        print('read %s lines, wrote %s' % (nread, nwrote))\n",
    "        print('Prepending Gensim W2V Line Count and Embedding Size')\n",
    "        gensim.scripts.glove2word2vec.glove2word2vec(filtered_glove_path,gensim_filtered_glove_path)\n",
    "        return gensim_filtered_glove_path\n",
    "\n",
    "    #Step 1 - Split Words And Make Lowercase\n",
    "    comment_streamer = FileToComments(input_file)\n",
    "    counter = 0\n",
    "    entities_removed = 0\n",
    "    punctuation = re.compile(r'[►*/-@.?!~,\":#$;\\'()=+|0-9]') #[punctuation and numbers to remove]\n",
    "    with open('intermediate_comments_preproc.txt', \"w+\") as output:\n",
    "        for comment in comment_streamer:\n",
    "            comment =punctuation.sub(\"\", comment)\n",
    "            comment = comment.lower().split()\n",
    "            #Step 1 - Remove Stopwords\n",
    "            if remove_stops:\n",
    "                comment = [word for word in comment if not word in stops]\n",
    "            #Step 2 - Lemmatize\n",
    "            if lemmatize:\n",
    "                tagged_words = pos_tag(comment)\n",
    "                tokenized_words = []                           \n",
    "                for word in tagged_words:\n",
    "                    if (remove_proper_nouns) & ((word[1] == \"NNP\") | (word[1] == \"NNPS\")):\n",
    "                        entities_removed += 1\n",
    "                    else:\n",
    "                        thisTag = get_wordnet_pos(word[1])\n",
    "                        if thisTag == '':\n",
    "                            tokenized_words.append(word[0])\n",
    "                        else:\n",
    "                            thisLemma = WordNetLemmatizer().lemmatize(word[0],thisTag)\n",
    "                            tokenized_words.append(thisLemma)\n",
    "                output.write(\"%s\\n\" % ' '.join(tokenized_words))\n",
    "            else:\n",
    "                output.write(\"%s\\n\" % ' '.join(comment))\n",
    "\n",
    "            counter += 1\n",
    "            sys.stdout.write(\"Cleaned %d comments from %s. Removed %d entities. \\r\" % (counter,input_file,entities_removed))\n",
    "            sys.stdout.flush()\n",
    "        sys.stdout.write(\"\\r\\n\")\n",
    "\n",
    "    # Step 4 - Filter Word Embedding File (For Speed In Training)\n",
    "    # We now have our commenst rewritten in a clean format with stop words removed and lemmatization in place\n",
    "    # Now we need to take our chosen vector_model and sort it for words found in our sentences\n",
    "    if filter_embeddings_file:\n",
    "        filtered_vectors_path = filter_vectors_gensim(embeddings_file,comment_file = 'intermediate_comments_preproc.txt')\n",
    "        print('Embedding file filtered and cleaned for corpora. Location : %s' % gensim_filtered_glove_path)\n",
    "        return\n",
    "    else:\n",
    "        filtered_vectors_path = gensim_filtered_glove_path\n",
    "    \n",
    "    #Step 5- Use Fasttext Model For Spelling Correction\n",
    "    if fasttext_spellcheck:\n",
    "        #Load Glove Model\n",
    "        glove_word_vectors = gensim.models.KeyedVectors.load_word2vec_format(filtered_vectors_path)\n",
    "        print(\"Filtered Gensim Model Loaded\")\n",
    "        #Load Fasttext Model\n",
    "        fasttext_vectors = FastText.load_fasttext_format(fasttext_file)\n",
    "        print(\"Fasttext Model Loaded\")\n",
    "\n",
    "        intermediate_token_streamer = FileToComments('intermediate_comments_preproc.txt')\n",
    "        model_vocabulary = set(glove_word_vectors.wv.vocab)\n",
    "        word_subs = 0\n",
    "        counter = 0\n",
    "        print(\"Substituting mispelled words using Fasttext Model\")                                      \n",
    "        with open(output_file, \"w+\") as output:\n",
    "            for wordlist in intermediate_token_streamer:\n",
    "                wordlist = wordlist.split()\n",
    "                output_words = []\n",
    "                for word in wordlist:\n",
    "                    if word in model_vocabulary:\n",
    "                        output_words.append(word)\n",
    "                    else:\n",
    "                        try:\n",
    "                            for pairing in fasttext_vectors.wv.most_similar(word,topn=3):\n",
    "                                if pairing[0] in model_vocabulary:\n",
    "                                    output_words.append(pairing[0])\n",
    "                                    word_subs += 1\n",
    "                        except:\n",
    "                            next\n",
    "                output.write(\"%s\\n\" % ' '.join(output_words))\n",
    "                counter += 1\n",
    "                sys.stdout.write(\"Substituted %d Mispelled Words from %d comments\\r\" % (word_subs,counter))\n",
    "                sys.stdout.flush()\n",
    "        print(\"Substituted %s Words. Printed to %s\" % (word_subs,output_file))\n",
    "    \n",
    "    else:\n",
    "        intermediate_token_streamer = FileToComments('intermediate_comments_preproc.txt')\n",
    "        print(\"Saving to Result File\")                                      \n",
    "        with open(output_file, \"w+\") as output:\n",
    "            for wordlist in intermediate_token_streamer:\n",
    "                wordlist = wordlist.split()\n",
    "                output.write(\"%s\\n\" % ' '.join(wordlist))\n",
    "        print(\"Printed to %s\" % (output_file))        \n",
    "    \n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available Models:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['glove.42B.300d.txt',\n",
       " 'googlenews.100B.300d.bin',\n",
       " 'glove.840B.300d.txt',\n",
       " '.ipynb_checkpoints',\n",
       " 'glove.twitter.27B.200d.txt']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('Available Models:')\n",
    "os.listdir('w2v_models/unfiltered_models/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set Params For Preprocessing\n",
    "remove_stops = True\n",
    "lemmatize = True\n",
    "fasttext_spellcheck = False\n",
    "remove_proper_nouns = True\n",
    "embeddings_file = 'glove.840B.300d.txt'\n",
    "fasttext_file = 'fasttext-training/word_vector_models/cbow_fasttext_model.bin'\n",
    "max_vocab = 100000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result File = data/tokenized_comments/vocab=100000.remove_pn=True.remove_stops=True.lemmatize=True.spellcheck=False.glove.840B.300d.txt.all_comments.csv\n",
      "Cleaned 322849 comments from data/raw_comments/all_comments.csv. Removed 16400 entities. \n",
      "Vocabulary set, starting to filter word embeddings\n",
      "Read: 1014048 lines. Wrote 99999 To Word Embedding File  \n",
      "read 1014049 lines, wrote 100000\n",
      "Prepending Gensim W2V Line Count and Embedding Size\n",
      "Embedding file filtered and cleaned for corpora. Location : w2v_models/gensim_filtered_models/vocab=100000.remove_pn=True.remove_stops=True.lemmatize=True.spellcheck=False.glove.840B.300d.txt\n"
     ]
    }
   ],
   "source": [
    "#Filter Vectors\n",
    "nlp_utility(input_file = 'data/raw_comments/all_comments.csv',\n",
    "            embeddings_file =embeddings_file,\n",
    "            fasttext_file = fasttext_file,\n",
    "            remove_stops = remove_stops,\n",
    "            remove_proper_nouns = remove_proper_nouns,\n",
    "            lemmatize = lemmatize,\n",
    "            fasttext_spellcheck = fasttext_spellcheck,\n",
    "            filter_embeddings_file = True,\n",
    "            max_vocabulary = max_vocab\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result File = data/tokenized_comments/vocab=100000.remove_pn=True.remove_stops=True.lemmatize=True.spellcheck=False.glove.840B.300d.txt.train_comments.csv\n",
      "Cleaned 95851 comments from data/raw_comments/train_comments.csv. Removed 3294 entities. \n",
      "Saving to Result File\n",
      "Printed to data/tokenized_comments/vocab=100000.remove_pn=True.remove_stops=True.lemmatize=True.spellcheck=False.glove.840B.300d.txt.train_comments.csv\n"
     ]
    }
   ],
   "source": [
    "nlp_utility(input_file = 'data/raw_comments/train_comments.csv',\n",
    "            embeddings_file = embeddings_file,\n",
    "            fasttext_file = fasttext_file,\n",
    "            remove_stops = remove_stops,\n",
    "            lemmatize = lemmatize,\n",
    "            remove_proper_nouns = remove_proper_nouns,\n",
    "            fasttext_spellcheck = fasttext_spellcheck,\n",
    "            max_vocabulary = max_vocab\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result File = data/tokenized_comments/vocab=100000.remove_pn=True.remove_stops=True.lemmatize=True.spellcheck=False.glove.840B.300d.txt.test_comments.csv\n",
      "Cleaned 226998 comments from data/raw_comments/test_comments.csv. Removed 13106 entities. \n",
      "Saving to Result File\n",
      "Printed to data/tokenized_comments/vocab=100000.remove_pn=True.remove_stops=True.lemmatize=True.spellcheck=False.glove.840B.300d.txt.test_comments.csv\n"
     ]
    }
   ],
   "source": [
    "nlp_utility(input_file = 'data/raw_comments/test_comments.csv',\n",
    "            embeddings_file = embeddings_file,\n",
    "            fasttext_file = fasttext_file,\n",
    "            remove_stops = remove_stops,\n",
    "            lemmatize = lemmatize,\n",
    "            remove_proper_nouns = remove_proper_nouns,\n",
    "            fasttext_spellcheck = fasttext_spellcheck,\n",
    "            max_vocabulary = max_vocab\n",
    "           )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}