word2vec-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "#from bs4 import BeautifulSoup\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from gensim.models import word2vec\n",
    "import pickle\n",
    "import nltk.data\n",
    "import os\n",
    "# Load the punkt tokenizer\n",
    "tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "File b'../data/train.csv' does not exist",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-288ee8bba2e5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# Read data from files\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrain\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"../data/train.csv\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mtest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m \u001b[0;34m\"../data/test.csv\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# Verify the number of comments that were read\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jmoore/Desktop/tensorflow-gpu/tensorflow-gpu/lib/python3.5/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, skipfooter, skip_footer, doublequote, delim_whitespace, as_recarray, compact_ints, use_unsigned, low_memory, buffer_lines, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    707\u001b[0m                     skip_blank_lines=skip_blank_lines)\n\u001b[1;32m    708\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 709\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    710\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    711\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jmoore/Desktop/tensorflow-gpu/tensorflow-gpu/lib/python3.5/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    447\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    450\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    451\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jmoore/Desktop/tensorflow-gpu/tensorflow-gpu/lib/python3.5/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    816\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    817\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 818\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    819\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    820\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jmoore/Desktop/tensorflow-gpu/tensorflow-gpu/lib/python3.5/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1047\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1048\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1049\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1050\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1051\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/jmoore/Desktop/tensorflow-gpu/tensorflow-gpu/lib/python3.5/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1693\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'allow_leading_cols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex_col\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1694\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1695\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1696\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1697\u001b[0m         \u001b[0;31m# XXX\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: File b'../data/train.csv' does not exist"
     ]
    }
   ],
   "source": [
    "# Read data from files \n",
    "train = pd.read_csv(\"../data/train.csv\", header=0)\n",
    "test = pd.read_csv( \"../data/test.csv\", header=0)\n",
    "\n",
    "# Verify the number of comments that were read\n",
    "print(\"Read %d labeled train reviews and  %d unlabelled test reviews\" % (len(train),len(test)))\n",
    "train_comments = train['comment_text'].fillna(\"_na_\").tolist()\n",
    "test_comments = test['comment_text'].fillna(\"_na_\").tolist()\n",
    "all_comments = train['comment_text'].fillna(\"_na_\").tolist() + test['comment_text'].fillna(\"_na_\").tolist() \n",
    "\n",
    "with open(\"data/raw_comments/all_comments.csv\", \"w+\") as comments_file:\n",
    "    i=0\n",
    "    for comment in all_comments:\n",
    "        comment = re.sub(\"[^a-zA-Z]\",\" \",str(comment))\n",
    "        comments_file.write(\"%s\\n\" % comment)\n",
    "        \n",
    "with open(\"data/raw_comments/train_comments.csv\", \"w+\") as comments_file:\n",
    "    i=0\n",
    "    for comment in train_comments:\n",
    "        comment = re.sub(\"[^a-zA-Z]\",\" \",str(comment))\n",
    "        comments_file.write(\"%s\\n\" % comment)\n",
    "        \n",
    "with open(\"data/raw_comments/test_comments.csv\", \"w+\") as comments_file:\n",
    "    i=0\n",
    "    for comment in test_comments:\n",
    "        comment = re.sub(\"[^a-zA-Z]\",\" \",str(comment))\n",
    "        comments_file.write(\"%s\\n\" % comment)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 1. Preprocess Text Reviews For Creating Word Vectors"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The <b>sentences</b> iterable can be simply a list, but for larger corpora, consider a generator that streams the sentences directly from disk/network, without storing everything in RAM. See BrownCorpus, Text8Corpus or LineSentence in the gensim.models.word2vec module for such examples.\n",
    "\n",
    "<b>min_count</b> ignore all words and bigrams with total collected count lower than this.\n",
    "\n",
    "<b>threshold</b> represents a score threshold for forming the phrases (higher means fewer phrases). A phrase of words a followed by b is accepted if the score of the phrase is greater than threshold. see the scoring setting.\n",
    "\n",
    "<b>max_vocab_size</b> is the maximum size of the vocabulary. Used to control pruning of less common words, to keep memory under control. The default of 40M needs about 3.6GB of RAM; increase/decrease max_vocab_size depending on how much available memory you have.\n",
    "\n",
    "<b>delimiter</b> is the glue character used to join collocation tokens, and should be a byte string (e.g. b’_’).\n",
    "\n",
    "<b>scoring</b> specifies how potential phrases are scored for comparison to the threshold setting. scoring can be set with either a string that refers to a built-in scoring function, or with a function with the expected parameter names. Two built-in scoring functions are available by setting scoring to a string:\n",
    "\n",
    "‘default’: from “Efficient Estimaton of Word Representations in Vector Space” by\n",
    "Mikolov, et. al.: (count(worda followed by wordb) - min_count) * N / (count(worda) * count(wordb)) > threshold`, where <b>N</b> is the total vocabulary size.\n",
    "<b>npmi</b>: normalized pointwise mutual information, from “Normalized (Pointwise) Mutual\n",
    "Information in Colocation Extraction” by Gerlof Bouma: ln(prop(worda followed by wordb) / (prop(worda)*prop(wordb))) / - ln(prop(worda followed by wordb) where prop(n) is the count of n / the count of everything in the entire corpus.\n",
    "‘npmi’ is more robust when dealing with common words that form part of common bigrams, and ranges from -1 to 1, but is slower to calculate than the default.\n",
    "\n",
    "To use a custom scoring function, create a function with the following parameters and set the scoring parameter to the custom function. You must use all the parameters in your function call, even if the function does not require all the parameters.\n",
    "\n",
    "<b>worda_count</b>: number of occurrances in sentences of the first token in the phrase being scored wordb_count: number of occurrances in sentences of the second token in the phrase being scored bigram_count: number of occurrances in sentences of the phrase being scored len_vocab: the number of unique tokens in sentences min_count: the min_count setting of the Phrases class corpus_word_count: the total number of (non-unique) tokens in sentences\n",
    "A scoring function without any of these parameters (even if the parameters are not used) will raise a ValueError on initialization of the Phrases class. The scoring function must be picklable.\n",
    "\n",
    "common_terms is an optionnal list of “stop words” that won’t affect frequency count of expressions containing them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FileToComments(object):    \n",
    "    def __init__(self, filename):\n",
    "        self.filename = filename\n",
    "        self.stop = set(nltk.corpus.stopwords.words('english'))\n",
    "        \n",
    "    def __iter__(self):\n",
    "        \n",
    "        def comment_to_wordlist(comment, remove_stopwords=True):\n",
    "            comment = str(comment)\n",
    "            words = comment.lower().split()\n",
    "            #if remove_stopwords:\n",
    "            #    stops = set(stopwords.words(\"english\"))\n",
    "            #    words = [w for w in words if not w in stops]\n",
    "            return(words)\n",
    "    \n",
    "        for line in open(self.filename, 'r'):\n",
    "            #line = unicode(line, 'utf-8')\n",
    "            tokenized_comment = comment_to_wordlist(line, tokenizer)\n",
    "            yield tokenized_comment\n",
    "        \n",
    "all_comments = FileToComments('data/raw_comments/all_comments.csv')\n",
    "train_comments = FileToComments('data/raw_comments/train_comments.csv')\n",
    "test_comments = FileToComments('data/raw_comments/test_comments.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-12-29 12:21:56,677 : INFO : collecting all words and their counts\n",
      "2017-12-29 12:21:56,679 : INFO : PROGRESS: at sentence #0, processed 0 words and 0 word types\n",
      "2017-12-29 12:21:59,229 : INFO : PROGRESS: at sentence #10000, processed 676828 words and 287566 word types\n",
      "2017-12-29 12:22:01,831 : INFO : PROGRESS: at sentence #20000, processed 1351623 words and 489953 word types\n",
      "2017-12-29 12:22:04,331 : INFO : PROGRESS: at sentence #30000, processed 2029376 words and 665776 word types\n",
      "2017-12-29 12:22:07,023 : INFO : PROGRESS: at sentence #40000, processed 2728907 words and 830315 word types\n",
      "2017-12-29 12:22:09,500 : INFO : PROGRESS: at sentence #50000, processed 3397508 words and 977689 word types\n",
      "2017-12-29 12:22:12,016 : INFO : PROGRESS: at sentence #60000, processed 4076917 words and 1119668 word types\n",
      "2017-12-29 12:22:14,524 : INFO : PROGRESS: at sentence #70000, processed 4757328 words and 1255084 word types\n",
      "2017-12-29 12:22:17,144 : INFO : PROGRESS: at sentence #80000, processed 5455470 words and 1387281 word types\n",
      "2017-12-29 12:22:19,781 : INFO : PROGRESS: at sentence #90000, processed 6141473 words and 1511470 word types\n",
      "2017-12-29 12:22:22,627 : INFO : PROGRESS: at sentence #100000, processed 6892434 words and 1637372 word types\n",
      "2017-12-29 12:22:25,657 : INFO : PROGRESS: at sentence #110000, processed 7716745 words and 1764071 word types\n",
      "2017-12-29 12:22:28,509 : INFO : PROGRESS: at sentence #120000, processed 8482838 words and 1882437 word types\n",
      "2017-12-29 12:22:31,410 : INFO : PROGRESS: at sentence #130000, processed 9270991 words and 1994342 word types\n",
      "2017-12-29 12:22:34,245 : INFO : PROGRESS: at sentence #140000, processed 10036555 words and 2101876 word types\n",
      "2017-12-29 12:22:36,996 : INFO : PROGRESS: at sentence #150000, processed 10788413 words and 2205591 word types\n",
      "2017-12-29 12:22:39,747 : INFO : PROGRESS: at sentence #160000, processed 11529315 words and 2306730 word types\n",
      "2017-12-29 12:22:42,702 : INFO : PROGRESS: at sentence #170000, processed 12299785 words and 2413530 word types\n",
      "2017-12-29 12:22:45,641 : INFO : PROGRESS: at sentence #180000, processed 13085207 words and 2520428 word types\n",
      "2017-12-29 12:22:48,664 : INFO : PROGRESS: at sentence #190000, processed 13898461 words and 2625670 word types\n",
      "2017-12-29 12:22:51,469 : INFO : PROGRESS: at sentence #200000, processed 14656345 words and 2723973 word types\n",
      "2017-12-29 12:22:54,425 : INFO : PROGRESS: at sentence #210000, processed 15413798 words and 2821352 word types\n",
      "2017-12-29 12:22:57,132 : INFO : PROGRESS: at sentence #220000, processed 16172340 words and 2914579 word types\n",
      "2017-12-29 12:22:59,861 : INFO : PROGRESS: at sentence #230000, processed 16923670 words and 3008707 word types\n",
      "2017-12-29 12:23:02,756 : INFO : PROGRESS: at sentence #240000, processed 17704977 words and 3103640 word types\n",
      "2017-12-29 12:23:05,418 : INFO : PROGRESS: at sentence #250000, processed 18448435 words and 3196137 word types\n",
      "2017-12-29 12:23:08,521 : INFO : PROGRESS: at sentence #260000, processed 19300751 words and 3287328 word types\n",
      "2017-12-29 12:23:11,327 : INFO : PROGRESS: at sentence #270000, processed 20057891 words and 3380678 word types\n",
      "2017-12-29 12:23:14,187 : INFO : PROGRESS: at sentence #280000, processed 20862979 words and 3474121 word types\n",
      "2017-12-29 12:23:16,924 : INFO : PROGRESS: at sentence #290000, processed 21621780 words and 3560832 word types\n",
      "2017-12-29 12:23:19,788 : INFO : PROGRESS: at sentence #300000, processed 22407858 words and 3652206 word types\n",
      "2017-12-29 12:23:22,837 : INFO : PROGRESS: at sentence #310000, processed 23241411 words and 3745113 word types\n",
      "2017-12-29 12:23:25,680 : INFO : PROGRESS: at sentence #320000, processed 24032710 words and 3832965 word types\n",
      "2017-12-29 12:23:26,515 : INFO : collected 3857811 word types from a corpus of 24259579 words (unigram + bigrams) and 322849 sentences\n",
      "2017-12-29 12:23:26,516 : INFO : using 3857811 counts as vocab in Phrases<0 vocab, min_count=30, threshold=15, max_vocab_size=40000000>\n",
      "2017-12-29 12:23:26,884 : INFO : source_vocab length 3857811\n",
      "2017-12-29 12:25:09,207 : INFO : Phraser built with 4080 4080 phrasegrams\n"
     ]
    }
   ],
   "source": [
    "from gensim.models import Phrases\n",
    "from gensim.models.phrases import Phraser\n",
    "# Train Tokenizer on all comments\n",
    "bigram = Phrases(all_comments, min_count=30, threshold=15)\n",
    "bigram_phraser = Phraser(bigram) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tokens cleaned\n"
     ]
    }
   ],
   "source": [
    "train_tokens = [bigram_phraser[comment] for comment in train_comments]\n",
    "test_tokens = [bigram_phraser[comment] for comment in test_comments]\n",
    "all_tokens = [bigram_phraser[comment] for comment in all_comments]\n",
    "\n",
    "stops = set(stopwords.words(\"english\"))\n",
    "\n",
    "clean_train_tokens = []\n",
    "for token in train_tokens:\n",
    "    words = [w for w in token if not w in stops]\n",
    "    clean_train_tokens += [words]\n",
    "    \n",
    "clean_test_tokens = []\n",
    "for token in test_tokens:\n",
    "    words = [w for w in token if not w in stops]\n",
    "    clean_test_tokens += [words]\n",
    "\n",
    "clean_all_tokens = []\n",
    "for token in all_tokens:\n",
    "    words = [w for w in token if not w in stops]\n",
    "    clean_all_tokens += [words]\n",
    "print('tokens cleaned')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "files saved to pickle data/tokenized_comments/...\n"
     ]
    }
   ],
   "source": [
    "#Pickle the tokens file for further use\n",
    "import pickle\n",
    "with open('data/tokenized_comments/tokenized_test_comments.pickle', 'wb') as filename:\n",
    "    pickle.dump(clean_test_tokens, filename, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "with open('data/tokenized_comments/tokenized_train_comments.pickle', 'wb') as filename:\n",
    "    pickle.dump(clean_train_tokens, filename, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "with open('data/tokenized_comments/tokenized_all_comments.pickle', 'wb') as filename:\n",
    "    pickle.dump(clean_all_tokens, filename, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "print('files saved to pickle data/tokenized_comments/...')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2. Training and Saving Your Model\n",
    "\n",
    "With the list of nicely parsed sentences, we're ready to train the model. There are a number of parameter choices that affect the run time and the quality of the final model that is produced. For details on the algorithms below, see the word2vec API documentation as well as the Google documentation. \n",
    "\n",
    " - Architecture: Architecture options are skip-gram (default) or continuous bag of words. We found that skip-gram was very slightly slower but produced better results. \n",
    " - Training algorithm: Hierarchical softmax (default) or negative sampling. For us, the default worked well.\n",
    " - Downsampling of frequent words: The Google documentation recommends values between .00001 and .001. For us, values closer 0.001 seemed to improve the accuracy of the final model.\n",
    " - Word vector dimensionality: More features result in longer runtimes, and often, but not always, result in better models. Reasonable values can be in the tens to hundreds; we used 300. \n",
    " - Context / window size: How many words of context should the training algorithm take into account? 10 seems to work well for hierarchical softmax (more is better, up to a point).\n",
    " - Worker threads: Number of parallel processes to run. This is computer-specific, but between 4 and 6 should work on most systems.\n",
    " - Minimum word count: This helps limit the size of the vocabulary to meaningful words. Any word that does not occur at least this many times across all documents is ignored. Reasonable values could be between 10 and 100. In this case, since each movie occurs 30 times, we set the minimum word count to 40, to avoid attaching too much importance to individual movie titles. This resulted in an overall vocabulary size of around 15,000 words. Higher values also help limit run time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-12-29 12:31:15,777 : INFO : collecting all words and their counts\n",
      "2017-12-29 12:31:15,778 : INFO : PROGRESS: at sentence #0, processed 0 words, keeping 0 word types\n",
      "2017-12-29 12:31:15,884 : INFO : PROGRESS: at sentence #10000, processed 332303 words, keeping 35481 word types\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-12-29 12:31:15,988 : INFO : PROGRESS: at sentence #20000, processed 664751 words, keeping 52404 word types\n",
      "2017-12-29 12:31:16,097 : INFO : PROGRESS: at sentence #30000, processed 998240 words, keeping 66195 word types\n",
      "2017-12-29 12:31:16,205 : INFO : PROGRESS: at sentence #40000, processed 1342842 words, keeping 78285 word types\n",
      "2017-12-29 12:31:16,310 : INFO : PROGRESS: at sentence #50000, processed 1672684 words, keeping 89015 word types\n",
      "2017-12-29 12:31:16,412 : INFO : PROGRESS: at sentence #60000, processed 2007133 words, keeping 98715 word types\n",
      "2017-12-29 12:31:16,515 : INFO : PROGRESS: at sentence #70000, processed 2341001 words, keeping 107515 word types\n",
      "2017-12-29 12:31:16,620 : INFO : PROGRESS: at sentence #80000, processed 2684149 words, keeping 116238 word types\n",
      "2017-12-29 12:31:16,723 : INFO : PROGRESS: at sentence #90000, processed 3020627 words, keeping 124056 word types\n",
      "2017-12-29 12:31:16,838 : INFO : PROGRESS: at sentence #100000, processed 3398527 words, keeping 132573 word types\n",
      "2017-12-29 12:31:16,966 : INFO : PROGRESS: at sentence #110000, processed 3830665 words, keeping 141592 word types\n",
      "2017-12-29 12:31:17,085 : INFO : PROGRESS: at sentence #120000, processed 4230437 words, keeping 150523 word types\n",
      "2017-12-29 12:31:17,208 : INFO : PROGRESS: at sentence #130000, processed 4638940 words, keeping 158220 word types\n",
      "2017-12-29 12:31:17,334 : INFO : PROGRESS: at sentence #140000, processed 5037621 words, keeping 166017 word types\n",
      "2017-12-29 12:31:17,453 : INFO : PROGRESS: at sentence #150000, processed 5430026 words, keeping 173705 word types\n",
      "2017-12-29 12:31:17,592 : INFO : PROGRESS: at sentence #160000, processed 5814654 words, keeping 181318 word types\n",
      "2017-12-29 12:31:17,731 : INFO : PROGRESS: at sentence #170000, processed 6219979 words, keeping 189442 word types\n",
      "2017-12-29 12:31:17,858 : INFO : PROGRESS: at sentence #180000, processed 6628605 words, keeping 196903 word types\n",
      "2017-12-29 12:31:18,002 : INFO : PROGRESS: at sentence #190000, processed 7050797 words, keeping 204479 word types\n",
      "2017-12-29 12:31:18,134 : INFO : PROGRESS: at sentence #200000, processed 7447170 words, keeping 211687 word types\n",
      "2017-12-29 12:31:18,252 : INFO : PROGRESS: at sentence #210000, processed 7841060 words, keeping 218143 word types\n",
      "2017-12-29 12:31:18,373 : INFO : PROGRESS: at sentence #220000, processed 8237093 words, keeping 224674 word types\n",
      "2017-12-29 12:31:18,501 : INFO : PROGRESS: at sentence #230000, processed 8631028 words, keeping 231610 word types\n",
      "2017-12-29 12:31:18,636 : INFO : PROGRESS: at sentence #240000, processed 9038117 words, keeping 238239 word types\n",
      "2017-12-29 12:31:18,773 : INFO : PROGRESS: at sentence #250000, processed 9426690 words, keeping 244940 word types\n",
      "2017-12-29 12:31:18,911 : INFO : PROGRESS: at sentence #260000, processed 9862408 words, keeping 250833 word types\n",
      "2017-12-29 12:31:19,032 : INFO : PROGRESS: at sentence #270000, processed 10259517 words, keeping 257969 word types\n",
      "2017-12-29 12:31:19,162 : INFO : PROGRESS: at sentence #280000, processed 10676016 words, keeping 264403 word types\n",
      "2017-12-29 12:31:19,283 : INFO : PROGRESS: at sentence #290000, processed 11069543 words, keeping 270111 word types\n",
      "2017-12-29 12:31:19,406 : INFO : PROGRESS: at sentence #300000, processed 11477811 words, keeping 276188 word types\n",
      "2017-12-29 12:31:19,532 : INFO : PROGRESS: at sentence #310000, processed 11912148 words, keeping 282692 word types\n",
      "2017-12-29 12:31:19,653 : INFO : PROGRESS: at sentence #320000, processed 12322648 words, keeping 288663 word types\n",
      "2017-12-29 12:31:19,690 : INFO : collected 290436 word types from a corpus of 12441325 raw words and 322849 sentences\n",
      "2017-12-29 12:31:19,691 : INFO : Loading a fresh vocabulary\n",
      "2017-12-29 12:31:19,979 : INFO : min_count=20 retains 28600 unique words (9% of original 290436, drops 261836)\n",
      "2017-12-29 12:31:19,980 : INFO : min_count=20 leaves 11749654 word corpus (94% of original 12441325, drops 691671)\n",
      "2017-12-29 12:31:20,154 : INFO : deleting the raw counts dictionary of 290436 items\n",
      "2017-12-29 12:31:20,168 : INFO : sample=0.001 downsamples 24 most-common words\n",
      "2017-12-29 12:31:20,169 : INFO : downsampling leaves estimated 10648730 word corpus (90.6% of prior 11749654)\n",
      "2017-12-29 12:31:20,170 : INFO : estimated required memory for 28600 words and 300 dimensions: 82940000 bytes\n",
      "2017-12-29 12:31:20,328 : INFO : resetting layer weights\n",
      "2017-12-29 12:31:21,036 : INFO : training model with 16 workers on 28600 vocabulary and 300 features, using sg=0 hs=0 sample=0.001 negative=5 window=10\n",
      "2017-12-29 12:31:22,062 : INFO : PROGRESS: at 1.10% examples, 524601 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:23,105 : INFO : PROGRESS: at 2.35% examples, 559922 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:31:24,108 : INFO : PROGRESS: at 3.58% examples, 569861 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:31:25,125 : INFO : PROGRESS: at 4.92% examples, 590999 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:26,196 : INFO : PROGRESS: at 6.20% examples, 594595 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:27,225 : INFO : PROGRESS: at 7.38% examples, 603404 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:31:28,250 : INFO : PROGRESS: at 8.51% examples, 603380 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:29,258 : INFO : PROGRESS: at 9.75% examples, 609585 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:30,267 : INFO : PROGRESS: at 10.83% examples, 606533 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:31,281 : INFO : PROGRESS: at 12.07% examples, 613654 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:31:32,291 : INFO : PROGRESS: at 13.27% examples, 616257 words/s, in_qsize 31, out_qsize 1\n",
      "2017-12-29 12:31:33,305 : INFO : PROGRESS: at 14.46% examples, 616829 words/s, in_qsize 31, out_qsize 2\n",
      "2017-12-29 12:31:34,323 : INFO : PROGRESS: at 15.62% examples, 616461 words/s, in_qsize 31, out_qsize 3\n",
      "2017-12-29 12:31:35,339 : INFO : PROGRESS: at 16.79% examples, 617846 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:36,389 : INFO : PROGRESS: at 17.97% examples, 617951 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:31:37,404 : INFO : PROGRESS: at 19.20% examples, 622888 words/s, in_qsize 32, out_qsize 0\n",
      "2017-12-29 12:31:38,409 : INFO : PROGRESS: at 20.36% examples, 622354 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:31:39,429 : INFO : PROGRESS: at 21.77% examples, 625051 words/s, in_qsize 32, out_qsize 0\n",
      "2017-12-29 12:31:40,432 : INFO : PROGRESS: at 23.06% examples, 625596 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:41,448 : INFO : PROGRESS: at 24.47% examples, 627956 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:42,462 : INFO : PROGRESS: at 25.71% examples, 627151 words/s, in_qsize 32, out_qsize 1\n",
      "2017-12-29 12:31:43,475 : INFO : PROGRESS: at 26.86% examples, 627735 words/s, in_qsize 32, out_qsize 1\n",
      "2017-12-29 12:31:44,489 : INFO : PROGRESS: at 28.12% examples, 630097 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:45,489 : INFO : PROGRESS: at 29.30% examples, 630183 words/s, in_qsize 29, out_qsize 0\n",
      "2017-12-29 12:31:46,563 : INFO : PROGRESS: at 30.55% examples, 629706 words/s, in_qsize 26, out_qsize 5\n",
      "2017-12-29 12:31:47,577 : INFO : PROGRESS: at 31.67% examples, 629360 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:31:48,579 : INFO : PROGRESS: at 32.87% examples, 629933 words/s, in_qsize 30, out_qsize 0\n",
      "2017-12-29 12:31:49,585 : INFO : PROGRESS: at 34.07% examples, 630211 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:50,588 : INFO : PROGRESS: at 35.34% examples, 631626 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:51,628 : INFO : PROGRESS: at 36.55% examples, 632296 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:52,643 : INFO : PROGRESS: at 37.71% examples, 632327 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:31:53,671 : INFO : PROGRESS: at 38.90% examples, 633087 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:54,685 : INFO : PROGRESS: at 40.08% examples, 633320 words/s, in_qsize 32, out_qsize 0\n",
      "2017-12-29 12:31:55,706 : INFO : PROGRESS: at 41.48% examples, 634135 words/s, in_qsize 32, out_qsize 0\n",
      "2017-12-29 12:31:56,729 : INFO : PROGRESS: at 42.87% examples, 635317 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:31:57,760 : INFO : PROGRESS: at 44.21% examples, 635112 words/s, in_qsize 32, out_qsize 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2017-12-29 12:31:58,762 : INFO : PROGRESS: at 45.62% examples, 636825 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:31:59,784 : INFO : PROGRESS: at 46.76% examples, 636224 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:00,786 : INFO : PROGRESS: at 47.99% examples, 637385 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:01,805 : INFO : PROGRESS: at 49.21% examples, 637537 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:02,809 : INFO : PROGRESS: at 50.41% examples, 637494 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:32:03,818 : INFO : PROGRESS: at 51.60% examples, 637980 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:32:04,824 : INFO : PROGRESS: at 52.76% examples, 637733 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:05,831 : INFO : PROGRESS: at 54.00% examples, 638108 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:06,846 : INFO : PROGRESS: at 55.12% examples, 637213 words/s, in_qsize 29, out_qsize 2\n",
      "2017-12-29 12:32:07,893 : INFO : PROGRESS: at 56.24% examples, 636071 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:08,895 : INFO : PROGRESS: at 57.16% examples, 633120 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:09,904 : INFO : PROGRESS: at 58.35% examples, 633837 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:10,923 : INFO : PROGRESS: at 59.53% examples, 634260 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:32:11,932 : INFO : PROGRESS: at 60.84% examples, 634852 words/s, in_qsize 32, out_qsize 0\n",
      "2017-12-29 12:32:12,951 : INFO : PROGRESS: at 62.16% examples, 634854 words/s, in_qsize 31, out_qsize 2\n",
      "2017-12-29 12:32:14,009 : INFO : PROGRESS: at 63.51% examples, 634528 words/s, in_qsize 29, out_qsize 2\n",
      "2017-12-29 12:32:15,025 : INFO : PROGRESS: at 64.93% examples, 635598 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:16,028 : INFO : PROGRESS: at 66.18% examples, 635659 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:17,045 : INFO : PROGRESS: at 67.38% examples, 636007 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:18,053 : INFO : PROGRESS: at 68.60% examples, 636640 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:19,074 : INFO : PROGRESS: at 69.90% examples, 637210 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:32:20,085 : INFO : PROGRESS: at 71.14% examples, 637813 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:21,151 : INFO : PROGRESS: at 72.41% examples, 638185 words/s, in_qsize 32, out_qsize 1\n",
      "2017-12-29 12:32:22,157 : INFO : PROGRESS: at 73.64% examples, 638544 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:32:23,164 : INFO : PROGRESS: at 74.85% examples, 638686 words/s, in_qsize 29, out_qsize 2\n",
      "2017-12-29 12:32:24,171 : INFO : PROGRESS: at 76.12% examples, 639549 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:25,177 : INFO : PROGRESS: at 77.29% examples, 639465 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:26,193 : INFO : PROGRESS: at 78.51% examples, 639803 words/s, in_qsize 31, out_qsize 1\n",
      "2017-12-29 12:32:27,245 : INFO : PROGRESS: at 79.77% examples, 640482 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:32:28,250 : INFO : PROGRESS: at 81.05% examples, 640364 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:29,254 : INFO : PROGRESS: at 82.38% examples, 640567 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:30,309 : INFO : PROGRESS: at 83.74% examples, 640388 words/s, in_qsize 31, out_qsize 2\n",
      "2017-12-29 12:32:31,358 : INFO : PROGRESS: at 85.24% examples, 641318 words/s, in_qsize 32, out_qsize 3\n",
      "2017-12-29 12:32:32,372 : INFO : PROGRESS: at 86.51% examples, 641673 words/s, in_qsize 30, out_qsize 1\n",
      "2017-12-29 12:32:33,380 : INFO : PROGRESS: at 87.75% examples, 642218 words/s, in_qsize 32, out_qsize 1\n",
      "2017-12-29 12:32:34,389 : INFO : PROGRESS: at 88.93% examples, 642297 words/s, in_qsize 31, out_qsize 3\n",
      "2017-12-29 12:32:35,428 : INFO : PROGRESS: at 90.23% examples, 642458 words/s, in_qsize 28, out_qsize 3\n",
      "2017-12-29 12:32:36,431 : INFO : PROGRESS: at 91.47% examples, 642971 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:37,439 : INFO : PROGRESS: at 92.76% examples, 643795 words/s, in_qsize 31, out_qsize 1\n",
      "2017-12-29 12:32:38,486 : INFO : PROGRESS: at 94.10% examples, 644330 words/s, in_qsize 32, out_qsize 1\n",
      "2017-12-29 12:32:39,497 : INFO : PROGRESS: at 95.40% examples, 644827 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:40,517 : INFO : PROGRESS: at 96.64% examples, 645290 words/s, in_qsize 32, out_qsize 0\n",
      "2017-12-29 12:32:41,548 : INFO : PROGRESS: at 97.88% examples, 645524 words/s, in_qsize 32, out_qsize 2\n",
      "2017-12-29 12:32:42,563 : INFO : PROGRESS: at 99.09% examples, 645883 words/s, in_qsize 31, out_qsize 0\n",
      "2017-12-29 12:32:43,210 : INFO : worker thread finished; awaiting finish of 15 more threads\n",
      "2017-12-29 12:32:43,226 : INFO : worker thread finished; awaiting finish of 14 more threads\n",
      "2017-12-29 12:32:43,251 : INFO : worker thread finished; awaiting finish of 13 more threads\n",
      "2017-12-29 12:32:43,260 : INFO : worker thread finished; awaiting finish of 12 more threads\n",
      "2017-12-29 12:32:43,262 : INFO : worker thread finished; awaiting finish of 11 more threads\n",
      "2017-12-29 12:32:43,263 : INFO : worker thread finished; awaiting finish of 10 more threads\n",
      "2017-12-29 12:32:43,270 : INFO : worker thread finished; awaiting finish of 9 more threads\n",
      "2017-12-29 12:32:43,292 : INFO : worker thread finished; awaiting finish of 8 more threads\n",
      "2017-12-29 12:32:43,311 : INFO : worker thread finished; awaiting finish of 7 more threads\n",
      "2017-12-29 12:32:43,317 : INFO : worker thread finished; awaiting finish of 6 more threads\n",
      "2017-12-29 12:32:43,324 : INFO : worker thread finished; awaiting finish of 5 more threads\n",
      "2017-12-29 12:32:43,336 : INFO : worker thread finished; awaiting finish of 4 more threads\n",
      "2017-12-29 12:32:43,337 : INFO : worker thread finished; awaiting finish of 3 more threads\n",
      "2017-12-29 12:32:43,347 : INFO : worker thread finished; awaiting finish of 2 more threads\n",
      "2017-12-29 12:32:43,348 : INFO : worker thread finished; awaiting finish of 1 more threads\n",
      "2017-12-29 12:32:43,354 : INFO : worker thread finished; awaiting finish of 0 more threads\n",
      "2017-12-29 12:32:43,355 : INFO : training on 62206625 raw words (53157807 effective words) took 82.3s, 645873 effective words/s\n",
      "2017-12-29 12:32:43,376 : INFO : precomputing L2-norms of word weight vectors\n",
      "2017-12-29 12:32:43,869 : INFO : saving Word2Vec object under models/300features_20minwords_10context, separately None\n",
      "2017-12-29 12:32:43,870 : INFO : not storing attribute syn0norm\n",
      "2017-12-29 12:32:43,871 : INFO : not storing attribute cum_table\n",
      "2017-12-29 12:32:44,791 : INFO : saved models/300features_20minwords_10context\n"
     ]
    }
   ],
   "source": [
    "#Load Pre-saved tokenized comments\n",
    "with open('data/tokenized_comments/tokenized_all_comments.pickle', 'rb') as filename:\n",
    "    all_comments = pickle.load(filename)\n",
    "    \n",
    "import logging\n",
    "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',\\\n",
    "    level=logging.INFO)\n",
    "\n",
    "\n",
    "# Set values for various parameters\n",
    "num_features = 300    # Word vector dimensionality                      \n",
    "min_word_count = 20   # Minimum word count                        \n",
    "num_workers = 16       # Number of threads to run in parallel\n",
    "context = 10          # Context window size                                                                                    \n",
    "downsampling = 1e-3   # Downsample setting for frequent words\n",
    "\n",
    "# Initialize and train the model (this will take some time)\n",
    "print(\"Training model...\")\n",
    "model = word2vec.Word2Vec(all_comments,\n",
    "                          workers=num_workers,\n",
    "                          size=num_features,\n",
    "                          min_count = min_word_count,\n",
    "                          window = context,\n",
    "                          sample = downsampling\n",
    "                         )\n",
    "\n",
    "# init_sims will make the model much more memory-efficient.\n",
    "model.init_sims(replace=True)\n",
    "model_name = \"models/%sfeatures_%sminwords_%scontext\" % (num_features,min_word_count,context)\n",
    "model.save(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}